1
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
3
* Copyright (C) 1998-2004 Gerwin Klein <lsf@jflex.de> *
4
* All rights reserved. *
6
* This program is free software; you can redistribute it and/or modify *
7
* it under the terms of the GNU General Public License. See the file *
8
* COPYRIGHT for more information. *
10
* This program is distributed in the hope that it will be useful, *
11
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
12
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
13
* GNU General Public License for more details. *
15
* You should have received a copy of the GNU General Public License along *
16
* with this program; if not, write to the Free Software Foundation, Inc., *
17
* 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA *
19
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
23
import java_cup.runtime.Symbol;
24
import java.util.Vector;
26
import java.util.Stack;
31
* Generated by <a href="http://www.jflex.de/">JFlex</a>.
33
* @author Gerwin Klein
34
* @version JFlex 1.4.1, $Revision: 1.1 $, $Date: 2008/05/09 09:14:10 $
41
%implements sym, java_cup.runtime.Scanner
52
%state COMMENT, STATELIST, MACROS, REGEXPSTART
53
%state REGEXP, JAVA_CODE, STATES, STRING_CONTENT
54
%state CHARCLASS, COPY, REPEATEXP, EATWSPNL
60
int commentbalance = 0;
62
int bufferSize = 16384;
65
Stack files = new Stack();
67
StringBuffer userCode = new StringBuffer();
76
String scanErrorException;
77
String cupSymbol = "sym";
79
StringBuffer actionText = new StringBuffer();
80
StringBuffer string = new StringBuffer();
85
boolean cupCompatible;
94
boolean lookAheadUsed;
98
boolean useRowMap = Options.gen_method == Options.PACK || Options.gen_method == Options.TABLE;
99
boolean packed = Options.gen_method == Options.PACK;
101
boolean inclusive_states;
104
String isImplementing;
106
String className = "Yylex";
109
String visibility = "public";
111
LexicalStates states = new LexicalStates();
113
Vector actions = new Vector();
115
private int nextState;
117
boolean macroDefinition;
119
Timer t = new Timer();
121
public int currentLine() {
125
public void setFile(File file) {
129
private Symbol symbol(int type, Object value) {
130
return new Symbol(type, yyline, yycolumn, value);
133
private Symbol symbol(int type) {
134
return new Symbol(type, yyline, yycolumn);
137
// updates line and column count to the beginning of the first
138
// non whitespace character in yytext, but leaves yyline+yycolumn
140
private Symbol symbol_countUpdate(int type, Object value) {
143
String text = yytext();
145
for (int i=0; i < text.length(); i++) {
146
char c = text.charAt(i);
148
if (c != '\n' && c != '\r' && c != ' ' && c != '\t' )
149
return new Symbol(type, lc, cc, value);
159
return new Symbol(type, yyline, yycolumn, value);
162
private String makeMacroIdent() {
163
String matched = yytext().trim();
164
return matched.substring(1, matched.length()-1).trim();
167
public static String conc(Object a, Object b) {
168
if (a == null && b == null) return null;
169
if (a == null) return b.toString();
170
if (b == null) return a.toString();
172
return a.toString()+b.toString();
175
public static String concExc(Object a, Object b) {
176
if (a == null && b == null) return null;
177
if (a == null) return b.toString();
178
if (b == null) return a.toString();
180
return a.toString()+", "+b.toString();
185
states.insert("YYINITIAL", true);
190
HexDigit = [0-9a-fA-F]
194
HexNumber = \\ x {HexDigit} {2}
195
Unicode = \\ u {HexDigit} {1, 4}
196
OctNumber = \\ [0-3]? {OctDigit} {1, 2}
198
// see http://www.unicode.org/unicode/reports/tr18/
200
WSPNL = [\u2028\u2029\u000A\u000B\u000C\u000D\u0085\t\b\ ]
201
NL = [\u2028\u2029\u000A\u000B\u000C\u000D\u0085] | \u000D\u000A
202
NNL = [^\u2028\u2029\u000A\u000B\u000C\u000D\u0085]
204
Ident = {IdentStart} {IdentPart}*
205
QualIdent = {Ident} ( {WSP}* "." {WSP}* {Ident} )*
206
QUIL = {QualIdent} ( {WSP}* "," {WSP}* {QualIdent} )*
207
Array = "[" {WSP}* "]"
208
ArrType = {QualIdent} ({WSP}* {Array})*
210
IdentStart = [:jletter:]
211
IdentPart = [:jletterdigit:]
213
JFlexCommentChar = [^*/]|"/"+[^*/]|"*"+[^*/]
214
JFlexComment = {JFlexCommentChar}*
217
JavaComment = {TraditionalComment}|{EndOfLineComment}
218
TraditionalComment = "/*"{CommentContent}\*+"/"
219
EndOfLineComment = "//".*{NL}
221
CommentContent = ([^*]|\*+[^*/])*
223
StringCharacter = [^\u2028\u2029\u000A\u000B\u000C\u000D\u0085\"\\]
225
CharLiteral = \'([^\u2028\u2029\u000A\u000B\u000C\u000D\u0085\'\\]|{EscapeSequence})\'
226
StringLiteral = \"({StringCharacter}|{EscapeSequence})*\"
228
EscapeSequence = \\[^\u2028\u2029\u000A\u000B\u000C\u000D\u0085]|\\+u{HexDigit}{4}|\\[0-3]?{OctDigit}{1,2}
230
/* \\(b|t|n|f|r|\"|\'|\\|[0-3]?{OctDigit}{1,2}|u{HexDigit}{4}) */
232
JavaRest = [^\{\}\"\'/]|"/"[^*/]
233
JavaCode = ({JavaRest}|{StringLiteral}|{CharLiteral}|{JavaComment})+
241
macroDefinition = true;
242
return symbol(USERCODE,userCode);
244
.*{NL} { userCode.append(yytext()); }
245
.* { return symbol(EOF); }
248
<MACROS> ("%{"|"%init{"|"%initthrow{"|"%eof{"|"%eofthrow{"|"%yylexthrow{"|"%eofval{").*{NL}
249
{ string.setLength(0); yybegin(COPY); }
251
"%}".*{NL} { classCode = conc(classCode,string); yybegin(MACROS); }
252
"%init}".*{NL} { initCode = conc(initCode,string); yybegin(MACROS); }
253
"%initthrow}".*{NL} { initThrow = concExc(initThrow,string); yybegin(MACROS); }
254
"%eof}".*{NL} { eofCode = conc(eofCode,string); yybegin(MACROS); }
255
"%eofthrow}".*{NL} { eofThrow = concExc(eofThrow,string); yybegin(MACROS); }
256
"%yylexthrow}".*{NL} { lexThrow = concExc(lexThrow,string); yybegin(MACROS); }
257
"%eofval}".*{NL} { eofVal = string.toString(); yybegin(MACROS); }
259
.*{NL} { string.append(yytext()); }
261
<<EOF>> { throw new ScannerException(file,ErrorMessages.EOF_IN_MACROS); }
265
<MACROS> ^"%s" ("tate" "s"?)? {WSP}+ { inclusive_states = true; yybegin(STATELIST); }
266
<MACROS> ^"%x" ("state" "s"?)? {WSP}+ { inclusive_states = false; yybegin(STATELIST); }
268
{Ident} { states.insert(yytext(),inclusive_states); }
269
([\ \t]*","[\ \t]*)|([\ \t]+) { }
270
{NL} { yybegin(MACROS); }
271
<<EOF>> { throw new ScannerException(file,ErrorMessages.EOF_IN_MACROS); }
275
"%char" { charCount = true; }
276
"%line" { lineCount = true; }
277
"%column" { columnCount = true; }
278
"%byaccj" { isInteger = true;
280
eofVal = "return 0;";
283
"%cup" { cupCompatible = true;
284
isImplementing = concExc(isImplementing, "java_cup.runtime.Scanner");
285
if (functionName == null)
286
functionName = "next_token";
287
if (tokenType == null)
288
tokenType = "java_cup.runtime.Symbol";
290
eofVal = "return new java_cup.runtime.Symbol("+cupSymbol+".EOF);";
291
if (!Options.jlex) eofclose = true;
293
"%cupsym"{WSP}+{QualIdent} {WSP}* { cupSymbol = yytext().substring(8).trim();
294
if (cupCompatible) Out.warning(ErrorMessages.CUPSYM_AFTER_CUP, yyline); }
295
"%cupsym"{WSP}+{NNL}* { throw new ScannerException(file,ErrorMessages.QUIL_CUPSYM, yyline); }
296
"%cupdebug" { cupDebug = true; }
297
"%eofclose"({WSP}+"true")? { eofclose = true; }
298
"%eofclose"({WSP}+"false") { eofclose = false; }
299
"%class"{WSP}+{Ident} {WSP}* { className = yytext().substring(7).trim(); }
300
"%function"{WSP}+{Ident} {WSP}* { functionName = yytext().substring(10).trim(); }
301
"%type"{WSP}+{ArrType} {WSP}* { tokenType = yytext().substring(6).trim(); }
302
"%integer"|"%int" { isInteger = true; }
303
"%intwrap" { isIntWrap = true; }
304
"%yyeof" { isYYEOF = true; }
305
"%notunix" { notUnix = true; }
307
"%full"|"%8bit" { return symbol(FULL); }
308
"%unicode"|"%16bit" { return symbol(UNICODE); }
309
"%caseless"|"%ignorecase" { caseless = true; }
310
"%implements"{WSP}+.* { isImplementing = concExc(isImplementing, yytext().substring(12).trim()); }
311
"%extends"{WSP}+{QualIdent}{WSP}* { isExtending = yytext().substring(9).trim(); }
312
"%public" { isPublic = true; }
313
"%apiprivate" { visibility = "private"; Skeleton.makePrivate(); }
314
"%final" { isFinal = true; }
315
"%abstract" { isAbstract = true; }
316
"%debug" { debugOption = true; }
317
"%standalone" { standalone = true; isInteger = true; }
318
"%switch" { packed = false; useRowMap = false; }
319
"%table" { packed = false; useRowMap = true; }
320
"%pack" { packed = true; useRowMap = true; }
321
"%include" {WSP}+ .* { File f = new File(yytext().substring(9).trim());
323
throw new ScannerException(file,ErrorMessages.NOT_READABLE, yyline);
325
if (files.search(f) > 0)
326
throw new ScannerException(file,ErrorMessages.FILE_CYCLE, yyline);
328
yypushStream( new FileReader(f) );
331
Out.println("Including \""+file+"\"");
333
catch (FileNotFoundException e) {
334
throw new ScannerException(file,ErrorMessages.NOT_READABLE, yyline);
337
"%buffer" {WSP}+ {Number} {WSP}* { bufferSize = Integer.parseInt(yytext().substring(8).trim()); }
338
"%buffer" {WSP}+ {NNL}* { throw new ScannerException(file,ErrorMessages.NO_BUFFER_SIZE, yyline); }
339
"%initthrow" {WSP}+ {QUIL} {WSP}* { initThrow = concExc(initThrow,yytext().substring(11).trim()); }
340
"%initthrow" {WSP}+ {NNL}* { throw new ScannerException(file,ErrorMessages.QUIL_INITTHROW, yyline); }
341
"%eofthrow" {WSP}+ {QUIL} {WSP}* { eofThrow = concExc(eofThrow,yytext().substring(10).trim()); }
342
"%eofthrow" {WSP}+ {NNL}* { throw new ScannerException(file,ErrorMessages.QUIL_EOFTHROW, yyline); }
343
"%yylexthrow"{WSP}+ {QUIL} {WSP}* { lexThrow = concExc(lexThrow,yytext().substring(12).trim()); }
344
"%throws" {WSP}+ {QUIL} {WSP}* { lexThrow = concExc(lexThrow,yytext().substring(8).trim()); }
345
"%yylexthrow"{WSP}+ {NNL}* { throw new ScannerException(file,ErrorMessages.QUIL_YYLEXTHROW, yyline); }
346
"%throws" {WSP}+ {NNL}* { throw new ScannerException(file,ErrorMessages.QUIL_THROW, yyline); }
347
"%scanerror" {WSP}+ {QualIdent} {WSP}* { scanErrorException = yytext().substring(11).trim(); }
348
"%scanerror" {WSP}+ {NNL}* { throw new ScannerException(file,ErrorMessages.QUIL_SCANERROR, yyline); }
350
{Ident} { return symbol(IDENT, yytext()); }
351
"="{WSP}* { yybegin(REGEXP); return symbol(EQUALS); }
353
"/*" { nextState = MACROS; yybegin(COMMENT); }
355
{EndOfLineComment} { }
357
/* no {NL} at the end of this expression, because <REGEXPSTART>
358
needs at least one {WSPNL} to start a regular expression! */
359
^"%%" {NNL}* { macroDefinition = false; yybegin(REGEXPSTART); return symbol(DELIMITER); }
360
"%"{Ident} { throw new ScannerException(file,ErrorMessages.UNKNOWN_OPTION, yyline, yycolumn); }
361
"%" { throw new ScannerException(file,ErrorMessages.UNKNOWN_OPTION, yyline, yycolumn); }
362
^{WSP}+"%" { Out.warning(ErrorMessages.NOT_AT_BOL, yyline); yypushback(1); }
366
<<EOF>> { if ( yymoreStreams() ) {
367
file = (File) files.pop();
371
throw new ScannerException(file,ErrorMessages.EOF_IN_MACROS);
376
{WSPNL}* "/*" { nextState = REGEXPSTART; yybegin(COMMENT); }
377
{WSPNL}+ { yybegin(REGEXP); }
378
{WSPNL}* "<" { yybegin(STATES); return symbol_countUpdate(LESSTHAN, null); }
379
{WSPNL}* "}" { return symbol_countUpdate(RBRACE, null); }
380
{WSPNL}* "//" {NNL}* { }
381
{WSPNL}* "<<EOF>>" {WSPNL}* "{"
382
{ actionText.setLength(0); yybegin(JAVA_CODE);
383
Symbol s = symbol_countUpdate(EOFRULE, null);
384
action_line = s.left+1;
390
{Ident} { return symbol(IDENT, yytext()); }
391
"," { return symbol(COMMA); }
394
// "{" will be caught in REGEXP
395
">"{WSPNL}* { yybegin(REGEXP); return symbol(MORETHAN); }
397
<<EOF>> { throw new ScannerException(file,ErrorMessages.EOF_IN_STATES); }
402
"<<EOF>>" {WSPNL}+ "{" { actionText.setLength(0); yybegin(JAVA_CODE); action_line = yyline+1; return symbol(EOFRULE); }
403
"<<EOF>>" { throw new ScannerException(file,ErrorMessages.EOF_WO_ACTION); }
405
{WSPNL}*"|"{WSP}*$ { if (macroDefinition) {
410
yybegin(REGEXPSTART);
411
return symbol(NOACTION);
416
"{" { yybegin(REGEXPSTART); return symbol(LBRACE); }
418
{WSPNL}*"|" { return symbol(BAR); }
420
{WSPNL}*\" { string.setLength(0); nextState = REGEXP; yybegin(STRING_CONTENT); }
421
{WSPNL}*"!" { return symbol(BANG); }
422
{WSPNL}*"~" { return symbol(TILDE); }
423
{WSPNL}*"(" { return symbol(OPENBRACKET); }
424
{WSPNL}*")" { return symbol(CLOSEBRACKET); }
425
{WSPNL}*"*" { return symbol(STAR); }
426
{WSPNL}*"+" { return symbol(PLUS); }
427
{WSPNL}*"?" { return symbol(QUESTION); }
428
{WSPNL}*"$" { lookAheadUsed = true; return symbol(DOLLAR); }
429
{WSPNL}*"^" { bolUsed = true; return symbol(HAT); }
430
{WSPNL}*"." { return symbol(POINT); }
431
{WSPNL}*"[" { yybegin(CHARCLASS); return symbol(OPENCLASS); }
432
{WSPNL}*"/" { lookAheadUsed = true; return symbol(LOOKAHEAD); }
434
{WSPNL}* "{" {WSP}* {Ident} {WSP}* "}" { return symbol_countUpdate(MACROUSE, makeMacroIdent()); }
435
{WSPNL}* "{" {WSP}* {Number} { yybegin(REPEATEXP); return symbol(REPEAT, new Integer(yytext().trim().substring(1).trim())); }
437
{WSPNL}+ "{" { actionText.setLength(0); yybegin(JAVA_CODE); action_line = yyline+1; return symbol(REGEXPEND); }
438
{NL} { if (macroDefinition) { yybegin(MACROS); } return symbol(REGEXPEND); }
440
{WSPNL}*"/*" { nextState = REGEXP; yybegin(COMMENT); }
442
{WSPNL}*"//"{NNL}* { }
447
{WSPNL}*"[:jletter:]" { return symbol(JLETTERCLASS); }
448
{WSPNL}*"[:jletterdigit:]" { return symbol(JLETTERDIGITCLASS); }
449
{WSPNL}*"[:letter:]" { return symbol(LETTERCLASS); }
450
{WSPNL}*"[:digit:]" { return symbol(DIGITCLASS); }
451
{WSPNL}*"[:uppercase:]" { return symbol(UPPERCLASS); }
452
{WSPNL}*"[:lowercase:]" { return symbol(LOWERCLASS); }
455
. { return symbol(CHAR, new Character(yytext().charAt(0))); }
458
<EATWSPNL> {WSPNL}+ { yybegin(REGEXP); }
462
"}" { yybegin(REGEXP); return symbol(RBRACE); }
463
"," {WSP}* {Number} { return symbol(REPEAT, new Integer(yytext().substring(1).trim())); }
466
<<EOF>> { throw new ScannerException(file,ErrorMessages.EOF_IN_REGEXP); }
470
"{"{Ident}"}" { return symbol(MACROUSE, yytext().substring(1,yytext().length()-1)); }
471
"[" { balance++; return symbol(OPENCLASS); }
472
"]" { if (balance > 0) balance--; else yybegin(REGEXP); return symbol(CLOSECLASS); }
473
"^" { return symbol(HAT); }
474
"-" { return symbol(DASH); }
476
// this is a hack to keep JLex compatibilty with char class
477
// expressions like [+-]
478
"-]" { yypushback(1); yycolumn--; return symbol(CHAR, new Character(yytext().charAt(0))); }
480
\" { string.setLength(0); nextState = CHARCLASS; yybegin(STRING_CONTENT); }
482
. { return symbol(CHAR, new Character(yytext().charAt(0))); }
484
\n { throw new ScannerException(file,ErrorMessages.EOL_IN_CHARCLASS,yyline,yycolumn); }
486
<<EOF>> { throw new ScannerException(file,ErrorMessages.EOF_IN_REGEXP); }
490
\" { yybegin(nextState); return symbol(STRING, string.toString()); }
491
\\\" { string.append('\"'); }
492
[^\"\\\u2028\u2029\u000A\u000B\u000C\u000D\u0085]+ { string.append(yytext()); }
494
{NL} { throw new ScannerException(file,ErrorMessages.UNTERMINATED_STR, yyline, yycolumn); }
496
{HexNumber} { string.append( (char) Integer.parseInt(yytext().substring(2,yytext().length()), 16)); }
497
{Unicode} { string.append( (char) Integer.parseInt(yytext().substring(2,yytext().length()), 16)); }
498
{OctNumber} { string.append( (char) Integer.parseInt(yytext().substring(1,yytext().length()), 8)); }
500
\\b { string.append('\b'); }
501
\\n { string.append('\n'); }
502
\\t { string.append('\t'); }
503
\\f { string.append('\f'); }
504
\\r { string.append('\r'); }
506
\\. { string.append(yytext().charAt(1)); }
508
<<EOF>> { throw new ScannerException(file,ErrorMessages.EOF_IN_STRING); }
512
<REGEXP, CHARCLASS> {
513
{HexNumber} { return symbol(CHAR, new Character( (char) Integer.parseInt(yytext().substring(2,yytext().length()), 16))); }
514
{Unicode} { return symbol(CHAR, new Character( (char) Integer.parseInt(yytext().substring(2,yytext().length()), 16))); }
515
{OctNumber} { return symbol(CHAR, new Character( (char) Integer.parseInt(yytext().substring(1,yytext().length()), 8))); }
517
\\b { return symbol(CHAR,new Character('\b')); }
518
\\n { return symbol(CHAR,new Character('\n')); }
519
\\t { return symbol(CHAR,new Character('\t')); }
520
\\f { return symbol(CHAR,new Character('\f')); }
521
\\r { return symbol(CHAR,new Character('\r')); }
523
\\. { return symbol(CHAR, new Character(yytext().charAt(1))); }
528
"{" { balance++; actionText.append('{'); }
529
"}" { if (balance > 0) {
531
actionText.append('}');
534
yybegin(REGEXPSTART);
535
Action a = new Action(actionText.toString(), action_line);
536
actions.addElement(a);
537
return symbol(ACTION, a);
541
{JavaCode} { actionText.append(yytext()); }
543
<<EOF>> { throw new ScannerException(file,ErrorMessages.EOF_IN_ACTION, action_line-1); }
548
"/"+ "*" { commentbalance++; }
549
"*"+ "/" { if (commentbalance > 0)
555
{JFlexComment} { /* ignore */ }
557
<<EOF>> { throw new ScannerException(file,ErrorMessages.EOF_IN_COMMENT); }
561
. { throw new ScannerException(file,ErrorMessages.UNEXPECTED_CHAR, yyline, yycolumn); }
562
\n { throw new ScannerException(file,ErrorMessages.UNEXPECTED_NL, yyline, yycolumn); }
564
<<EOF>> { if ( yymoreStreams() ) {
565
file = (File) files.pop();
569
return symbol(EOF); }
added
removed
build/classes/build/classes/build/classes/build/classes/build/classes/weka/core/parser/JFlex/LexScan.flex
