2
// <copyright see="prj:///doc/copyright.txt"/>
3
// <license see="prj:///doc/license.txt"/>
4
// <owner name="Andrea Paatz" email="andrea@icsharpcode.net"/>
5
// <version>$Revision: 4482 $</version>
9
using System.Collections.Generic;
10
using System.Globalization;
14
namespace ICSharpCode.OldNRefactory.Parser.CSharp
16
public sealed class Lexer : AbstractLexer
18
bool isAtLineBegin = true;
20
public Lexer(TextReader reader) : base(reader)
24
protected override Token Next()
28
bool hadLineEnd = false;
29
if (Line == 1 && Col == 1) {
31
hadLineEnd = true; // beginning of document
34
while ((nextChar = ReaderRead()) != -1) {
44
// second line end before getting to a token
45
// -> here was a blank line
46
specialTracker.AddEndOfLine(new Location(Col, Line));
48
HandleLineEnd((char)nextChar);
53
int peek = ReaderPeek();
54
if (peek == '/' || peek == '*') {
58
isAtLineBegin = false;
59
token = ReadOperator('/');
63
ReadPreProcessingDirective();
64
isAtLineBegin = false;
68
isAtLineBegin = false;
72
isAtLineBegin = false;
75
isAtLineBegin = false;
76
int next = ReaderRead();
78
errors.Error(Line, Col, String.Format("EOF after @"));
85
token = ReadVerbatimString();
86
} else if (Char.IsLetterOrDigit(ch) || ch == '_') {
88
token = new Token(Tokens.Identifier, x - 1, y, ReadIdent(ch, out canBeKeyword));
91
errors.Error(y, x, String.Format("Unexpected char in Lexer.Next() : {0}", ch));
97
isAtLineBegin = false; // non-ws chars are handled here
99
if (Char.IsLetter(ch) || ch == '_' || ch == '\\') {
100
int x = Col - 1; // Col was incremented above, but we want the start of the identifier
103
string s = ReadIdent(ch, out canBeKeyword);
105
int keyWordToken = Keywords.GetToken(s);
106
if (keyWordToken >= 0) {
107
return new Token(keyWordToken, x, y, s);
110
return new Token(Tokens.Identifier, x, y, s);
111
} else if (Char.IsDigit(ch)) {
112
token = ReadDigit(ch, Col - 1);
114
token = ReadOperator(ch);
119
// try error recovery (token = null -> continue with next char)
125
return new Token(Tokens.EOF, Col, Line, String.Empty);
128
// The C# compiler has a fixed size length therefore we'll use a fixed size char array for identifiers
129
// it's also faster than using a string builder.
130
const int MAX_IDENTIFIER_LENGTH = 512;
131
char[] identBuffer = new char[MAX_IDENTIFIER_LENGTH];
133
string ReadIdent(char ch, out bool canBeKeyword)
141
if (peek != 'u' && peek != 'U') {
142
errors.Error(Line, Col, "Identifiers can only contain unicode escape sequences");
144
canBeKeyword = false;
145
string surrogatePair;
146
ReadEscapeSequence(out ch, out surrogatePair);
147
if (surrogatePair != null) {
148
if (!char.IsLetterOrDigit(surrogatePair, 0)) {
149
errors.Error(Line, Col, "Unicode escape sequences in identifiers cannot be used to represent characters that are invalid in identifiers");
151
for (int i = 0; i < surrogatePair.Length - 1; i++) {
152
if (curPos < MAX_IDENTIFIER_LENGTH) {
153
identBuffer[curPos++] = surrogatePair[i];
156
ch = surrogatePair[surrogatePair.Length - 1];
158
if (!IsIdentifierPart(ch)) {
159
errors.Error(Line, Col, "Unicode escape sequences in identifiers cannot be used to represent characters that are invalid in identifiers");
164
if (curPos < MAX_IDENTIFIER_LENGTH) {
165
identBuffer[curPos++] = ch;
167
errors.Error(Line, Col, String.Format("Identifier too long"));
168
while (IsIdentifierPart(ReaderPeek())) {
174
if (IsIdentifierPart(peek) || peek == '\\') {
175
ch = (char)ReaderRead();
180
return new String(identBuffer, 0, curPos);
183
Token ReadDigit(char ch, int x)
185
unchecked { // prevent exception when ReaderPeek() = -1 is cast to char
189
string prefix = null;
190
string suffix = null;
193
bool isunsigned = false;
195
bool isfloat = false;
196
bool isdouble = false;
197
bool isdecimal = false;
199
char peek = (char)ReaderPeek();
204
while (Char.IsDigit((char)ReaderPeek())) { // read decimal digits beyond the dot
205
sb.Append((char)ReaderRead());
207
peek = (char)ReaderPeek();
208
} else if (ch == '0' && (peek == 'x' || peek == 'X')) {
209
ReaderRead(); // skip 'x'
210
sb.Length = 0; // Remove '0' from 0x prefix from the stringvalue
211
while (IsHex((char)ReaderPeek())) {
212
sb.Append((char)ReaderRead());
214
if (sb.Length == 0) {
215
sb.Append('0'); // dummy value to prevent exception
216
errors.Error(y, x, "Invalid hexadecimal integer literal");
220
peek = (char)ReaderPeek();
222
while (Char.IsDigit((char)ReaderPeek())) {
223
sb.Append((char)ReaderRead());
225
peek = (char)ReaderPeek();
228
Token nextToken = null; // if we accidently read a 'dot'
229
if (peek == '.') { // read floating point number
231
peek = (char)ReaderPeek();
232
if (!Char.IsDigit(peek)) {
233
nextToken = new Token(Tokens.Dot, Col - 1, Line);
236
isdouble = true; // double is default
238
errors.Error(y, x, String.Format("No hexadecimal floating point values allowed"));
242
while (Char.IsDigit((char)ReaderPeek())) { // read decimal digits beyond the dot
243
sb.Append((char)ReaderRead());
245
peek = (char)ReaderPeek();
249
if (peek == 'e' || peek == 'E') { // read exponent
251
sb.Append((char)ReaderRead());
252
peek = (char)ReaderPeek();
253
if (peek == '-' || peek == '+') {
254
sb.Append((char)ReaderRead());
256
while (Char.IsDigit((char)ReaderPeek())) { // read exponent value
257
sb.Append((char)ReaderRead());
260
peek = (char)ReaderPeek();
263
if (peek == 'f' || peek == 'F') { // float value
267
} else if (peek == 'd' || peek == 'D') { // double type suffix (obsolete, double is default)
271
} else if (peek == 'm' || peek == 'M') { // decimal value
275
} else if (!isdouble) {
276
if (peek == 'u' || peek == 'U') {
280
peek = (char)ReaderPeek();
283
if (peek == 'l' || peek == 'L') {
285
peek = (char)ReaderPeek();
287
if (!isunsigned && (peek == 'u' || peek == 'U')) {
292
suffix = isunsigned ? "uL" : "L";
297
string digit = sb.ToString();
298
string stringValue = prefix + digit + suffix;
302
if (float.TryParse(digit, NumberStyles.Any, CultureInfo.InvariantCulture, out num)) {
303
return new Token(Tokens.Literal, x, y, stringValue, num, LiteralFormat.DecimalNumber);
305
errors.Error(y, x, String.Format("Can't parse float {0}", digit));
306
return new Token(Tokens.Literal, x, y, stringValue, 0f, LiteralFormat.DecimalNumber);
311
if (decimal.TryParse(digit, NumberStyles.Any, CultureInfo.InvariantCulture, out num)) {
312
return new Token(Tokens.Literal, x, y, stringValue, num, LiteralFormat.DecimalNumber);
314
errors.Error(y, x, String.Format("Can't parse decimal {0}", digit));
315
return new Token(Tokens.Literal, x, y, stringValue, 0m, LiteralFormat.DecimalNumber);
320
if (double.TryParse(digit, NumberStyles.Any, CultureInfo.InvariantCulture, out num)) {
321
return new Token(Tokens.Literal, x, y, stringValue, num, LiteralFormat.DecimalNumber);
323
errors.Error(y, x, String.Format("Can't parse double {0}", digit));
324
return new Token(Tokens.Literal, x, y, stringValue, 0d, LiteralFormat.DecimalNumber);
328
// Try to determine a parsable value using ranges.
331
if (!ulong.TryParse(digit, NumberStyles.HexNumber, null, out result)) {
332
errors.Error(y, x, String.Format("Can't parse hexadecimal constant {0}", digit));
333
return new Token(Tokens.Literal, x, y, stringValue.ToString(), 0, LiteralFormat.HexadecimalNumber);
336
if (!ulong.TryParse(digit, NumberStyles.Integer, null, out result)) {
337
errors.Error(y, x, String.Format("Can't parse integral constant {0}", digit));
338
return new Token(Tokens.Literal, x, y, stringValue.ToString(), 0, LiteralFormat.DecimalNumber);
342
if (result > long.MaxValue) {
345
} else if (result > uint.MaxValue) {
347
} else if (islong == false && result > int.MaxValue) {
353
LiteralFormat literalFormat = ishex ? LiteralFormat.HexadecimalNumber : LiteralFormat.DecimalNumber;
357
if (ulong.TryParse(digit, ishex ? NumberStyles.HexNumber : NumberStyles.Number, CultureInfo.InvariantCulture, out num)) {
358
token = new Token(Tokens.Literal, x, y, stringValue, num, literalFormat);
360
errors.Error(y, x, String.Format("Can't parse unsigned long {0}", digit));
361
token = new Token(Tokens.Literal, x, y, stringValue, 0UL, literalFormat);
365
if (long.TryParse(digit, ishex ? NumberStyles.HexNumber : NumberStyles.Number, CultureInfo.InvariantCulture, out num)) {
366
token = new Token(Tokens.Literal, x, y, stringValue, num, literalFormat);
368
errors.Error(y, x, String.Format("Can't parse long {0}", digit));
369
token = new Token(Tokens.Literal, x, y, stringValue, 0L, literalFormat);
375
if (uint.TryParse(digit, ishex ? NumberStyles.HexNumber : NumberStyles.Number, CultureInfo.InvariantCulture, out num)) {
376
token = new Token(Tokens.Literal, x, y, stringValue, num, literalFormat);
378
errors.Error(y, x, String.Format("Can't parse unsigned int {0}", digit));
379
token = new Token(Tokens.Literal, x, y, stringValue, (uint)0, literalFormat);
383
if (int.TryParse(digit, ishex ? NumberStyles.HexNumber : NumberStyles.Number, CultureInfo.InvariantCulture, out num)) {
384
token = new Token(Tokens.Literal, x, y, stringValue, num, literalFormat);
386
errors.Error(y, x, String.Format("Can't parse int {0}", digit));
387
token = new Token(Tokens.Literal, x, y, stringValue, 0, literalFormat);
391
token.next = nextToken;
401
bool doneNormally = false;
403
while ((nextChar = ReaderRead()) != -1) {
404
char ch = (char)nextChar;
412
SkipEscapeSequence();
413
} else if (HandleLineEnd(ch)) {
414
// call HandleLineEnd to ensure line numbers are still correct after the error
415
errors.Error(y, x, String.Format("No new line is allowed inside a string literal"));
420
errors.Error(y, x, String.Format("End of file reached inside string literal"));
430
originalValue.Length = 0;
431
originalValue.Append('"');
432
bool doneNormally = false;
434
while ((nextChar = ReaderRead()) != -1) {
435
char ch = (char)nextChar;
439
originalValue.Append('"');
444
originalValue.Append('\\');
445
string surrogatePair;
446
originalValue.Append(ReadEscapeSequence(out ch, out surrogatePair));
447
if (surrogatePair != null) {
448
sb.Append(surrogatePair);
452
} else if (HandleLineEnd(ch)) {
453
// call HandleLineEnd to ensure line numbers are still correct after the error
454
errors.Error(y, x, String.Format("No new line is allowed inside a string literal"));
457
originalValue.Append(ch);
463
errors.Error(y, x, String.Format("End of file reached inside string literal"));
466
return new Token(Tokens.Literal, x, y, originalValue.ToString(), sb.ToString(), LiteralFormat.StringLiteral);
469
Token ReadVerbatimString()
472
originalValue.Length = 0;
473
originalValue.Append("@\"");
474
Location startLocation = new Location(Col - 2, Line); // @ and " already read
476
while ((nextChar = ReaderRead()) != -1) {
477
char ch = (char)nextChar;
480
if (ReaderPeek() != '"') {
481
originalValue.Append('"');
484
originalValue.Append("\"\"");
487
} else if (HandleLineEnd(ch)) {
489
originalValue.Append("\r\n");
492
originalValue.Append(ch);
496
if (nextChar == -1) {
497
errors.Error(startLocation.Line, startLocation.Column, String.Format("End of file reached inside verbatim string literal"));
500
return new Token(Tokens.Literal, startLocation, new Location(Col, Line), originalValue.ToString(), sb.ToString(), LiteralFormat.VerbatimStringLiteral);
503
readonly char[] escapeSequenceBuffer = new char[12];
506
/// reads an escape sequence
508
/// <param name="ch">The character represented by the escape sequence,
509
/// or '\0' if there was an error or the escape sequence represents a character that
510
/// can be represented only be a suggorate pair</param>
511
/// <param name="surrogatePair">Null, except when the character represented
512
/// by the escape sequence can only be represented by a surrogate pair (then the string
513
/// contains the surrogate pair)</param>
514
/// <returns>The escape sequence</returns>
515
string ReadEscapeSequence(out char ch, out string surrogatePair)
517
surrogatePair = null;
519
int nextChar = ReaderRead();
520
if (nextChar == -1) {
521
errors.Error(Line, Col, String.Format("End of file reached inside escape sequence"));
526
char c = (char)nextChar;
528
escapeSequenceBuffer[0] = c;
565
// 16 bit unicode character
566
c = (char)ReaderRead();
567
number = GetHexNumber(c);
568
escapeSequenceBuffer[curPos++] = c;
571
errors.Error(Line, Col - 1, String.Format("Invalid char in literal : {0}", c));
573
for (int i = 0; i < 3; ++i) {
574
if (IsHex((char)ReaderPeek())) {
575
c = (char)ReaderRead();
576
int idx = GetHexNumber(c);
577
escapeSequenceBuffer[curPos++] = c;
578
number = 16 * number + idx;
586
// 32 bit unicode character
588
for (int i = 0; i < 8; ++i) {
589
if (IsHex((char)ReaderPeek())) {
590
c = (char)ReaderRead();
591
int idx = GetHexNumber(c);
592
escapeSequenceBuffer[curPos++] = c;
593
number = 16 * number + idx;
595
errors.Error(Line, Col - 1, String.Format("Invalid char in literal : {0}", (char)ReaderPeek()));
599
if (number > 0xffff) {
601
surrogatePair = char.ConvertFromUtf32(number);
607
errors.Error(Line, Col, String.Format("Unexpected escape sequence : {0}", c));
611
return new String(escapeSequenceBuffer, 0, curPos);
614
void SkipEscapeSequence()
616
int nextChar = ReaderRead();
617
if (nextChar == -1) {
618
errors.Error(Line, Col, String.Format("End of file reached inside escape sequence"));
636
// 16 bit unicode character
637
char c = (char)ReaderRead();
638
if (GetHexNumber(c) < 0)
639
errors.Error(Line, Col - 1, String.Format("Invalid char in literal : {0}", c));
640
for (int i = 0; i < 3; ++i) {
641
if (IsHex((char)ReaderPeek())) {
649
for (int i = 0; i < 8; ++i) {
650
if (IsHex((char)ReaderPeek())) {
653
errors.Error(Line, Col - 1, String.Format("Invalid char in literal : {0}", (char)ReaderPeek()));
659
errors.Error(Line, Col, String.Format("Unexpected escape sequence : {0}", nextChar));
668
int nextChar = ReaderRead();
669
if (nextChar == -1 || HandleLineEnd((char)nextChar)) {
670
errors.Error(y, x, String.Format("End of line reached inside character literal"));
673
char ch = (char)nextChar;
675
string escapeSequence = String.Empty;
677
string surrogatePair;
678
escapeSequence = ReadEscapeSequence(out chValue, out surrogatePair);
679
if (surrogatePair != null) {
680
errors.Error(y, x, String.Format("The unicode character must be represented by a surrogate pair and does not fit into a System.Char"));
685
if ((char)ReaderRead() != '\'') {
686
errors.Error(y, x, String.Format("Char not terminated"));
689
return new Token(Tokens.Literal, x, y, "'" + ch + escapeSequence + "'", chValue, LiteralFormat.CharLiteral);
692
Token ReadOperator(char ch)
698
switch (ReaderPeek()) {
701
return new Token(Tokens.Increment, x, y);
704
return new Token(Tokens.PlusAssign, x, y);
706
return new Token(Tokens.Plus, x, y);
708
switch (ReaderPeek()) {
711
return new Token(Tokens.Decrement, x, y);
714
return new Token(Tokens.MinusAssign, x, y);
717
return new Token(Tokens.Pointer, x, y);
719
return new Token(Tokens.Minus, x, y);
721
switch (ReaderPeek()) {
724
return new Token(Tokens.TimesAssign, x, y);
728
return new Token(Tokens.Times, x, y);
730
switch (ReaderPeek()) {
733
return new Token(Tokens.DivAssign, x, y);
735
return new Token(Tokens.Div, x, y);
737
switch (ReaderPeek()) {
740
return new Token(Tokens.ModAssign, x, y);
742
return new Token(Tokens.Mod, x, y);
744
switch (ReaderPeek()) {
747
return new Token(Tokens.LogicalAnd, x, y);
750
return new Token(Tokens.BitwiseAndAssign, x, y);
752
return new Token(Tokens.BitwiseAnd, x, y);
754
switch (ReaderPeek()) {
757
return new Token(Tokens.LogicalOr, x, y);
760
return new Token(Tokens.BitwiseOrAssign, x, y);
762
return new Token(Tokens.BitwiseOr, x, y);
764
switch (ReaderPeek()) {
767
return new Token(Tokens.XorAssign, x, y);
771
return new Token(Tokens.Xor, x, y);
773
switch (ReaderPeek()) {
776
return new Token(Tokens.NotEqual, x, y);
778
return new Token(Tokens.Not, x, y);
780
return new Token(Tokens.BitwiseComplement, x, y);
782
switch (ReaderPeek()) {
785
return new Token(Tokens.Equal, x, y);
788
return new Token(Tokens.LambdaArrow, x, y);
790
return new Token(Tokens.Assign, x, y);
792
switch (ReaderPeek()) {
795
switch (ReaderPeek()) {
798
return new Token(Tokens.ShiftLeftAssign, x, y);
802
return new Token(Tokens.ShiftLeft, x, y);
805
return new Token(Tokens.LessEqual, x, y);
807
return new Token(Tokens.LessThan, x, y);
809
switch (ReaderPeek()) {
810
// Removed because of generics:
813
// if (ReaderPeek() != -1) {
814
// switch ((char)ReaderPeek()) {
817
// return new Token(Tokens.ShiftRightAssign, x, y);
822
// return new Token(Tokens.ShiftRight, x, y);
825
return new Token(Tokens.GreaterEqual, x, y);
827
return new Token(Tokens.GreaterThan, x, y);
829
if (ReaderPeek() == '?') {
831
return new Token(Tokens.DoubleQuestion, x, y);
833
return new Token(Tokens.Question, x, y);
835
return new Token(Tokens.Semicolon, x, y);
837
if (ReaderPeek() == ':') {
839
return new Token(Tokens.DoubleColon, x, y);
841
return new Token(Tokens.Colon, x, y);
843
return new Token(Tokens.Comma, x, y);
845
// Prevent OverflowException when ReaderPeek returns -1
846
int tmp = ReaderPeek();
847
if (tmp > 0 && Char.IsDigit((char)tmp)) {
848
return ReadDigit('.', Col - 1);
850
return new Token(Tokens.Dot, x, y);
852
return new Token(Tokens.CloseParenthesis, x, y);
854
return new Token(Tokens.OpenParenthesis, x, y);
856
return new Token(Tokens.CloseSquareBracket, x, y);
858
return new Token(Tokens.OpenSquareBracket, x, y);
860
return new Token(Tokens.CloseCurlyBrace, x, y);
862
return new Token(Tokens.OpenCurlyBrace, x, y);
870
switch (ReaderRead()) {
872
ReadMultiLineComment();
873
isAtLineBegin = false;
876
if (ReaderPeek() == '/') {
878
ReadSingleLineComment(CommentType.Documentation);
880
ReadSingleLineComment(CommentType.SingleLine);
882
isAtLineBegin = true;
885
errors.Error(Line, Col, String.Format("Error while reading comment"));
890
string ReadCommentToEOL()
892
if (specialCommentHash == null) {
893
return ReadToEndOfLine();
896
StringBuilder curWord = new StringBuilder();
898
while ((nextChar = ReaderRead()) != -1) {
899
char ch = (char)nextChar;
901
if (HandleLineEnd(ch)) {
906
if (IsIdentifierPart(nextChar)) {
909
string tag = curWord.ToString();
911
if (specialCommentHash.ContainsKey(tag)) {
912
Location p = new Location(Col, Line);
913
string comment = ReadToEndOfLine ();
914
this.TagComments.Add(new TagComment(tag, comment, isAtLineBegin, p, new Location(Col, Line)));
919
return sb.ToString();
922
void ReadSingleLineComment(CommentType commentType)
924
if (this.SkipAllComments) {
927
specialTracker.StartComment(commentType, isAtLineBegin, new Location(Col, Line));
928
specialTracker.AddString(ReadCommentToEOL());
929
specialTracker.FinishComment(new Location(lineBreakPosition.Column, lineBreakPosition.Line));
933
void ReadMultiLineComment()
936
if (this.SkipAllComments) {
937
while ((nextChar = ReaderRead()) != -1) {
938
char ch = (char)nextChar;
939
if (ch == '*' && ReaderPeek() == '/') {
947
specialTracker.StartComment(CommentType.Block, isAtLineBegin, new Location(Col, Line));
949
// sc* = special comment handling (TO DO markers)
950
string scTag = null; // is set to non-null value when we are inside a comment marker
951
StringBuilder scCurWord = new StringBuilder(); // current word, (scTag == null) or comment (when scTag != null)
952
Location scStartLocation = Location.Empty;
954
while ((nextChar = ReaderRead()) != -1) {
955
char ch = (char)nextChar;
957
if (HandleLineEnd(ch)) {
959
this.TagComments.Add(new TagComment(scTag, scCurWord.ToString(), isAtLineBegin, scStartLocation, new Location(Col, Line)));
962
scCurWord.Length = 0;
963
specialTracker.AddString(Environment.NewLine);
967
// End of multiline comment reached ?
968
if (ch == '*' && ReaderPeek() == '/') {
970
this.TagComments.Add(new TagComment(scTag, scCurWord.ToString(), isAtLineBegin, scStartLocation, new Location(Col, Line)));
973
specialTracker.FinishComment(new Location(Col, Line));
976
specialTracker.AddChar(ch);
977
if (scTag != null || IsIdentifierPart(ch)) {
978
scCurWord.Append(ch);
980
if (specialCommentHash != null && specialCommentHash.ContainsKey(scCurWord.ToString())) {
981
scTag = scCurWord.ToString();
982
scStartLocation = new Location(Col, Line);
984
scCurWord.Length = 0;
987
specialTracker.FinishComment(new Location(Col, Line));
989
// Reached EOF before end of multiline comment.
990
errors.Error(Line, Col, String.Format("Reached EOF before the end of a multiline comment"));
994
/// Skips to the end of the current code block.
995
/// For this, the lexer must have read the next token AFTER the token opening the
996
/// block (so that Lexer.Token is the block-opening token, not Lexer.LookAhead).
997
/// After the call, Lexer.LookAhead will be the block-closing token.
999
public override void SkipCurrentBlock(int targetToken)
1002
while (curToken != null) {
1003
if (curToken.kind == Tokens.OpenCurlyBrace) {
1005
} else if (curToken.kind == Tokens.CloseCurlyBrace) {
1006
if (--braceCount < 0)
1009
lastToken = curToken;
1010
curToken = curToken.next;
1012
isAtLineBegin = true;
1014
while ((nextChar = ReaderRead()) != -1) {
1017
isAtLineBegin = false;
1021
isAtLineBegin = false;
1022
if (--braceCount < 0) {
1023
curToken = new Token(Tokens.CloseCurlyBrace, Col - 1, Line);
1028
int peek = ReaderPeek();
1029
if (peek == '/' || peek == '*') {
1032
isAtLineBegin = false;
1035
ReadPreProcessingDirective();
1036
isAtLineBegin = false;
1040
isAtLineBegin = false;
1044
isAtLineBegin = false;
1048
HandleLineEnd((char)nextChar);
1049
isAtLineBegin = true;
1052
int next = ReaderRead();
1054
errors.Error(Line, Col, String.Format("EOF after @"));
1055
} else if (next == '"') {
1056
ReadVerbatimString();
1058
isAtLineBegin = false;
1062
curToken = new Token(Tokens.EOF, Col, Line);
1065
public override IDictionary<string, object> ConditionalCompilationSymbols {
1066
get { return conditionalCompilation.Symbols; }
1069
public override void SetConditionalCompilationSymbols (string symbols)
1071
foreach (string symbol in GetSymbols (symbols)) {
1072
conditionalCompilation.Define (symbol);
1077
ConditionalCompilation conditionalCompilation = new ConditionalCompilation();
1079
void ReadPreProcessingDirective()
1081
PreprocessingDirective d = ReadPreProcessingDirectiveInternal(true, true);
1082
this.specialTracker.AddPreprocessingDirective(d);
1084
if (EvaluateConditionalCompilation) {
1087
conditionalCompilation.Define(d.Arg);
1090
conditionalCompilation.Undefine(d.Arg);
1093
if (!conditionalCompilation.Evaluate(d.Expression)) {
1094
// skip to valid #elif or #else or #endif
1097
d = SkipToPreProcessingDirective(false, level == 1);
1100
if (d.Cmd == "#if") {
1102
} else if (d.Cmd == "#endif") {
1106
} else if (level == 1 && (d.Cmd == "#else"
1107
|| d.Cmd == "#elif" && conditionalCompilation.Evaluate(d.Expression)))
1113
this.specialTracker.AddPreprocessingDirective(d);
1118
// we already visited the #if part or a previous #elif part, so skip until #endif
1122
d = SkipToPreProcessingDirective(false, false);
1125
if (d.Cmd == "#if") {
1127
} else if (d.Cmd == "#endif") {
1134
this.specialTracker.AddPreprocessingDirective(d);
1141
PreprocessingDirective SkipToPreProcessingDirective(bool parseIfExpression, bool parseElifExpression)
1148
errors.Error(Line, Col, String.Format("Reached EOF but expected #endif"));
1150
} else if (c == '#') {
1153
if (c != '\n') // only skip non empty lines.
1157
return ReadPreProcessingDirectiveInternal(parseIfExpression, parseElifExpression);
1160
PreprocessingDirective ReadPreProcessingDirectiveInternal(bool parseIfExpression, bool parseElifExpression)
1162
Location start = new Location(Col - 1, Line);
1164
// skip spaces between # and the directive
1168
string directive = ReadIdent('#', out canBeKeyword);
1171
if (parseIfExpression && directive == "#if" || parseElifExpression && directive == "#elif") {
1172
recordedText.Length = 0;
1174
Ast.Expression expr = PPExpression();
1175
string arg = recordedText.ToString ();
1178
Location endLocation = new Location(Col, Line);
1179
int c = ReaderRead();
1180
if (c >= 0 && !HandleLineEnd((char)c)) {
1181
if (c == '/' && ReaderRead() == '/') {
1182
// comment to end of line
1184
errors.Error(Col, Line, "Expected end of line");
1186
SkipToEndOfLine(); // skip comment
1188
return new PreprocessingDirective(directive, arg, start, endLocation) { Expression = expr, LastLineEnd = lastLineEnd };
1190
Location endLocation = new Location(Col, Line);
1191
string arg = ReadToEndOfLine();
1192
endLocation.Column += arg.Length;
1193
int pos = arg.IndexOf("//");
1195
arg = arg.Substring(0, pos);
1197
return new PreprocessingDirective(directive, arg, start, endLocation) { LastLineEnd = lastLineEnd };
1203
while (ReaderPeek() == ' ' || ReaderPeek() == '\t')
1207
public Ast.Expression PPExpression()
1209
Ast.Expression expr = PPAndExpression();
1210
while (ReaderPeek() == '|') {
1211
Token token = ReadOperator((char)ReaderRead());
1212
if (token == null || token.kind != Tokens.LogicalOr) {
1215
Ast.Expression expr2 = PPAndExpression();
1216
expr = new Ast.BinaryOperatorExpression(expr, Ast.BinaryOperatorType.LogicalOr, expr2);
1221
Ast.Expression PPAndExpression()
1223
Ast.Expression expr = PPEqualityExpression();
1224
while (ReaderPeek() == '&') {
1225
Token token = ReadOperator((char)ReaderRead());
1226
if (token == null || token.kind != Tokens.LogicalAnd) {
1229
Ast.Expression expr2 = PPEqualityExpression();
1230
expr = new Ast.BinaryOperatorExpression(expr, Ast.BinaryOperatorType.LogicalAnd, expr2);
1235
Ast.Expression PPEqualityExpression()
1237
Ast.Expression expr = PPUnaryExpression();
1238
while (ReaderPeek() == '=' || ReaderPeek() == '!') {
1239
Token token = ReadOperator((char)ReaderRead());
1240
if (token == null || token.kind != Tokens.Equal && token.kind != Tokens.NotEqual) {
1243
Ast.Expression expr2 = PPUnaryExpression();
1244
expr = new Ast.BinaryOperatorExpression(expr, token.kind == Tokens.Equal ? Ast.BinaryOperatorType.Equality : Ast.BinaryOperatorType.InEquality, expr2);
1249
Ast.Expression PPUnaryExpression()
1252
if (ReaderPeek() == '!') {
1255
return new Ast.UnaryOperatorExpression(PPUnaryExpression(), Ast.UnaryOperatorType.Not);
1257
return PPPrimaryExpression();
1261
Ast.Expression PPPrimaryExpression()
1263
int c = ReaderRead();
1265
return Ast.Expression.Null;
1267
Ast.Expression expr = new Ast.ParenthesizedExpression(PPExpression());
1269
if (ReaderRead() != ')')
1270
errors.Error(Col, Line, "Expected ')'");
1274
if (c != '_' && !char.IsLetterOrDigit((char)c) && c != '\\')
1275
errors.Error(Col, Line, "Expected conditional symbol");
1277
string symbol = ReadIdent((char)c, out canBeKeyword);
1279
if (canBeKeyword && symbol == "true")
1280
return new Ast.PrimitiveExpression(true, "true");
1281
else if (canBeKeyword && symbol == "false")
1282
return new Ast.PrimitiveExpression(false, "false");
1284
return new Ast.IdentifierExpression(symbol);