~ubuntu-branches/ubuntu/trusty/monodevelop/trusty-proposed

« back to all changes in this revision

Viewing changes to contrib/NRefactory/Project/Src/Lexer/CSharp/Lexer.cs

  • Committer: Package Import Robot
  • Author(s): Jo Shields
  • Date: 2013-05-12 09:46:03 UTC
  • mto: This revision was merged to the branch mainline in revision 29.
  • Revision ID: package-import@ubuntu.com-20130512094603-mad323bzcxvmcam0
Tags: upstream-4.0.5+dfsg
ImportĀ upstreamĀ versionĀ 4.0.5+dfsg

Show diffs side-by-side

added added

removed removed

Lines of Context:
1
 
// <file>
2
 
//     <copyright see="prj:///doc/copyright.txt"/>
3
 
//     <license see="prj:///doc/license.txt"/>
4
 
//     <owner name="Andrea Paatz" email="andrea@icsharpcode.net"/>
5
 
//     <version>$Revision: 4482 $</version>
6
 
// </file>
7
 
 
8
 
using System;
9
 
using System.Collections.Generic;
10
 
using System.Globalization;
11
 
using System.IO;
12
 
using System.Text;
13
 
 
14
 
namespace ICSharpCode.OldNRefactory.Parser.CSharp
15
 
{
16
 
        public sealed class Lexer : AbstractLexer
17
 
        {
18
 
                bool isAtLineBegin = true;
19
 
                
20
 
                public Lexer(TextReader reader) : base(reader)
21
 
                {
22
 
                }
23
 
                
24
 
                protected override Token Next()
25
 
                {
26
 
                        int nextChar;
27
 
                        char ch;
28
 
                        bool hadLineEnd = false;
29
 
                        if (Line == 1 && Col == 1) {
30
 
                                isAtLineBegin = true;
31
 
                                hadLineEnd = true; // beginning of document
32
 
                        }
33
 
                        
34
 
                        while ((nextChar = ReaderRead()) != -1) {
35
 
                                Token token;
36
 
                                
37
 
                                switch (nextChar) {
38
 
                                        case ' ':
39
 
                                        case '\t':
40
 
                                                continue;
41
 
                                        case '\r':
42
 
                                        case '\n':
43
 
                                                if (hadLineEnd) {
44
 
                                                        // second line end before getting to a token
45
 
                                                        // -> here was a blank line
46
 
                                                        specialTracker.AddEndOfLine(new Location(Col, Line));
47
 
                                                }
48
 
                                                HandleLineEnd((char)nextChar);
49
 
                                                hadLineEnd = true;
50
 
                                                isAtLineBegin = true;
51
 
                                                continue;
52
 
                                        case '/':
53
 
                                                int peek = ReaderPeek();
54
 
                                                if (peek == '/' || peek == '*') {
55
 
                                                        ReadComment();
56
 
                                                        continue;
57
 
                                                } else {
58
 
                                                        isAtLineBegin = false;
59
 
                                                        token = ReadOperator('/');
60
 
                                                }
61
 
                                                break;
62
 
                                        case '#':
63
 
                                                ReadPreProcessingDirective();
64
 
                                                isAtLineBegin = false;
65
 
                                                continue;
66
 
                                        case '"':
67
 
                                                token = ReadString();
68
 
                                                isAtLineBegin = false;
69
 
                                                break;
70
 
                                        case '\'':
71
 
                                                token = ReadChar();
72
 
                                                isAtLineBegin = false;
73
 
                                                break;
74
 
                                        case '@':
75
 
                                                isAtLineBegin = false;
76
 
                                                int next = ReaderRead();
77
 
                                                if (next == -1) {
78
 
                                                        errors.Error(Line, Col, String.Format("EOF after @"));
79
 
                                                        continue;
80
 
                                                } else {
81
 
                                                        int x = Col - 1;
82
 
                                                        int y = Line;
83
 
                                                        ch = (char)next;
84
 
                                                        if (ch == '"') {
85
 
                                                                token = ReadVerbatimString();
86
 
                                                        } else if (Char.IsLetterOrDigit(ch) || ch == '_') {
87
 
                                                                bool canBeKeyword;
88
 
                                                                token = new Token(Tokens.Identifier, x - 1, y, ReadIdent(ch, out canBeKeyword));
89
 
                                                        } else {
90
 
                                                                HandleLineEnd(ch);
91
 
                                                                errors.Error(y, x, String.Format("Unexpected char in Lexer.Next() : {0}", ch));
92
 
                                                                continue;
93
 
                                                        }
94
 
                                                }
95
 
                                                break;
96
 
                                        default: 
97
 
                                                isAtLineBegin = false; // non-ws chars are handled here
98
 
                                                ch = (char)nextChar;
99
 
                                                if (Char.IsLetter(ch) || ch == '_' || ch == '\\') {
100
 
                                                        int x = Col - 1; // Col was incremented above, but we want the start of the identifier
101
 
                                                        int y = Line;
102
 
                                                        bool canBeKeyword;
103
 
                                                        string s = ReadIdent(ch, out canBeKeyword);
104
 
                                                        if (canBeKeyword) {
105
 
                                                                int keyWordToken = Keywords.GetToken(s);
106
 
                                                                if (keyWordToken >= 0) {
107
 
                                                                        return new Token(keyWordToken, x, y, s);
108
 
                                                                }
109
 
                                                        }
110
 
                                                        return new Token(Tokens.Identifier, x, y, s);
111
 
                                                } else if (Char.IsDigit(ch)) {
112
 
                                                        token = ReadDigit(ch, Col - 1);
113
 
                                                } else {
114
 
                                                        token = ReadOperator(ch);
115
 
                                                }
116
 
                                                break;
117
 
                                }
118
 
                                
119
 
                                // try error recovery (token = null -> continue with next char)
120
 
                                if (token != null) {
121
 
                                        return token;
122
 
                                }
123
 
                        }
124
 
                        
125
 
                        return new Token(Tokens.EOF, Col, Line, String.Empty);
126
 
                }
127
 
                
128
 
                // The C# compiler has a fixed size length therefore we'll use a fixed size char array for identifiers
129
 
                // it's also faster than using a string builder.
130
 
                const int MAX_IDENTIFIER_LENGTH = 512;
131
 
                char[] identBuffer = new char[MAX_IDENTIFIER_LENGTH];
132
 
                
133
 
                string ReadIdent(char ch, out bool canBeKeyword)
134
 
                {
135
 
                        int peek;
136
 
                        int curPos     = 0;
137
 
                        canBeKeyword = true;
138
 
                        while (true) {
139
 
                                if (ch == '\\') {
140
 
                                        peek = ReaderPeek();
141
 
                                        if (peek != 'u' && peek != 'U') {
142
 
                                                errors.Error(Line, Col, "Identifiers can only contain unicode escape sequences");
143
 
                                        }
144
 
                                        canBeKeyword = false;
145
 
                                        string surrogatePair;
146
 
                                        ReadEscapeSequence(out ch, out surrogatePair);
147
 
                                        if (surrogatePair != null) {
148
 
                                                if (!char.IsLetterOrDigit(surrogatePair, 0)) {
149
 
                                                        errors.Error(Line, Col, "Unicode escape sequences in identifiers cannot be used to represent characters that are invalid in identifiers");
150
 
                                                }
151
 
                                                for (int i = 0; i < surrogatePair.Length - 1; i++) {
152
 
                                                        if (curPos < MAX_IDENTIFIER_LENGTH) {
153
 
                                                                identBuffer[curPos++] = surrogatePair[i];
154
 
                                                        }
155
 
                                                }
156
 
                                                ch = surrogatePair[surrogatePair.Length - 1];
157
 
                                        } else {
158
 
                                                if (!IsIdentifierPart(ch)) {
159
 
                                                        errors.Error(Line, Col, "Unicode escape sequences in identifiers cannot be used to represent characters that are invalid in identifiers");
160
 
                                                }
161
 
                                        }
162
 
                                }
163
 
                                
164
 
                                if (curPos < MAX_IDENTIFIER_LENGTH) {
165
 
                                        identBuffer[curPos++] = ch;
166
 
                                } else {
167
 
                                        errors.Error(Line, Col, String.Format("Identifier too long"));
168
 
                                        while (IsIdentifierPart(ReaderPeek())) {
169
 
                                                ReaderRead();
170
 
                                        }
171
 
                                        break;
172
 
                                }
173
 
                                peek = ReaderPeek();
174
 
                                if (IsIdentifierPart(peek) || peek == '\\') {
175
 
                                        ch = (char)ReaderRead();
176
 
                                } else {
177
 
                                        break;
178
 
                                }
179
 
                        }
180
 
                        return new String(identBuffer, 0, curPos);
181
 
                }
182
 
                
183
 
                Token ReadDigit(char ch, int x)
184
 
                {
185
 
                        unchecked { // prevent exception when ReaderPeek() = -1 is cast to char
186
 
                                int y = Line;
187
 
                                sb.Length = 0;
188
 
                                sb.Append(ch);
189
 
                                string prefix = null;
190
 
                                string suffix = null;
191
 
                                
192
 
                                bool ishex      = false;
193
 
                                bool isunsigned = false;
194
 
                                bool islong     = false;
195
 
                                bool isfloat    = false;
196
 
                                bool isdouble   = false;
197
 
                                bool isdecimal  = false;
198
 
                                
199
 
                                char peek = (char)ReaderPeek();
200
 
                                
201
 
                                if (ch == '.')  {
202
 
                                        isdouble = true;
203
 
                                        
204
 
                                        while (Char.IsDigit((char)ReaderPeek())) { // read decimal digits beyond the dot
205
 
                                                sb.Append((char)ReaderRead());
206
 
                                        }
207
 
                                        peek = (char)ReaderPeek();
208
 
                                } else if (ch == '0' && (peek == 'x' || peek == 'X')) {
209
 
                                        ReaderRead(); // skip 'x'
210
 
                                        sb.Length = 0; // Remove '0' from 0x prefix from the stringvalue
211
 
                                        while (IsHex((char)ReaderPeek())) {
212
 
                                                sb.Append((char)ReaderRead());
213
 
                                        }
214
 
                                        if (sb.Length == 0) {
215
 
                                                sb.Append('0'); // dummy value to prevent exception
216
 
                                                errors.Error(y, x, "Invalid hexadecimal integer literal");
217
 
                                        }
218
 
                                        ishex = true;
219
 
                                        prefix = "0x";
220
 
                                        peek = (char)ReaderPeek();
221
 
                                } else {
222
 
                                        while (Char.IsDigit((char)ReaderPeek())) {
223
 
                                                sb.Append((char)ReaderRead());
224
 
                                        }
225
 
                                        peek = (char)ReaderPeek();
226
 
                                }
227
 
                                
228
 
                                Token nextToken = null; // if we accidently read a 'dot'
229
 
                                if (peek == '.') { // read floating point number
230
 
                                        ReaderRead();
231
 
                                        peek = (char)ReaderPeek();
232
 
                                        if (!Char.IsDigit(peek)) {
233
 
                                                nextToken = new Token(Tokens.Dot, Col - 1, Line);
234
 
                                                peek = '.';
235
 
                                        } else {
236
 
                                                isdouble = true; // double is default
237
 
                                                if (ishex) {
238
 
                                                        errors.Error(y, x, String.Format("No hexadecimal floating point values allowed"));
239
 
                                                }
240
 
                                                sb.Append('.');
241
 
                                                
242
 
                                                while (Char.IsDigit((char)ReaderPeek())) { // read decimal digits beyond the dot
243
 
                                                        sb.Append((char)ReaderRead());
244
 
                                                }
245
 
                                                peek = (char)ReaderPeek();
246
 
                                        }
247
 
                                }
248
 
                                
249
 
                                if (peek == 'e' || peek == 'E') { // read exponent
250
 
                                        isdouble = true;
251
 
                                        sb.Append((char)ReaderRead());
252
 
                                        peek = (char)ReaderPeek();
253
 
                                        if (peek == '-' || peek == '+') {
254
 
                                                sb.Append((char)ReaderRead());
255
 
                                        }
256
 
                                        while (Char.IsDigit((char)ReaderPeek())) { // read exponent value
257
 
                                                sb.Append((char)ReaderRead());
258
 
                                        }
259
 
                                        isunsigned = true;
260
 
                                        peek = (char)ReaderPeek();
261
 
                                }
262
 
                                
263
 
                                if (peek == 'f' || peek == 'F') { // float value
264
 
                                        ReaderRead();
265
 
                                        suffix = "f";
266
 
                                        isfloat = true;
267
 
                                } else if (peek == 'd' || peek == 'D') { // double type suffix (obsolete, double is default)
268
 
                                        ReaderRead();
269
 
                                        suffix = "d";
270
 
                                        isdouble = true;
271
 
                                } else if (peek == 'm' || peek == 'M') { // decimal value
272
 
                                        ReaderRead();
273
 
                                        suffix = "m";
274
 
                                        isdecimal = true;
275
 
                                } else if (!isdouble) {
276
 
                                        if (peek == 'u' || peek == 'U') {
277
 
                                                ReaderRead();
278
 
                                                suffix = "u";
279
 
                                                isunsigned = true;
280
 
                                                peek = (char)ReaderPeek();
281
 
                                        }
282
 
                                        
283
 
                                        if (peek == 'l' || peek == 'L') {
284
 
                                                ReaderRead();
285
 
                                                peek = (char)ReaderPeek();
286
 
                                                islong = true;
287
 
                                                if (!isunsigned && (peek == 'u' || peek == 'U')) {
288
 
                                                        ReaderRead();
289
 
                                                        suffix = "Lu";
290
 
                                                        isunsigned = true;
291
 
                                                } else {
292
 
                                                        suffix = isunsigned ? "uL" : "L";
293
 
                                                }
294
 
                                        }
295
 
                                }
296
 
                                
297
 
                                string digit       = sb.ToString();
298
 
                                string stringValue = prefix + digit + suffix;
299
 
                                
300
 
                                if (isfloat) {
301
 
                                        float num;
302
 
                                        if (float.TryParse(digit, NumberStyles.Any, CultureInfo.InvariantCulture, out num)) {
303
 
                                                return new Token(Tokens.Literal, x, y, stringValue, num, LiteralFormat.DecimalNumber);
304
 
                                        } else {
305
 
                                                errors.Error(y, x, String.Format("Can't parse float {0}", digit));
306
 
                                                return new Token(Tokens.Literal, x, y, stringValue, 0f, LiteralFormat.DecimalNumber);
307
 
                                        }
308
 
                                }
309
 
                                if (isdecimal) {
310
 
                                        decimal num;
311
 
                                        if (decimal.TryParse(digit, NumberStyles.Any, CultureInfo.InvariantCulture, out num)) {
312
 
                                                return new Token(Tokens.Literal, x, y, stringValue, num, LiteralFormat.DecimalNumber);
313
 
                                        } else {
314
 
                                                errors.Error(y, x, String.Format("Can't parse decimal {0}", digit));
315
 
                                                return new Token(Tokens.Literal, x, y, stringValue, 0m, LiteralFormat.DecimalNumber);
316
 
                                        }
317
 
                                }
318
 
                                if (isdouble) {
319
 
                                        double num;
320
 
                                        if (double.TryParse(digit, NumberStyles.Any, CultureInfo.InvariantCulture, out num)) {
321
 
                                                return new Token(Tokens.Literal, x, y, stringValue, num, LiteralFormat.DecimalNumber);
322
 
                                        } else {
323
 
                                                errors.Error(y, x, String.Format("Can't parse double {0}", digit));
324
 
                                                return new Token(Tokens.Literal, x, y, stringValue, 0d, LiteralFormat.DecimalNumber);
325
 
                                        }
326
 
                                }
327
 
                                
328
 
                                // Try to determine a parsable value using ranges.
329
 
                                ulong result;
330
 
                                if (ishex) {
331
 
                                        if (!ulong.TryParse(digit, NumberStyles.HexNumber, null, out result)) {
332
 
                                                errors.Error(y, x, String.Format("Can't parse hexadecimal constant {0}", digit));
333
 
                                                return new Token(Tokens.Literal, x, y, stringValue.ToString(), 0, LiteralFormat.HexadecimalNumber);
334
 
                                        }
335
 
                                } else {
336
 
                                        if (!ulong.TryParse(digit, NumberStyles.Integer, null, out result)) {
337
 
                                                errors.Error(y, x, String.Format("Can't parse integral constant {0}", digit));
338
 
                                                return new Token(Tokens.Literal, x, y, stringValue.ToString(), 0, LiteralFormat.DecimalNumber);
339
 
                                        }
340
 
                                }
341
 
                                
342
 
                                if (result > long.MaxValue) {
343
 
                                        islong     = true;
344
 
                                        isunsigned = true;
345
 
                                } else if (result > uint.MaxValue) {
346
 
                                        islong = true;
347
 
                                } else if (islong == false && result > int.MaxValue) {
348
 
                                        isunsigned = true;
349
 
                                }
350
 
                                
351
 
                                Token token;
352
 
                                
353
 
                                LiteralFormat literalFormat = ishex ? LiteralFormat.HexadecimalNumber : LiteralFormat.DecimalNumber;
354
 
                                if (islong) {
355
 
                                        if (isunsigned) {
356
 
                                                ulong num;
357
 
                                                if (ulong.TryParse(digit, ishex ? NumberStyles.HexNumber : NumberStyles.Number, CultureInfo.InvariantCulture, out num)) {
358
 
                                                        token = new Token(Tokens.Literal, x, y, stringValue, num, literalFormat);
359
 
                                                } else {
360
 
                                                        errors.Error(y, x, String.Format("Can't parse unsigned long {0}", digit));
361
 
                                                        token = new Token(Tokens.Literal, x, y, stringValue, 0UL, literalFormat);
362
 
                                                }
363
 
                                        } else {
364
 
                                                long num;
365
 
                                                if (long.TryParse(digit, ishex ? NumberStyles.HexNumber : NumberStyles.Number, CultureInfo.InvariantCulture, out num)) {
366
 
                                                        token = new Token(Tokens.Literal, x, y, stringValue, num, literalFormat);
367
 
                                                } else {
368
 
                                                        errors.Error(y, x, String.Format("Can't parse long {0}", digit));
369
 
                                                        token = new Token(Tokens.Literal, x, y, stringValue, 0L, literalFormat);
370
 
                                                }
371
 
                                        }
372
 
                                } else {
373
 
                                        if (isunsigned) {
374
 
                                                uint num;
375
 
                                                if (uint.TryParse(digit, ishex ? NumberStyles.HexNumber : NumberStyles.Number, CultureInfo.InvariantCulture, out num)) {
376
 
                                                        token = new Token(Tokens.Literal, x, y, stringValue, num, literalFormat);
377
 
                                                } else {
378
 
                                                        errors.Error(y, x, String.Format("Can't parse unsigned int {0}", digit));
379
 
                                                        token = new Token(Tokens.Literal, x, y, stringValue, (uint)0, literalFormat);
380
 
                                                }
381
 
                                        } else {
382
 
                                                int num;
383
 
                                                if (int.TryParse(digit, ishex ? NumberStyles.HexNumber : NumberStyles.Number, CultureInfo.InvariantCulture, out num)) {
384
 
                                                        token = new Token(Tokens.Literal, x, y, stringValue, num, literalFormat);
385
 
                                                } else {
386
 
                                                        errors.Error(y, x, String.Format("Can't parse int {0}", digit));
387
 
                                                        token = new Token(Tokens.Literal, x, y, stringValue, 0, literalFormat);
388
 
                                                }
389
 
                                        }
390
 
                                }
391
 
                                token.next = nextToken;
392
 
                                return token;
393
 
                        }
394
 
                }
395
 
                
396
 
                void SkipString()
397
 
                {
398
 
                        int x = Col - 1;
399
 
                        int y = Line;
400
 
                        
401
 
                        bool doneNormally = false;
402
 
                        int nextChar;
403
 
                        while ((nextChar = ReaderRead()) != -1) {
404
 
                                char ch = (char)nextChar;
405
 
                                
406
 
                                if (ch == '"') {
407
 
                                        doneNormally = true;
408
 
                                        break;
409
 
                                }
410
 
                                
411
 
                                if (ch == '\\') {
412
 
                                        SkipEscapeSequence();
413
 
                                } else if (HandleLineEnd(ch)) {
414
 
                                        // call HandleLineEnd to ensure line numbers are still correct after the error
415
 
                                        errors.Error(y, x, String.Format("No new line is allowed inside a string literal"));
416
 
                                        break;
417
 
                                }
418
 
                        }
419
 
                        if (!doneNormally)
420
 
                                errors.Error(y, x, String.Format("End of file reached inside string literal"));
421
 
                }
422
 
 
423
 
                
424
 
                Token ReadString()
425
 
                {
426
 
                        int x = Col - 1;
427
 
                        int y = Line;
428
 
                        
429
 
                        sb.Length = 0;
430
 
                        originalValue.Length = 0;
431
 
                        originalValue.Append('"');
432
 
                        bool doneNormally = false;
433
 
                        int nextChar;
434
 
                        while ((nextChar = ReaderRead()) != -1) {
435
 
                                char ch = (char)nextChar;
436
 
                                
437
 
                                if (ch == '"') {
438
 
                                        doneNormally = true;
439
 
                                        originalValue.Append('"');
440
 
                                        break;
441
 
                                }
442
 
                                
443
 
                                if (ch == '\\') {
444
 
                                        originalValue.Append('\\');
445
 
                                        string surrogatePair;
446
 
                                        originalValue.Append(ReadEscapeSequence(out ch, out surrogatePair));
447
 
                                        if (surrogatePair != null) {
448
 
                                                sb.Append(surrogatePair);
449
 
                                        } else {
450
 
                                                sb.Append(ch);
451
 
                                        }
452
 
                                } else if (HandleLineEnd(ch)) {
453
 
                                        // call HandleLineEnd to ensure line numbers are still correct after the error
454
 
                                        errors.Error(y, x, String.Format("No new line is allowed inside a string literal"));
455
 
                                        break;
456
 
                                } else {
457
 
                                        originalValue.Append(ch);
458
 
                                        sb.Append(ch);
459
 
                                }
460
 
                        }
461
 
                        
462
 
                        if (!doneNormally) {
463
 
                                errors.Error(y, x, String.Format("End of file reached inside string literal"));
464
 
                        }
465
 
                        
466
 
                        return new Token(Tokens.Literal, x, y, originalValue.ToString(), sb.ToString(), LiteralFormat.StringLiteral);
467
 
                }
468
 
                
469
 
                Token ReadVerbatimString()
470
 
                {
471
 
                        sb.Length            = 0;
472
 
                        originalValue.Length = 0;
473
 
                        originalValue.Append("@\"");
474
 
                        Location startLocation = new Location(Col - 2, Line); // @ and " already read
475
 
                        int nextChar;
476
 
                        while ((nextChar = ReaderRead()) != -1) {
477
 
                                char ch = (char)nextChar;
478
 
                                
479
 
                                if (ch == '"') {
480
 
                                        if (ReaderPeek() != '"') {
481
 
                                                originalValue.Append('"');
482
 
                                                break;
483
 
                                        }
484
 
                                        originalValue.Append("\"\"");
485
 
                                        sb.Append('"');
486
 
                                        ReaderRead();
487
 
                                } else if (HandleLineEnd(ch)) {
488
 
                                        sb.Append("\r\n");
489
 
                                        originalValue.Append("\r\n");
490
 
                                } else {
491
 
                                        sb.Append(ch);
492
 
                                        originalValue.Append(ch);
493
 
                                }
494
 
                        }
495
 
                        
496
 
                        if (nextChar == -1) {
497
 
                                errors.Error(startLocation.Line, startLocation.Column, String.Format("End of file reached inside verbatim string literal"));
498
 
                        }
499
 
                        
500
 
                        return new Token(Tokens.Literal, startLocation, new Location(Col, Line), originalValue.ToString(), sb.ToString(), LiteralFormat.VerbatimStringLiteral);
501
 
                }
502
 
                
503
 
                readonly char[] escapeSequenceBuffer = new char[12];
504
 
                
505
 
                /// <summary>
506
 
                /// reads an escape sequence
507
 
                /// </summary>
508
 
                /// <param name="ch">The character represented by the escape sequence,
509
 
                /// or '\0' if there was an error or the escape sequence represents a character that
510
 
                /// can be represented only be a suggorate pair</param>
511
 
                /// <param name="surrogatePair">Null, except when the character represented
512
 
                /// by the escape sequence can only be represented by a surrogate pair (then the string
513
 
                /// contains the surrogate pair)</param>
514
 
                /// <returns>The escape sequence</returns>
515
 
                string ReadEscapeSequence(out char ch, out string surrogatePair)
516
 
                {
517
 
                        surrogatePair = null;
518
 
                        
519
 
                        int nextChar = ReaderRead();
520
 
                        if (nextChar == -1) {
521
 
                                errors.Error(Line, Col, String.Format("End of file reached inside escape sequence"));
522
 
                                ch = '\0';
523
 
                                return String.Empty;
524
 
                        }
525
 
                        int number;
526
 
                        char c = (char)nextChar;
527
 
                        int curPos              = 1;
528
 
                        escapeSequenceBuffer[0] = c;
529
 
                        switch (c)  {
530
 
                                case '\'':
531
 
                                        ch = '\'';
532
 
                                        break;
533
 
                                case '\"':
534
 
                                        ch = '\"';
535
 
                                        break;
536
 
                                case '\\':
537
 
                                        ch = '\\';
538
 
                                        break;
539
 
                                case '0':
540
 
                                        ch = '\0';
541
 
                                        break;
542
 
                                case 'a':
543
 
                                        ch = '\a';
544
 
                                        break;
545
 
                                case 'b':
546
 
                                        ch = '\b';
547
 
                                        break;
548
 
                                case 'f':
549
 
                                        ch = '\f';
550
 
                                        break;
551
 
                                case 'n':
552
 
                                        ch = '\n';
553
 
                                        break;
554
 
                                case 'r':
555
 
                                        ch = '\r';
556
 
                                        break;
557
 
                                case 't':
558
 
                                        ch = '\t';
559
 
                                        break;
560
 
                                case 'v':
561
 
                                        ch = '\v';
562
 
                                        break;
563
 
                                case 'u':
564
 
                                case 'x':
565
 
                                        // 16 bit unicode character
566
 
                                        c = (char)ReaderRead();
567
 
                                        number = GetHexNumber(c);
568
 
                                        escapeSequenceBuffer[curPos++] = c;
569
 
                                        
570
 
                                        if (number < 0) {
571
 
                                                errors.Error(Line, Col - 1, String.Format("Invalid char in literal : {0}", c));
572
 
                                        }
573
 
                                        for (int i = 0; i < 3; ++i) {
574
 
                                                if (IsHex((char)ReaderPeek())) {
575
 
                                                        c = (char)ReaderRead();
576
 
                                                        int idx = GetHexNumber(c);
577
 
                                                        escapeSequenceBuffer[curPos++] = c;
578
 
                                                        number = 16 * number + idx;
579
 
                                                } else {
580
 
                                                        break;
581
 
                                                }
582
 
                                        }
583
 
                                        ch = (char)number;
584
 
                                        break;
585
 
                                case 'U':
586
 
                                        // 32 bit unicode character
587
 
                                        number = 0;
588
 
                                        for (int i = 0; i < 8; ++i) {
589
 
                                                if (IsHex((char)ReaderPeek())) {
590
 
                                                        c = (char)ReaderRead();
591
 
                                                        int idx = GetHexNumber(c);
592
 
                                                        escapeSequenceBuffer[curPos++] = c;
593
 
                                                        number = 16 * number + idx;
594
 
                                                } else {
595
 
                                                        errors.Error(Line, Col - 1, String.Format("Invalid char in literal : {0}", (char)ReaderPeek()));
596
 
                                                        break;
597
 
                                                }
598
 
                                        }
599
 
                                        if (number > 0xffff) {
600
 
                                                ch = '\0';
601
 
                                                surrogatePair = char.ConvertFromUtf32(number);
602
 
                                        } else {
603
 
                                                ch = (char)number;
604
 
                                        }
605
 
                                        break;
606
 
                                default:
607
 
                                        errors.Error(Line, Col, String.Format("Unexpected escape sequence : {0}", c));
608
 
                                        ch = '\0';
609
 
                                        break;
610
 
                        }
611
 
                        return new String(escapeSequenceBuffer, 0, curPos);
612
 
                }
613
 
                
614
 
                void SkipEscapeSequence()
615
 
                {
616
 
                        int nextChar = ReaderRead();
617
 
                        if (nextChar == -1) {
618
 
                                errors.Error(Line, Col, String.Format("End of file reached inside escape sequence"));
619
 
                                return;
620
 
                        }
621
 
                        switch (nextChar)  {
622
 
                                case '\'':
623
 
                                case '\"':
624
 
                                case '\\':
625
 
                                case '0':
626
 
                                case 'a':
627
 
                                case 'b':
628
 
                                case 'f':
629
 
                                case 'n':
630
 
                                case 'r':
631
 
                                case 't':
632
 
                                case 'v':
633
 
                                case 'u':
634
 
                                        break;
635
 
                                case 'x':
636
 
                                        // 16 bit unicode character
637
 
                                        char c = (char)ReaderRead();
638
 
                                        if (GetHexNumber(c) < 0)
639
 
                                                errors.Error(Line, Col - 1, String.Format("Invalid char in literal : {0}", c));
640
 
                                        for (int i = 0; i < 3; ++i) {
641
 
                                                if (IsHex((char)ReaderPeek())) {
642
 
                                                        ReaderRead();
643
 
                                                } else {
644
 
                                                        break;
645
 
                                                }
646
 
                                        }
647
 
                                        break;
648
 
                                case 'U':
649
 
                                        for (int i = 0; i < 8; ++i) {
650
 
                                                if (IsHex((char)ReaderPeek())) {
651
 
                                                        ReaderRead();
652
 
                                                } else {
653
 
                                                        errors.Error(Line, Col - 1, String.Format("Invalid char in literal : {0}", (char)ReaderPeek()));
654
 
                                                        break;
655
 
                                                }
656
 
                                        }
657
 
                                        break;
658
 
                                default:
659
 
                                        errors.Error(Line, Col, String.Format("Unexpected escape sequence : {0}", nextChar));
660
 
                                        break;
661
 
                        }
662
 
                }
663
 
                
664
 
                Token ReadChar()
665
 
                {
666
 
                        int x = Col - 1;
667
 
                        int y = Line;
668
 
                        int nextChar = ReaderRead();
669
 
                        if (nextChar == -1 || HandleLineEnd((char)nextChar)) {
670
 
                                errors.Error(y, x, String.Format("End of line reached inside character literal"));
671
 
                                return null;
672
 
                        }
673
 
                        char ch = (char)nextChar;
674
 
                        char chValue = ch;
675
 
                        string escapeSequence = String.Empty;
676
 
                        if (ch == '\\') {
677
 
                                string surrogatePair;
678
 
                                escapeSequence = ReadEscapeSequence(out chValue, out surrogatePair);
679
 
                                if (surrogatePair != null) {
680
 
                                        errors.Error(y, x, String.Format("The unicode character must be represented by a surrogate pair and does not fit into a System.Char"));
681
 
                                }
682
 
                        }
683
 
                        
684
 
                        unchecked {
685
 
                                if ((char)ReaderRead() != '\'') {
686
 
                                        errors.Error(y, x, String.Format("Char not terminated"));
687
 
                                }
688
 
                        }
689
 
                        return new Token(Tokens.Literal, x, y, "'" + ch + escapeSequence + "'", chValue, LiteralFormat.CharLiteral);
690
 
                }
691
 
                
692
 
                Token ReadOperator(char ch)
693
 
                {
694
 
                        int x = Col - 1;
695
 
                        int y = Line;
696
 
                        switch (ch) {
697
 
                                case '+':
698
 
                                        switch (ReaderPeek()) {
699
 
                                                case '+':
700
 
                                                        ReaderRead();
701
 
                                                        return new Token(Tokens.Increment, x, y);
702
 
                                                case '=':
703
 
                                                        ReaderRead();
704
 
                                                        return new Token(Tokens.PlusAssign, x, y);
705
 
                                        }
706
 
                                        return new Token(Tokens.Plus, x, y);
707
 
                                case '-':
708
 
                                        switch (ReaderPeek()) {
709
 
                                                case '-':
710
 
                                                        ReaderRead();
711
 
                                                        return new Token(Tokens.Decrement, x, y);
712
 
                                                case '=':
713
 
                                                        ReaderRead();
714
 
                                                        return new Token(Tokens.MinusAssign, x, y);
715
 
                                                case '>':
716
 
                                                        ReaderRead();
717
 
                                                        return new Token(Tokens.Pointer, x, y);
718
 
                                        }
719
 
                                        return new Token(Tokens.Minus, x, y);
720
 
                                case '*':
721
 
                                        switch (ReaderPeek()) {
722
 
                                                case '=':
723
 
                                                        ReaderRead();
724
 
                                                        return new Token(Tokens.TimesAssign, x, y);
725
 
                                                default:
726
 
                                                        break;
727
 
                                        }
728
 
                                        return new Token(Tokens.Times, x, y);
729
 
                                case '/':
730
 
                                        switch (ReaderPeek()) {
731
 
                                                case '=':
732
 
                                                        ReaderRead();
733
 
                                                        return new Token(Tokens.DivAssign, x, y);
734
 
                                        }
735
 
                                        return new Token(Tokens.Div, x, y);
736
 
                                case '%':
737
 
                                        switch (ReaderPeek()) {
738
 
                                                case '=':
739
 
                                                        ReaderRead();
740
 
                                                        return new Token(Tokens.ModAssign, x, y);
741
 
                                        }
742
 
                                        return new Token(Tokens.Mod, x, y);
743
 
                                case '&':
744
 
                                        switch (ReaderPeek()) {
745
 
                                                case '&':
746
 
                                                        ReaderRead();
747
 
                                                        return new Token(Tokens.LogicalAnd, x, y);
748
 
                                                case '=':
749
 
                                                        ReaderRead();
750
 
                                                        return new Token(Tokens.BitwiseAndAssign, x, y);
751
 
                                        }
752
 
                                        return new Token(Tokens.BitwiseAnd, x, y);
753
 
                                case '|':
754
 
                                        switch (ReaderPeek()) {
755
 
                                                case '|':
756
 
                                                        ReaderRead();
757
 
                                                        return new Token(Tokens.LogicalOr, x, y);
758
 
                                                case '=':
759
 
                                                        ReaderRead();
760
 
                                                        return new Token(Tokens.BitwiseOrAssign, x, y);
761
 
                                        }
762
 
                                        return new Token(Tokens.BitwiseOr, x, y);
763
 
                                case '^':
764
 
                                        switch (ReaderPeek()) {
765
 
                                                case '=':
766
 
                                                        ReaderRead();
767
 
                                                        return new Token(Tokens.XorAssign, x, y);
768
 
                                                default:
769
 
                                                        break;
770
 
                                        }
771
 
                                        return new Token(Tokens.Xor, x, y);
772
 
                                case '!':
773
 
                                        switch (ReaderPeek()) {
774
 
                                                case '=':
775
 
                                                        ReaderRead();
776
 
                                                        return new Token(Tokens.NotEqual, x, y);
777
 
                                        }
778
 
                                        return new Token(Tokens.Not, x, y);
779
 
                                case '~':
780
 
                                        return new Token(Tokens.BitwiseComplement, x, y);
781
 
                                case '=':
782
 
                                        switch (ReaderPeek()) {
783
 
                                                case '=':
784
 
                                                        ReaderRead();
785
 
                                                        return new Token(Tokens.Equal, x, y);
786
 
                                                case '>':
787
 
                                                        ReaderRead();
788
 
                                                        return new Token(Tokens.LambdaArrow, x, y);
789
 
                                        }
790
 
                                        return new Token(Tokens.Assign, x, y);
791
 
                                case '<':
792
 
                                        switch (ReaderPeek()) {
793
 
                                                case '<':
794
 
                                                        ReaderRead();
795
 
                                                        switch (ReaderPeek()) {
796
 
                                                                case '=':
797
 
                                                                        ReaderRead();
798
 
                                                                        return new Token(Tokens.ShiftLeftAssign, x, y);
799
 
                                                                default:
800
 
                                                                        break;
801
 
                                                        }
802
 
                                                        return new Token(Tokens.ShiftLeft, x, y);
803
 
                                                case '=':
804
 
                                                        ReaderRead();
805
 
                                                        return new Token(Tokens.LessEqual, x, y);
806
 
                                        }
807
 
                                        return new Token(Tokens.LessThan, x, y);
808
 
                                case '>':
809
 
                                        switch (ReaderPeek()) {
810
 
                                                        // Removed because of generics:
811
 
//                                              case '>':
812
 
//                                                      ReaderRead();
813
 
//                                                      if (ReaderPeek() != -1) {
814
 
//                                                              switch ((char)ReaderPeek()) {
815
 
//                                                                      case '=':
816
 
//                                                                              ReaderRead();
817
 
//                                                                              return new Token(Tokens.ShiftRightAssign, x, y);
818
 
//                                                                      default:
819
 
//                                                                              break;
820
 
//                                                              }
821
 
//                                                      }
822
 
//                                                      return new Token(Tokens.ShiftRight, x, y);
823
 
                                                case '=':
824
 
                                                        ReaderRead();
825
 
                                                        return new Token(Tokens.GreaterEqual, x, y);
826
 
                                        }
827
 
                                        return new Token(Tokens.GreaterThan, x, y);
828
 
                                case '?':
829
 
                                        if (ReaderPeek() == '?') {
830
 
                                                ReaderRead();
831
 
                                                return new Token(Tokens.DoubleQuestion, x, y);
832
 
                                        }
833
 
                                        return new Token(Tokens.Question, x, y);
834
 
                                case ';':
835
 
                                        return new Token(Tokens.Semicolon, x, y);
836
 
                                case ':':
837
 
                                        if (ReaderPeek() == ':') {
838
 
                                                ReaderRead();
839
 
                                                return new Token(Tokens.DoubleColon, x, y);
840
 
                                        }
841
 
                                        return new Token(Tokens.Colon, x, y);
842
 
                                case ',':
843
 
                                        return new Token(Tokens.Comma, x, y);
844
 
                                case '.':
845
 
                                        // Prevent OverflowException when ReaderPeek returns -1
846
 
                                        int tmp = ReaderPeek();
847
 
                                        if (tmp > 0 && Char.IsDigit((char)tmp)) {
848
 
                                                return ReadDigit('.', Col - 1);
849
 
                                        }
850
 
                                        return new Token(Tokens.Dot, x, y);
851
 
                                case ')':
852
 
                                        return new Token(Tokens.CloseParenthesis, x, y);
853
 
                                case '(':
854
 
                                        return new Token(Tokens.OpenParenthesis, x, y);
855
 
                                case ']':
856
 
                                        return new Token(Tokens.CloseSquareBracket, x, y);
857
 
                                case '[':
858
 
                                        return new Token(Tokens.OpenSquareBracket, x, y);
859
 
                                case '}':
860
 
                                        return new Token(Tokens.CloseCurlyBrace, x, y);
861
 
                                case '{':
862
 
                                        return new Token(Tokens.OpenCurlyBrace, x, y);
863
 
                                default:
864
 
                                        return null;
865
 
                        }
866
 
                }
867
 
                
868
 
                void ReadComment()
869
 
                {
870
 
                        switch (ReaderRead()) {
871
 
                                case '*':
872
 
                                        ReadMultiLineComment();
873
 
                                        isAtLineBegin = false;
874
 
                                        break;
875
 
                                case '/':
876
 
                                        if (ReaderPeek() == '/') {
877
 
                                                ReaderRead();
878
 
                                                ReadSingleLineComment(CommentType.Documentation);
879
 
                                        } else {
880
 
                                                ReadSingleLineComment(CommentType.SingleLine);
881
 
                                        }
882
 
                                        isAtLineBegin = true;
883
 
                                        break;
884
 
                                default:
885
 
                                        errors.Error(Line, Col, String.Format("Error while reading comment"));
886
 
                                        break;
887
 
                        }
888
 
                }
889
 
                
890
 
                string ReadCommentToEOL()
891
 
                {
892
 
                        if (specialCommentHash == null) {
893
 
                                return ReadToEndOfLine();
894
 
                        }
895
 
                        sb.Length = 0;
896
 
                        StringBuilder curWord = new StringBuilder();
897
 
                        int nextChar;
898
 
                        while ((nextChar = ReaderRead()) != -1) {
899
 
                                char ch = (char)nextChar;
900
 
                                
901
 
                                if (HandleLineEnd(ch)) {
902
 
                                        break;
903
 
                                }
904
 
                                
905
 
                                sb.Append(ch);
906
 
                                if (IsIdentifierPart(nextChar)) {
907
 
                                        curWord.Append(ch);
908
 
                                } else {
909
 
                                        string tag = curWord.ToString();
910
 
                                        curWord.Length = 0;
911
 
                                        if (specialCommentHash.ContainsKey(tag)) {
912
 
                                                Location p = new Location(Col, Line);
913
 
                                                string comment = ReadToEndOfLine ();
914
 
                                                this.TagComments.Add(new TagComment(tag, comment, isAtLineBegin, p, new Location(Col, Line)));
915
 
                                                break;
916
 
                                        }
917
 
                                }
918
 
                        }
919
 
                        return sb.ToString();
920
 
                }
921
 
                
922
 
                void ReadSingleLineComment(CommentType commentType)
923
 
                {
924
 
                        if (this.SkipAllComments) {
925
 
                                SkipToEndOfLine();
926
 
                        } else {
927
 
                                specialTracker.StartComment(commentType, isAtLineBegin, new Location(Col, Line));
928
 
                                specialTracker.AddString(ReadCommentToEOL());
929
 
                                specialTracker.FinishComment(new Location(lineBreakPosition.Column, lineBreakPosition.Line));
930
 
                        }
931
 
                }
932
 
                
933
 
                void ReadMultiLineComment()
934
 
                {
935
 
                        int nextChar;
936
 
                        if (this.SkipAllComments) {
937
 
                                while ((nextChar = ReaderRead()) != -1) {
938
 
                                        char ch = (char)nextChar;
939
 
                                        if (ch == '*' && ReaderPeek() == '/') {
940
 
                                                ReaderRead();
941
 
                                                return;
942
 
                                        } else {
943
 
                                                HandleLineEnd(ch);
944
 
                                        }
945
 
                                }
946
 
                        } else {
947
 
                                specialTracker.StartComment(CommentType.Block, isAtLineBegin, new Location(Col, Line));
948
 
                                
949
 
                                // sc* = special comment handling (TO DO markers)
950
 
                                string scTag = null; // is set to non-null value when we are inside a comment marker
951
 
                                StringBuilder scCurWord = new StringBuilder(); // current word, (scTag == null) or comment (when scTag != null)
952
 
                                Location scStartLocation = Location.Empty;
953
 
                                
954
 
                                while ((nextChar = ReaderRead()) != -1) {
955
 
                                        char ch = (char)nextChar;
956
 
                                        
957
 
                                        if (HandleLineEnd(ch)) {
958
 
                                                if (scTag != null) {
959
 
                                                        this.TagComments.Add(new TagComment(scTag, scCurWord.ToString(), isAtLineBegin, scStartLocation, new Location(Col, Line)));
960
 
                                                        scTag = null;
961
 
                                                }
962
 
                                                scCurWord.Length = 0;
963
 
                                                specialTracker.AddString(Environment.NewLine);
964
 
                                                continue;
965
 
                                        }
966
 
                                        
967
 
                                        // End of multiline comment reached ?
968
 
                                        if (ch == '*' && ReaderPeek() == '/') {
969
 
                                                if (scTag != null) {
970
 
                                                        this.TagComments.Add(new TagComment(scTag, scCurWord.ToString(), isAtLineBegin, scStartLocation, new Location(Col, Line)));
971
 
                                                }
972
 
                                                ReaderRead();
973
 
                                                specialTracker.FinishComment(new Location(Col, Line));
974
 
                                                return;
975
 
                                        }
976
 
                                        specialTracker.AddChar(ch);
977
 
                                        if (scTag != null || IsIdentifierPart(ch)) {
978
 
                                                scCurWord.Append(ch);
979
 
                                        } else {
980
 
                                                if (specialCommentHash != null && specialCommentHash.ContainsKey(scCurWord.ToString())) {
981
 
                                                        scTag = scCurWord.ToString();
982
 
                                                        scStartLocation = new Location(Col, Line);
983
 
                                                }
984
 
                                                scCurWord.Length = 0;
985
 
                                        }
986
 
                                }
987
 
                                specialTracker.FinishComment(new Location(Col, Line));
988
 
                        }
989
 
                        // Reached EOF before end of multiline comment.
990
 
                        errors.Error(Line, Col, String.Format("Reached EOF before the end of a multiline comment"));
991
 
                }
992
 
                
993
 
                /// <summary>
994
 
                /// Skips to the end of the current code block.
995
 
                /// For this, the lexer must have read the next token AFTER the token opening the
996
 
                /// block (so that Lexer.Token is the block-opening token, not Lexer.LookAhead).
997
 
                /// After the call, Lexer.LookAhead will be the block-closing token.
998
 
                /// </summary>
999
 
                public override void SkipCurrentBlock(int targetToken)
1000
 
                {
1001
 
                        int braceCount = 0;
1002
 
                        while (curToken != null) {
1003
 
                                if (curToken.kind == Tokens.OpenCurlyBrace) {
1004
 
                                        ++braceCount;
1005
 
                                } else if (curToken.kind == Tokens.CloseCurlyBrace) {
1006
 
                                        if (--braceCount < 0)
1007
 
                                                return;
1008
 
                                }
1009
 
                                lastToken = curToken;
1010
 
                                curToken = curToken.next;
1011
 
                        }
1012
 
                        isAtLineBegin = true;
1013
 
                        int nextChar;
1014
 
                        while ((nextChar = ReaderRead()) != -1) {
1015
 
                                switch (nextChar) {
1016
 
                                        case '{':
1017
 
                                                isAtLineBegin = false;
1018
 
                                                braceCount++;
1019
 
                                                break;
1020
 
                                        case '}':
1021
 
                                                isAtLineBegin = false;
1022
 
                                                if (--braceCount < 0) {
1023
 
                                                        curToken = new Token(Tokens.CloseCurlyBrace, Col - 1, Line);
1024
 
                                                        return;
1025
 
                                                }
1026
 
                                                break;
1027
 
                                        case '/':
1028
 
                                                int peek = ReaderPeek();
1029
 
                                                if (peek == '/' || peek == '*') {
1030
 
                                                        ReadComment();
1031
 
                                                }
1032
 
                                                isAtLineBegin = false;
1033
 
                                                break;
1034
 
                                        case '#':
1035
 
                                                ReadPreProcessingDirective();
1036
 
                                                isAtLineBegin = false;
1037
 
                                                break;
1038
 
                                        case '"':
1039
 
                                                SkipString();
1040
 
                                                isAtLineBegin = false;
1041
 
                                                break;
1042
 
                                        case '\'':
1043
 
                                                ReadChar();
1044
 
                                                isAtLineBegin = false;
1045
 
                                                break;
1046
 
                                        case '\r':
1047
 
                                        case '\n':
1048
 
                                                HandleLineEnd((char)nextChar);
1049
 
                                                isAtLineBegin = true;
1050
 
                                                break;
1051
 
                                        case '@':
1052
 
                                                int next = ReaderRead();
1053
 
                                                if (next == -1) {
1054
 
                                                        errors.Error(Line, Col, String.Format("EOF after @"));
1055
 
                                                } else if (next == '"') {
1056
 
                                                        ReadVerbatimString();
1057
 
                                                }
1058
 
                                                isAtLineBegin = false;
1059
 
                                                break;
1060
 
                                }
1061
 
                        }
1062
 
                        curToken = new Token(Tokens.EOF, Col, Line);
1063
 
                }
1064
 
                
1065
 
                public override IDictionary<string, object> ConditionalCompilationSymbols {
1066
 
                        get { return conditionalCompilation.Symbols; }
1067
 
                }
1068
 
                
1069
 
                public override void SetConditionalCompilationSymbols (string symbols)
1070
 
                {
1071
 
                        foreach (string symbol in GetSymbols (symbols)) {
1072
 
                                conditionalCompilation.Define (symbol);
1073
 
                        }
1074
 
                }
1075
 
                
1076
 
                
1077
 
                ConditionalCompilation conditionalCompilation = new ConditionalCompilation();
1078
 
                
1079
 
                void ReadPreProcessingDirective()
1080
 
                {
1081
 
                        PreprocessingDirective d = ReadPreProcessingDirectiveInternal(true, true);
1082
 
                        this.specialTracker.AddPreprocessingDirective(d);
1083
 
                        
1084
 
                        if (EvaluateConditionalCompilation) {
1085
 
                                switch (d.Cmd) {
1086
 
                                        case "#define":
1087
 
                                                conditionalCompilation.Define(d.Arg);
1088
 
                                                break;
1089
 
                                        case "#undef":
1090
 
                                                conditionalCompilation.Undefine(d.Arg);
1091
 
                                                break;
1092
 
                                        case "#if":
1093
 
                                                if (!conditionalCompilation.Evaluate(d.Expression)) {
1094
 
                                                        // skip to valid #elif or #else or #endif
1095
 
                                                        int level = 1;
1096
 
                                                        while (true) {
1097
 
                                                                d = SkipToPreProcessingDirective(false, level == 1);
1098
 
                                                                if (d == null)
1099
 
                                                                        break;
1100
 
                                                                if (d.Cmd == "#if") {
1101
 
                                                                        level++;
1102
 
                                                                } else if (d.Cmd == "#endif") {
1103
 
                                                                        level--;
1104
 
                                                                        if (level == 0)
1105
 
                                                                                break;
1106
 
                                                                } else if (level == 1 &&  (d.Cmd == "#else"
1107
 
                                                                                           || d.Cmd == "#elif" && conditionalCompilation.Evaluate(d.Expression)))
1108
 
                                                                {
1109
 
                                                                        break;
1110
 
                                                                }
1111
 
                                                        }
1112
 
                                                        if (d != null)
1113
 
                                                                this.specialTracker.AddPreprocessingDirective(d);
1114
 
                                                }
1115
 
                                                break;
1116
 
                                        case "#elif":
1117
 
                                        case "#else":
1118
 
                                                // we already visited the #if part or a previous #elif part, so skip until #endif
1119
 
                                                {
1120
 
                                                        int level = 1;
1121
 
                                                        while (true) {
1122
 
                                                                d = SkipToPreProcessingDirective(false, false);
1123
 
                                                                if (d == null)
1124
 
                                                                        break;
1125
 
                                                                if (d.Cmd == "#if") {
1126
 
                                                                        level++;
1127
 
                                                                } else if (d.Cmd == "#endif") {
1128
 
                                                                        level--;
1129
 
                                                                        if (level == 0)
1130
 
                                                                                break;
1131
 
                                                                }
1132
 
                                                        }
1133
 
                                                        if (d != null)
1134
 
                                                                this.specialTracker.AddPreprocessingDirective(d);
1135
 
                                                }
1136
 
                                                break;
1137
 
                                }
1138
 
                        }
1139
 
                }
1140
 
                
1141
 
                PreprocessingDirective SkipToPreProcessingDirective(bool parseIfExpression, bool parseElifExpression)
1142
 
                {
1143
 
                        int c;
1144
 
                        while (true) {
1145
 
                                PPWhitespace();
1146
 
                                c = ReaderRead();
1147
 
                                if (c == -1) {
1148
 
                                        errors.Error(Line, Col, String.Format("Reached EOF but expected #endif"));
1149
 
                                        return null;
1150
 
                                } else if (c == '#') {
1151
 
                                        break;
1152
 
                                } else {
1153
 
                                        if (c != '\n') // only skip non empty lines.
1154
 
                                                SkipToEndOfLine();
1155
 
                                }
1156
 
                        }
1157
 
                        return ReadPreProcessingDirectiveInternal(parseIfExpression, parseElifExpression);
1158
 
                }
1159
 
                
1160
 
                PreprocessingDirective ReadPreProcessingDirectiveInternal(bool parseIfExpression, bool parseElifExpression)
1161
 
                {
1162
 
                        Location start = new Location(Col - 1, Line);
1163
 
                        
1164
 
                        // skip spaces between # and the directive
1165
 
                        PPWhitespace();
1166
 
                        
1167
 
                        bool canBeKeyword;
1168
 
                        string directive = ReadIdent('#', out canBeKeyword);
1169
 
                        
1170
 
                        PPWhitespace();
1171
 
                        if (parseIfExpression && directive == "#if" || parseElifExpression && directive == "#elif") {
1172
 
                                recordedText.Length = 0;
1173
 
                                recordRead = true;
1174
 
                                Ast.Expression expr = PPExpression();
1175
 
                                string arg = recordedText.ToString ();
1176
 
                                recordRead = false;
1177
 
                                
1178
 
                                Location endLocation = new Location(Col, Line);
1179
 
                                int c = ReaderRead();
1180
 
                                if (c >= 0 && !HandleLineEnd((char)c)) {
1181
 
                                        if (c == '/' && ReaderRead() == '/') {
1182
 
                                                // comment to end of line
1183
 
                                        } else {
1184
 
                                                errors.Error(Col, Line, "Expected end of line");
1185
 
                                        }
1186
 
                                        SkipToEndOfLine(); // skip comment
1187
 
                                }
1188
 
                                return new PreprocessingDirective(directive, arg, start, endLocation) { Expression = expr, LastLineEnd = lastLineEnd };
1189
 
                        } else {
1190
 
                                Location endLocation = new Location(Col, Line);
1191
 
                                string arg = ReadToEndOfLine();
1192
 
                                endLocation.Column += arg.Length;
1193
 
                                int pos = arg.IndexOf("//");
1194
 
                                if (pos >= 0)
1195
 
                                        arg = arg.Substring(0, pos);
1196
 
                                arg = arg.Trim();
1197
 
                                return new PreprocessingDirective(directive, arg, start, endLocation) { LastLineEnd = lastLineEnd };
1198
 
                        }
1199
 
                }
1200
 
                
1201
 
                void PPWhitespace()
1202
 
                {
1203
 
                        while (ReaderPeek() == ' ' || ReaderPeek() == '\t')
1204
 
                                ReaderRead();
1205
 
                }
1206
 
                
1207
 
                public Ast.Expression PPExpression()
1208
 
                {
1209
 
                        Ast.Expression expr = PPAndExpression();
1210
 
                        while (ReaderPeek() == '|') {
1211
 
                                Token token = ReadOperator((char)ReaderRead());
1212
 
                                if (token == null || token.kind != Tokens.LogicalOr) {
1213
 
                                        return expr;
1214
 
                                }
1215
 
                                Ast.Expression expr2 = PPAndExpression();
1216
 
                                expr = new Ast.BinaryOperatorExpression(expr, Ast.BinaryOperatorType.LogicalOr, expr2);
1217
 
                        }
1218
 
                        return expr;
1219
 
                }
1220
 
                
1221
 
                Ast.Expression PPAndExpression()
1222
 
                {
1223
 
                        Ast.Expression expr = PPEqualityExpression();
1224
 
                        while (ReaderPeek() == '&') {
1225
 
                                Token token = ReadOperator((char)ReaderRead());
1226
 
                                if (token == null || token.kind != Tokens.LogicalAnd) {
1227
 
                                        break;
1228
 
                                }
1229
 
                                Ast.Expression expr2 = PPEqualityExpression();
1230
 
                                expr = new Ast.BinaryOperatorExpression(expr, Ast.BinaryOperatorType.LogicalAnd, expr2);
1231
 
                        }
1232
 
                        return expr;
1233
 
                }
1234
 
                
1235
 
                Ast.Expression PPEqualityExpression()
1236
 
                {
1237
 
                        Ast.Expression expr = PPUnaryExpression();
1238
 
                        while (ReaderPeek() == '=' || ReaderPeek() == '!') {
1239
 
                                Token token = ReadOperator((char)ReaderRead());
1240
 
                                if (token == null || token.kind != Tokens.Equal && token.kind != Tokens.NotEqual) {
1241
 
                                        break;
1242
 
                                }
1243
 
                                Ast.Expression expr2 = PPUnaryExpression();
1244
 
                                expr = new Ast.BinaryOperatorExpression(expr, token.kind == Tokens.Equal ? Ast.BinaryOperatorType.Equality : Ast.BinaryOperatorType.InEquality, expr2);
1245
 
                        }
1246
 
                        return expr;
1247
 
                }
1248
 
                
1249
 
                Ast.Expression PPUnaryExpression()
1250
 
                {
1251
 
                        PPWhitespace();
1252
 
                        if (ReaderPeek() == '!') {
1253
 
                                ReaderRead();
1254
 
                                PPWhitespace();
1255
 
                                return new Ast.UnaryOperatorExpression(PPUnaryExpression(), Ast.UnaryOperatorType.Not);
1256
 
                        } else {
1257
 
                                return PPPrimaryExpression();
1258
 
                        }
1259
 
                }
1260
 
                
1261
 
                Ast.Expression PPPrimaryExpression()
1262
 
                {
1263
 
                        int c = ReaderRead();
1264
 
                        if (c < 0)
1265
 
                                return Ast.Expression.Null;
1266
 
                        if (c == '(') {
1267
 
                                Ast.Expression expr = new Ast.ParenthesizedExpression(PPExpression());
1268
 
                                PPWhitespace();
1269
 
                                if (ReaderRead() != ')')
1270
 
                                        errors.Error(Col, Line, "Expected ')'");
1271
 
                                PPWhitespace();
1272
 
                                return expr;
1273
 
                        } else {
1274
 
                                if (c != '_' && !char.IsLetterOrDigit((char)c) && c != '\\')
1275
 
                                        errors.Error(Col, Line, "Expected conditional symbol");
1276
 
                                bool canBeKeyword;
1277
 
                                string symbol = ReadIdent((char)c, out canBeKeyword);
1278
 
                                PPWhitespace();
1279
 
                                if (canBeKeyword && symbol == "true")
1280
 
                                        return new Ast.PrimitiveExpression(true, "true");
1281
 
                                else if (canBeKeyword && symbol == "false")
1282
 
                                        return new Ast.PrimitiveExpression(false, "false");
1283
 
                                else
1284
 
                                        return new Ast.IdentifierExpression(symbol);
1285
 
                        }
1286
 
                }
1287
 
        }
1288
 
}