~ubuntu-branches/ubuntu/trusty/monodevelop/trusty-proposed

return new Token(Tokens.LiteralDecimal, x, y, sb.ToString(), Decimal.Parse(digit, NumberStyles.Currency | NumberStyles.AllowExponent, CultureInfo.InvariantCulture));

744

}

745

if (isDouble) {

746

return new Token(Tokens.LiteralDouble, x, y, sb.ToString(), Double.Parse(digit, CultureInfo.InvariantCulture));

747

}

748

} catch (FormatException) {

749

errors.Error(Line, Col, String.Format("{0} is not a parseable number", digit));

750

if (isSingle)

751

return new Token(Tokens.LiteralSingle, x, y, sb.ToString(), 0f);

752

if (isDecimal)

753

return new Token(Tokens.LiteralDecimal, x, y, sb.ToString(), 0m);

754

if (isDouble)

755

return new Token(Tokens.LiteralDouble, x, y, sb.ToString(), 0.0);

756

}

757

Token token;

758

try {

759

token = new Token(Tokens.LiteralInteger, x, y, sb.ToString(), Int32.Parse(digit, isHex ? NumberStyles.HexNumber : NumberStyles.Number));

760

} catch (Exception) {

761

try {

762

token = new Token(Tokens.LiteralInteger, x, y, sb.ToString(), Int64.Parse(digit, isHex ? NumberStyles.HexNumber : NumberStyles.Number));

763

} catch (FormatException) {

764

errors.Error(Line, Col, String.Format("{0} is not a parseable number", digit));

765

// fallback, when nothing helps :)

766

token = new Token(Tokens.LiteralInteger, x, y, sb.ToString(), 0);

767

} catch (OverflowException) {

768

errors.Error(Line, Col, String.Format("{0} is too long for a integer literal", digit));

769

// fallback, when nothing helps :)

770

token = new Token(Tokens.LiteralInteger, x, y, sb.ToString(), 0);

771

}

772

}

773

token.next = nextToken;

774

return token;

775

}

776

777

void ReadPreprocessorDirective()

778

{

779

TextLocation start = new TextLocation(Line, Col - 1);

780

string directive = ReadIdent('#');

781

// TODO : expression parser for PP directives

782

// needed for proper conversion to e. g. C#

783

string argument = ReadToEndOfLine();

784

// this.specialTracker.AddPreprocessingDirective(new PreprocessingDirective(directive, argument.Trim(), start, new AstLocation(start.Line, start.Column + directive.Length + argument.Length)));

785

}

786

787

string ReadDate()

788

{

789

char ch = '\0';

790

sb.Length = 0;

791

int nextChar;

792

while ((nextChar = ReaderRead()) != -1) {

793

ch = (char)nextChar;

794

if (ch == '#') {

795

break;

796

} else if (ch == '\n') {

797

errors.Error(Line, Col, String.Format("No return allowed inside Date literal"));

798

} else {

799

sb.Append(ch);

800

}

801

}

802

if (ch != '#') {

803

errors.Error(Line, Col, String.Format("End of File reached before Date literal terminated"));

804

}

805

return sb.ToString();

806

}

807

808

string ReadString()

809

{

810

char ch = '\0';

811

sb.Length = 0;

812

int nextChar;

813

while ((nextChar = ReaderRead()) != -1) {

814

ch = (char)nextChar;

815

if (ch == '"') {

816

if (ReaderPeek() != -1 && ReaderPeek() == '"') {

817

sb.Append('"');

818

ReaderRead();

819

} else {

820

break;

821

}

822

} else if (ch == '\n') {

823

errors.Error(Line, Col, String.Format("No return allowed inside String literal"));

824

} else {

825

sb.Append(ch);

826

}

827

}

828

if (ch != '"') {

829

errors.Error(Line, Col, String.Format("End of File reached before String terminated "));

830

}

831

return sb.ToString();

832

}

833

834

void ReadComment()

835

{

836

TextLocation startPos = new TextLocation(Line, Col);

837

sb.Length = 0;

838

StringBuilder curWord = specialCommentHash != null ? new StringBuilder() : null;

839

int missingApostrophes = 2; // no. of ' missing until it is a documentation comment

840

int nextChar;

841

while ((nextChar = ReaderRead()) != -1) {

842

char ch = (char)nextChar;

843

844

if (HandleLineEnd(ch)) {

845

break;

846

}

847

848

sb.Append(ch);

849

850

if (missingApostrophes > 0) {

851

if (ch == '\'' || ch == '\u2018' || ch == '\u2019') {

852

if (--missingApostrophes == 0) {

853

// specialTracker.StartComment(CommentType.Documentation, isAtLineBegin, startPos);

854

sb.Length = 0;

855

}

856

} else {

857

// specialTracker.StartComment(CommentType.SingleLine, isAtLineBegin, startPos);

858

missingApostrophes = 0;

859

}

860

}

861

862

if (specialCommentHash != null) {

863

if (Char.IsLetter(ch)) {

864

curWord.Append(ch);

865

} else {

866

string tag = curWord.ToString();

867

curWord.Length = 0;

868

if (specialCommentHash.ContainsKey(tag)) {

869

TextLocation p = new TextLocation(Line, Col);

870

string comment = ch + ReadToEndOfLine();

871

// this.TagComments.Add(new TagComment(tag, comment, isAtLineBegin, p, new Location(Col, Line)));

872

sb.Append(comment);

873

break;

874

}

875

}

876

}

877

}

878

// if (missingApostrophes > 0) {

879

// specialTracker.StartComment(CommentType.SingleLine, isAtLineBegin, startPos);

880

// }

881

// specialTracker.AddString(sb.ToString());

882

// specialTracker.FinishComment(new Location(Col, Line));

883

}

884

885

Token ReadOperator(char ch)

886

{

887

int x = Col - 1;

888

int y = Line;

889

switch(ch) {

890

case '+':

891

switch (ReaderPeek()) {

892

case '=':

893

ReaderRead();

894

return new Token(Tokens.PlusAssign, x, y);

895

default:

896

break;

897

}

898

return new Token(Tokens.Plus, x, y);

899

case '-':

900

switch (ReaderPeek()) {

901

case '=':

902

ReaderRead();

903

return new Token(Tokens.MinusAssign, x, y);

904

default:

905

break;

906

}

907

return new Token(Tokens.Minus, x, y);

908

case '*':

909

switch (ReaderPeek()) {

910

case '=':

911

ReaderRead();

912

return new Token(Tokens.TimesAssign, x, y);

913

default:

914

break;

915

}

916

return new Token(Tokens.Times, x, y, "*");

917

case '/':

918

switch (ReaderPeek()) {

919

case '=':

920

ReaderRead();

921

return new Token(Tokens.DivAssign, x, y);

922

default:

923

break;

924

}

925

return new Token(Tokens.Div, x, y);

926

case '\\':

927

switch (ReaderPeek()) {

928

case '=':

929

ReaderRead();

930

return new Token(Tokens.DivIntegerAssign, x, y);

931

default:

932

break;

933

}

934

return new Token(Tokens.DivInteger, x, y);

935

case '&':

936

switch (ReaderPeek()) {

937

case '=':

938

ReaderRead();

939

return new Token(Tokens.ConcatStringAssign, x, y);

940

default:

941

break;

942

}

943

return new Token(Tokens.ConcatString, x, y);

944

case '^':

945

switch (ReaderPeek()) {

946

case '=':

947

ReaderRead();

948

return new Token(Tokens.PowerAssign, x, y);

949

default:

950

break;

951

}

952

return new Token(Tokens.Power, x, y);

953

case ':':

954

if (ReaderPeek() == '=') {

955

ReaderRead();

956

return new Token(Tokens.ColonAssign, x, y);

957

}

958

return new Token(Tokens.Colon, x, y);

959

case '=':

960

return new Token(Tokens.Assign, x, y);

961

case '<':

962

switch (ReaderPeek()) {

963

case '=':

964

ReaderRead();

965

return new Token(Tokens.LessEqual, x, y);

966

case '>':

967

ReaderRead();

968

return new Token(Tokens.NotEqual, x, y);

969

case '<':

970

ReaderRead();

971

switch (ReaderPeek()) {

972

case '=':

973

ReaderRead();

974

return new Token(Tokens.ShiftLeftAssign, x, y);

975

default:

976

break;

977

}

978

return new Token(Tokens.ShiftLeft, x, y);

979

}

980

return new Token(Tokens.LessThan, x, y);

981

case '>':

982

switch (ReaderPeek()) {

983

case '=':

984

ReaderRead();

985

return new Token(Tokens.GreaterEqual, x, y);

986

case '>':

987

ReaderRead();

988

if (ReaderPeek() != -1) {

989

switch (ReaderPeek()) {

990

case '=':

991

ReaderRead();

992

return new Token(Tokens.ShiftRightAssign, x, y);

993

default:

994

break;

995

}

996

}

997

return new Token(Tokens.ShiftRight, x, y);

998

}

999

return new Token(Tokens.GreaterThan, x, y);

1000

case ',':

1001

return new Token(Tokens.Comma, x, y);

1002

case '.':

1003

// Prevent OverflowException when Peek returns -1

1004

int tmp = ReaderPeek(); int tmp2 = ReaderPeek(1);

1005

if (tmp > 0) {

1006

if (char.IsDigit((char)tmp))

1007

return ReadDigit('.', Col);

1008

else if ((char)tmp == '@') {

1009

ReaderRead();

1010

return new Token(Tokens.DotAt, x, y);

1011

} else if ((char)tmp == '.' && tmp2 > 0 && (char)tmp2 == '.') {

1012

ReaderRead(); ReaderRead();

1013

return new Token(Tokens.TripleDot, x, y);

1014

}

1015

}

1016

return new Token(Tokens.Dot, x, y);

1017

case '(':

1018

return new Token(Tokens.OpenParenthesis, x, y);

1019

case ')':

1020

return new Token(Tokens.CloseParenthesis, x, y);

1021

case '{':

1022

return new Token(Tokens.OpenCurlyBrace, x, y);

1023

case '}':

1024

return new Token(Tokens.CloseCurlyBrace, x, y);

1025

case '?':

1026

return new Token(Tokens.QuestionMark, x, y);

1027

case '!':

1028

return new Token(Tokens.ExclamationMark, x, y);

1029

}

1030

return null;

1031

}

1032

#endregion

1033

1034

#region XML Readers

1035

Token ReadXmlProcessingInstruction(int x, int y)

1036

{

1037

sb.Length = 0;

1038

int nextChar = -1;

1039

1040

while (ReaderPeek() != '?' || ReaderPeek(1) != '>') {

1041

nextChar = ReaderRead();

1042

if (nextChar == -1)

1043

break;

1044

sb.Append((char)nextChar);

1045

}

1046

1047

ReaderSkip("?>".Length);

1048

1049

return new Token(Tokens.XmlProcessingInstruction, new TextLocation(y, x), new TextLocation(Line, Col), sb.ToString(), null);

1050

}

1051

1052

Token ReadXmlCommentOrCData(int x, int y)

1053

{

1054

sb.Length = 0;

1055

int nextChar = -1;

1056

1057

if (string.CompareOrdinal(ReaderPeekString("--".Length), "--") == 0) {

1058

ReaderSkip("--".Length);

1059

while ((nextChar = ReaderRead()) != -1) {

1060

sb.Append((char)nextChar);

1061

if (string.CompareOrdinal(ReaderPeekString("-->".Length), "-->") == 0) {

1062

ReaderSkip("-->".Length);

1063

return new Token(Tokens.XmlComment, new TextLocation(y, x), new TextLocation(Line, Col), sb.ToString(), null);

1064

}

1065

}

1066

}

1067

1068

if (string.CompareOrdinal(ReaderPeekString("[CDATA[".Length), "[CDATA[") == 0) {

1069

ReaderSkip("[CDATA[".Length);

1070

while ((nextChar = ReaderRead()) != -1) {

1071

sb.Append((char)nextChar);

1072

if (string.CompareOrdinal(ReaderPeekString("]]>".Length), "]]>") == 0) {

1073

ReaderSkip("]]>".Length);

1074

return new Token(Tokens.XmlCData, new TextLocation(y, x), new TextLocation(Line, Col), sb.ToString(), null);

1075

}

1076

}

1077

}

1078

1079

return new Token(Tokens.XmlComment, new TextLocation(y, x), new TextLocation(Line, Col), sb.ToString(), null);

1080

}

1081

1082

string ReadXmlContent(char ch)

1083

{

1084

sb.Length = 0;

1085

while (true) {

1086

sb.Append(ch);

1087

int next = ReaderPeek();

1088

1089

if (next == -1 || next == '<')

1090

break;

1091

ch = (char)ReaderRead();

1092

}

1093

1094

return sb.ToString();

1095

}

1096

1097

string ReadXmlString(char terminator)

1098

{

1099

char ch = '\0';

1100

sb.Length = 0;

1101

int nextChar;

1102

while ((nextChar = ReaderRead()) != -1) {

1103

ch = (char)nextChar;

1104

if (ch == terminator) {

1105

break;

1106

} else if (ch == '\n') {

1107

errors.Error(Line, Col, String.Format("No return allowed inside String literal"));

1108

} else {

1109

sb.Append(ch);

1110

}

1111

}

1112

if (ch != terminator) {

1113

errors.Error(Line, Col, String.Format("End of File reached before String terminated "));

1114

}

1115

return sb.ToString();

1116

}

1117

1118

string ReadXmlIdent(char ch)

1119

{

1120

sb.Length = 0;

1121

sb.Append(ch);

1122

1123

int peek;

1124

1125

while ((peek = ReaderPeek()) != -1 && (peek == ':' || XmlConvert.IsNCNameChar((char)peek))) {

1126

sb.Append((char)ReaderRead());

1127

}

1128

1129

return sb.ToString();

1130

}

1131

#endregion

1132

1133

char PeekUpperChar()

1134

{

1135

return Char.ToUpper((char)ReaderPeek(), CultureInfo.InvariantCulture);

1136

}

1137

1138

/// <summary>

1139

/// Skips to the end of the current code block.

1140

/// For this, the lexer must have read the next token AFTER the token opening the

1141

/// block (so that Lexer.Token is the block-opening token, not Lexer.LookAhead).

1142

/// After the call, Lexer.LookAhead will be the block-closing token.

1143

/// </summary>

1144

public void SkipCurrentBlock(int targetToken)

1145

{

1146

int lastKind = -1;

1147

int kind = lastToken.kind;

1148

while (kind != Tokens.EOF &&

1149

!(lastKind == Tokens.End && kind == targetToken))

1150

{

1151

lastKind = kind;

1152

NextToken();

1153

kind = lastToken.kind;

1154

}

1155

}

1156

1157

public void SetInitialContext(SnippetType type)

1158

{

1159

ef.SetContext(type);

1160

}

1161

1162

public VBLexerMemento Export()

1163

{

1164

return new VBLexerMemento() {

1165

Column = Col,

1166

Line = Line,

1167

EncounteredLineContinuation = encounteredLineContinuation,

1168

ExpressionFinder = ef.Export(),

1169

InXmlMode = inXmlMode,

1170

IsAtLineBegin = isAtLineBegin,

1171

LineEnd = lineEnd,

1172

PrevTokenKind = prevToken.kind,

1173

MisreadExclamationMarkAsTypeCharacter = misreadExclamationMarkAsTypeCharacter,

1174

XmlModeInfoStack = new Stack<XmlModeInfo>(xmlModeStack.Select(i => (XmlModeInfo)i.Clone()).Reverse())

1175

};

1176

}

1177

1178

LATextReader reader;

1179

int col = 1;

1180

int line = 1;

1181

1182

protected Errors errors = new Errors();

1183

1184

protected Token lastToken = null;

1185

protected Token curToken = null;

1186

protected Token peekToken = null;

1187

1188

string[] specialCommentTags = null;

1189

protected Hashtable specialCommentHash = null;

1190

// List<TagComment> tagComments = new List<TagComment>();

1191

protected StringBuilder sb = new StringBuilder();

1192

// protected SpecialTracker specialTracker = new SpecialTracker();

1193

1194

// used for the original value of strings (with escape sequences).

1195

protected StringBuilder originalValue = new StringBuilder();

1196

1197

public bool SkipAllComments { get; set; }

1198

public bool EvaluateConditionalCompilation { get; set; }

1199

public virtual IDictionary<string, object> ConditionalCompilationSymbols {

1200

get { throw new NotSupportedException(); }

1201

}

1202

1203

protected static IEnumerable<string> GetSymbols (string symbols)

1204

{

1205

if (!string.IsNullOrEmpty(symbols)) {

1206

foreach (string symbol in symbols.Split (';', ' ', '\t')) {

1207

string s = symbol.Trim ();

1208

if (s.Length == 0)

1209

continue;

1210

yield return s;

1211

}

1212

}

1213

}

1214

1215

public void SetConditionalCompilationSymbols (string symbols)

1216

{

1217

throw new NotSupportedException ();

1218

}

1219

1220

protected int Line {

1221

get {

1222

return line;

1223

}

1224

}

1225

protected int Col {

1226

get {

1227

return col;

1228

}

1229

}

1230

1231

protected bool recordRead = false;

1232

protected StringBuilder recordedText = new StringBuilder ();

1233

1234

protected int ReaderRead()

1235

{

1236

int val = reader.Read();

1237

if (recordRead && val >= 0)

1238

recordedText.Append ((char)val);

1239

if ((val == '\r' && reader.Peek() != '\n') || val == '\n') {

1240

++line;

1241

col = 1;

1242

LineBreak();

1243

} else if (val >= 0) {

1244

col++;

1245

}

1246

return val;

1247

}

1248

1249

protected int ReaderPeek()

1250

{

1251

return reader.Peek();

1252

}

1253

1254

protected int ReaderPeek(int step)

1255

{

1256

return reader.Peek(step);

1257

}

1258

1259

protected void ReaderSkip(int steps)

1260

{

1261

for (int i = 0; i < steps; i++) {

1262

ReaderRead();

1263

}

1264

}

1265

1266

protected string ReaderPeekString(int length)

1267

{

1268

StringBuilder builder = new StringBuilder();

1269

1270

for (int i = 0; i < length; i++) {

1271

int peek = ReaderPeek(i);

1272

if (peek != -1)

1273

builder.Append((char)peek);

1274

}

1275

1276

return builder.ToString();

1277

}

1278

1279

public void SetInitialLocation(TextLocation location)

1280

{

1281

if (lastToken != null || curToken != null || peekToken != null)

1282

throw new InvalidOperationException();

1283

this.line = location.Line;

1284

this.col = location.Column;

1285

}

1286

1287

public Errors Errors {

1288

get {

1289

return errors;

1290

}

1291

}

1292

1293

/// <summary>

1294

/// Returns the comments that had been read and containing tag key words.

1295

/// </summary>

1296

// public List<TagComment> TagComments {

1297

// get {

1298

// return tagComments;

1299

// }

1300

// }

1301

1302

// public SpecialTracker SpecialTracker {

1303

// get {

1304

// return specialTracker;

1305

// }

1306

// }

1307

1308

/// <summary>

1309

/// Special comment tags are tags like TODO, HACK or UNDONE which are read by the lexer and stored in <see cref="TagComments"/>.

1310

/// </summary>

1311

public string[] SpecialCommentTags {

1312

get {

1313

return specialCommentTags;

1314

}

1315

set {

1316

specialCommentTags = value;

1317

specialCommentHash = null;

1318

if (specialCommentTags != null && specialCommentTags.Length > 0) {

1319

specialCommentHash = new Hashtable();

1320

foreach (string str in specialCommentTags) {

1321

specialCommentHash.Add(str, null);

1322

}

1323

}

1324

}

1325

}

1326

1327

/// <summary>

1328

/// The current Token. <seealso cref="ICSharpCode.NRefactory.VB.Parser.Token"/>

1329

/// </summary>

1330

public Token Token {

1331

get {

1332

// Console.WriteLine("Call to Token");

1333

return lastToken;

1334

}

1335

}

1336

1337

/// <summary>

1338

/// The next Token (The <see cref="Token"/> after <see cref="NextToken"/> call) . <seealso cref="ICSharpCode.NRefactory.VB.Parser.Token"/>

1339

/// </summary>

1340

public Token LookAhead {

1341

get {

1342

// Console.WriteLine("Call to LookAhead");

1343

return curToken;

1344

}

1345

}

1346

1347

#region System.IDisposable interface implementation

1348

public virtual void Dispose()

1349

{

1350

reader.Close();

1351

reader = null;

1352

errors = null;

1353

lastToken = curToken = peekToken = null;

1354

specialCommentHash = null;

1355

sb = originalValue = null;

1356

}

1357

#endregion

1358

1359

/// <summary>

1360

/// Must be called before a peek operation.

1361

/// </summary>

1362

public void StartPeek()

1363

{

1364

peekToken = curToken;

1365

}

1366

1367

/// <summary>

1368

/// Gives back the next token. A second call to Peek() gives the next token after the last call for Peek() and so on.

1369

/// </summary>

1370

/// <returns>An <see cref="Token"/> object.</returns>

1371

public Token Peek()

1372

{

1373

// Console.WriteLine("Call to Peek");

1374

if (peekToken.next == null) {

1375

peekToken.next = Next();

1376

}

1377

peekToken = peekToken.next;

1378

return peekToken;

1379

}

1380

1381

protected static bool IsIdentifierPart(int ch)

1382

{

1383

if (ch == 95) return true; // 95 = '_'

1384

if (ch == -1) return false;

1385

return char.IsLetterOrDigit((char)ch); // accept unicode letters

1386

}

1387

1388

protected static bool IsHex(char digit)

1389

{

1390

return Char.IsDigit(digit) || ('A' <= digit && digit <= 'F') || ('a' <= digit && digit <= 'f');

1391

}

1392

1393

protected int GetHexNumber(char digit)

1394

{

1395

if (Char.IsDigit(digit)) {

1396

return digit - '0';

1397

}

1398

if ('A' <= digit && digit <= 'F') {

1399

return digit - 'A' + 0xA;

1400

}

1401

if ('a' <= digit && digit <= 'f') {

1402

return digit - 'a' + 0xA;

1403

}

1404

errors.Error(line, col, String.Format("Invalid hex number '" + digit + "'"));

1405

return 0;

1406

}

1407

protected TextLocation lastLineEnd = new TextLocation(1, 1);

1408

protected TextLocation curLineEnd = new TextLocation(1, 1);

1409

protected void LineBreak ()

1410

{

1411

lastLineEnd = curLineEnd;

1412

curLineEnd = new TextLocation (line, col - 1);

1413

}

1414

protected bool HandleLineEnd(char ch)

1415

{

1416

// Handle MS-DOS or MacOS line ends.

1417

if (ch == '\r') {

1418

if (reader.Peek() == '\n') { // MS-DOS line end '\r\n'

1419

ReaderRead(); // LineBreak (); called by ReaderRead ();

1420

return true;

1421

} else { // assume MacOS line end which is '\r'

1422

LineBreak ();

1423

return true;

1424

}

1425

}

1426

if (ch == '\n') {

1427

LineBreak ();

1428

return true;

1429

}

1430

return false;

1431

}

1432

1433

protected void SkipToEndOfLine()

1434

{

1435

int nextChar;

1436

while ((nextChar = reader.Read()) != -1) {

1437

if (nextChar == '\r') {

1438

if (reader.Peek() == '\n')

1439

reader.Read();

1440

nextChar = '\n';

1441

}

1442

if (nextChar == '\n') {

1443

++line;

1444

col = 1;

1445

break;

1446

}

1447

}

1448

}

1449

1450

protected string ReadToEndOfLine()

1451

{

1452

sb.Length = 0;

1453

int nextChar;

1454

while ((nextChar = reader.Read()) != -1) {

1455

char ch = (char)nextChar;

1456

1457

if (nextChar == '\r') {

1458

if (reader.Peek() == '\n')

1459

reader.Read();

1460

nextChar = '\n';

1461

}

1462

// Return read string, if EOL is reached

1463

if (nextChar == '\n') {

1464

++line;

1465

col = 1;

1466

return sb.ToString();

1467

}

1468

1469

sb.Append(ch);

1470

}

1471

1472

// Got EOF before EOL

1473

string retStr = sb.ToString();

1474

col += retStr.Length;

1475

return retStr;

1476

}

1477

1478

public event EventHandler<SavepointEventArgs> SavepointReached;

1479

1480

protected virtual void OnSavepointReached(SavepointEventArgs e)

1481

{

1482

if (SavepointReached != null) {

1483

SavepointReached(this, e);

1484

}

1485

}

1486

}

1487

}

Older »