4
** The author disclaims copyright to this source code. In place of
5
** a legal notice, here is a blessing:
7
** May you do good and not evil.
8
** May you find forgiveness for yourself and forgive others.
9
** May you share freely, never taking more than you give.
11
*************************************************************************
12
** An tokenizer for SQL
14
** This file contains C code that splits an SQL input string up into
15
** individual tokens and sends those tokens one-by-one over to the
16
** parser for analysis.
18
** $Id: tokenize.c,v 1.1 2004/07/07 21:25:53 pahlibar Exp $
20
#include "sqliteInt.h"
26
** All the keywords of the SQL language are stored as in a hash
27
** table composed of instances of the following structure.
29
typedef struct Keyword Keyword;
31
char *zName; /* The keyword name */
32
u8 tokenType; /* Token value for this keyword */
33
u8 len; /* Length of this keyword */
34
u8 iNext; /* Index in aKeywordTable[] of next with same hash */
38
** These are the keywords
40
static Keyword aKeywordTable[] = {
41
{ "ABORT", TK_ABORT, },
42
{ "AFTER", TK_AFTER, },
47
{ "ATTACH", TK_ATTACH, },
48
{ "BEFORE", TK_BEFORE, },
49
{ "BEGIN", TK_BEGIN, },
50
{ "BETWEEN", TK_BETWEEN, },
52
{ "CASCADE", TK_CASCADE, },
54
{ "CHECK", TK_CHECK, },
55
{ "CLUSTER", TK_CLUSTER, },
56
{ "COLLATE", TK_COLLATE, },
57
{ "COMMIT", TK_COMMIT, },
58
{ "CONFLICT", TK_CONFLICT, },
59
{ "CONSTRAINT", TK_CONSTRAINT, },
61
{ "CREATE", TK_CREATE, },
62
{ "CROSS", TK_JOIN_KW, },
63
{ "DATABASE", TK_DATABASE, },
64
{ "DEFAULT", TK_DEFAULT, },
65
{ "DEFERRED", TK_DEFERRED, },
66
{ "DEFERRABLE", TK_DEFERRABLE, },
67
{ "DELETE", TK_DELETE, },
68
{ "DELIMITERS", TK_DELIMITERS, },
70
{ "DETACH", TK_DETACH, },
71
{ "DISTINCT", TK_DISTINCT, },
76
{ "EXCEPT", TK_EXCEPT, },
77
{ "EXPLAIN", TK_EXPLAIN, },
80
{ "FOREIGN", TK_FOREIGN, },
82
{ "FULL", TK_JOIN_KW, },
84
{ "GROUP", TK_GROUP, },
85
{ "HAVING", TK_HAVING, },
86
{ "IGNORE", TK_IGNORE, },
87
{ "IMMEDIATE", TK_IMMEDIATE, },
89
{ "INDEX", TK_INDEX, },
90
{ "INITIALLY", TK_INITIALLY, },
91
{ "INNER", TK_JOIN_KW, },
92
{ "INSERT", TK_INSERT, },
93
{ "INSTEAD", TK_INSTEAD, },
94
{ "INTERSECT", TK_INTERSECT, },
97
{ "ISNULL", TK_ISNULL, },
100
{ "LEFT", TK_JOIN_KW, },
101
{ "LIKE", TK_LIKE, },
102
{ "LIMIT", TK_LIMIT, },
103
{ "MATCH", TK_MATCH, },
104
{ "NATURAL", TK_JOIN_KW, },
106
{ "NOTNULL", TK_NOTNULL, },
107
{ "NULL", TK_NULL, },
109
{ "OFFSET", TK_OFFSET, },
112
{ "ORDER", TK_ORDER, },
113
{ "OUTER", TK_JOIN_KW, },
114
{ "PRAGMA", TK_PRAGMA, },
115
{ "PRIMARY", TK_PRIMARY, },
116
{ "RAISE", TK_RAISE, },
117
{ "REFERENCES", TK_REFERENCES, },
118
{ "REPLACE", TK_REPLACE, },
119
{ "RESTRICT", TK_RESTRICT, },
120
{ "RIGHT", TK_JOIN_KW, },
121
{ "ROLLBACK", TK_ROLLBACK, },
123
{ "SELECT", TK_SELECT, },
125
{ "STATEMENT", TK_STATEMENT, },
126
{ "TABLE", TK_TABLE, },
127
{ "TEMP", TK_TEMP, },
128
{ "TEMPORARY", TK_TEMP, },
129
{ "THEN", TK_THEN, },
130
{ "TRANSACTION", TK_TRANSACTION, },
131
{ "TRIGGER", TK_TRIGGER, },
132
{ "UNION", TK_UNION, },
133
{ "UNIQUE", TK_UNIQUE, },
134
{ "UPDATE", TK_UPDATE, },
135
{ "USING", TK_USING, },
136
{ "VACUUM", TK_VACUUM, },
137
{ "VALUES", TK_VALUES, },
138
{ "VIEW", TK_VIEW, },
139
{ "WHEN", TK_WHEN, },
140
{ "WHERE", TK_WHERE, },
144
** This is the hash table
146
#define KEY_HASH_SIZE 101
147
static u8 aiHashTable[KEY_HASH_SIZE];
151
** This function looks up an identifier to determine if it is a
152
** keyword. If it is a keyword, the token code of that keyword is
153
** returned. If the input is not a keyword, TK_ID is returned.
155
int sqliteKeywordCode(const char *z, int n){
158
static char needInit = 1;
160
/* Initialize the keyword hash table */
161
sqliteOsEnterMutex();
164
nk = sizeof(aKeywordTable)/sizeof(aKeywordTable[0]);
166
aKeywordTable[i].len = strlen(aKeywordTable[i].zName);
167
h = sqliteHashNoCase(aKeywordTable[i].zName, aKeywordTable[i].len);
169
aKeywordTable[i].iNext = aiHashTable[h];
170
aiHashTable[h] = i+1;
174
sqliteOsLeaveMutex();
176
h = sqliteHashNoCase(z, n) % KEY_HASH_SIZE;
177
for(i=aiHashTable[h]; i; i=p->iNext){
178
p = &aKeywordTable[i-1];
179
if( p->len==n && sqliteStrNICmp(p->zName, z, n)==0 ){
188
** If X is a character that can be used in an identifier and
189
** X&0x80==0 then isIdChar[X] will be 1. If X&0x80==0x80 then
190
** X is always an identifier character. (Hence all UTF-8
191
** characters can be part of an identifier). isIdChar[X] will
192
** be 0 for every character in the lower 128 ASCII characters
193
** that cannot be used as part of an identifier.
195
** In this implementation, an identifier can be a string of
196
** alphabetic characters, digits, and "_" plus any character
197
** with the high-order bit set. The latter rule means that
198
** any sequence of UTF-8 characters or characters taken from
199
** an extended ISO8859 character set can form an identifier.
201
static const char isIdChar[] = {
202
/* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xA xB xC xD xE xF */
203
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x */
204
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 1x */
205
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 2x */
206
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 3x */
207
0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 4x */
208
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, /* 5x */
209
0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 6x */
210
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 7x */
215
** Return the length of the token that begins at z[0].
216
** Store the token type in *tokenType before returning.
218
static int sqliteGetToken(const unsigned char *z, int *tokenType){
221
case ' ': case '\t': case '\n': case '\f': case '\r': {
222
for(i=1; isspace(z[i]); i++){}
223
*tokenType = TK_SPACE;
228
for(i=2; z[i] && z[i]!='\n'; i++){}
229
*tokenType = TK_COMMENT;
232
*tokenType = TK_MINUS;
244
*tokenType = TK_SEMI;
248
*tokenType = TK_PLUS;
252
*tokenType = TK_STAR;
256
if( z[1]!='*' || z[2]==0 ){
257
*tokenType = TK_SLASH;
260
for(i=3; z[i] && (z[i]!='/' || z[i-1]!='*'); i++){}
262
*tokenType = TK_COMMENT;
271
return 1 + (z[1]=='=');
277
}else if( z[1]=='>' ){
280
}else if( z[1]=='<' ){
281
*tokenType = TK_LSHIFT;
292
}else if( z[1]=='>' ){
293
*tokenType = TK_RSHIFT;
302
*tokenType = TK_ILLEGAL;
311
*tokenType = TK_BITOR;
314
*tokenType = TK_CONCAT;
319
*tokenType = TK_COMMA;
323
*tokenType = TK_BITAND;
327
*tokenType = TK_BITNOT;
330
case '\'': case '"': {
342
*tokenType = TK_STRING;
349
case '0': case '1': case '2': case '3': case '4':
350
case '5': case '6': case '7': case '8': case '9': {
351
*tokenType = TK_INTEGER;
352
for(i=1; isdigit(z[i]); i++){}
353
if( z[i]=='.' && isdigit(z[i+1]) ){
355
while( isdigit(z[i]) ){ i++; }
356
*tokenType = TK_FLOAT;
358
if( (z[i]=='e' || z[i]=='E') &&
360
|| ((z[i+1]=='+' || z[i+1]=='-') && isdigit(z[i+2]))
364
while( isdigit(z[i]) ){ i++; }
365
*tokenType = TK_FLOAT;
370
for(i=1; z[i] && z[i-1]!=']'; i++){}
375
*tokenType = TK_VARIABLE;
379
if( (*z&0x80)==0 && !isIdChar[*z] ){
382
for(i=1; (z[i]&0x80)!=0 || isIdChar[z[i]]; i++){}
383
*tokenType = sqliteKeywordCode((char*)z, i);
387
*tokenType = TK_ILLEGAL;
392
** Run the parser on the given SQL string. The parser structure is
393
** passed in. An SQLITE_ status code is returned. If an error occurs
394
** and pzErrMsg!=NULL then an error message might be written into
395
** memory obtained from malloc() and *pzErrMsg made to point to that
396
** error message. Or maybe not.
398
int sqliteRunParser(Parse *pParse, const char *zSql, char **pzErrMsg){
403
int lastTokenParsed = -1;
404
sqlite *db = pParse->db;
405
extern void *sqliteParserAlloc(void*(*)(int));
406
extern void sqliteParserFree(void*, void(*)(void*));
407
extern int sqliteParser(void*, int, Token, Parse*);
409
db->flags &= ~SQLITE_Interrupt;
410
pParse->rc = SQLITE_OK;
412
pEngine = sqliteParserAlloc((void*(*)(int))malloc);
414
sqliteSetString(pzErrMsg, "out of memory", (char*)0);
417
pParse->sLastToken.dyn = 0;
418
pParse->zTail = zSql;
419
while( sqlite_malloc_failed==0 && zSql[i]!=0 ){
421
pParse->sLastToken.z = &zSql[i];
422
assert( pParse->sLastToken.dyn==0 );
423
pParse->sLastToken.n = sqliteGetToken((unsigned char*)&zSql[i], &tokenType);
424
i += pParse->sLastToken.n;
428
if( (db->flags & SQLITE_Interrupt)!=0 ){
429
pParse->rc = SQLITE_INTERRUPT;
430
sqliteSetString(pzErrMsg, "interrupt", (char*)0);
436
sqliteSetNString(pzErrMsg, "unrecognized token: \"", -1,
437
pParse->sLastToken.z, pParse->sLastToken.n, "\"", 1, 0);
442
pParse->zTail = &zSql[i];
443
/* Fall thru into the default case */
446
sqliteParser(pEngine, tokenType, pParse->sLastToken, pParse);
447
lastTokenParsed = tokenType;
448
if( pParse->rc!=SQLITE_OK ){
456
if( zSql[i]==0 && nErr==0 && pParse->rc==SQLITE_OK ){
457
if( lastTokenParsed!=TK_SEMI ){
458
sqliteParser(pEngine, TK_SEMI, pParse->sLastToken, pParse);
459
pParse->zTail = &zSql[i];
461
sqliteParser(pEngine, 0, pParse->sLastToken, pParse);
463
sqliteParserFree(pEngine, free);
464
if( pParse->rc!=SQLITE_OK && pParse->rc!=SQLITE_DONE && pParse->zErrMsg==0 ){
465
sqliteSetString(&pParse->zErrMsg, sqlite_error_string(pParse->rc),
468
if( pParse->zErrMsg ){
469
if( pzErrMsg && *pzErrMsg==0 ){
470
*pzErrMsg = pParse->zErrMsg;
472
sqliteFree(pParse->zErrMsg);
477
if( pParse->pVdbe && pParse->nErr>0 ){
478
sqliteVdbeDelete(pParse->pVdbe);
481
if( pParse->pNewTable ){
482
sqliteDeleteTable(pParse->db, pParse->pNewTable);
483
pParse->pNewTable = 0;
485
if( pParse->pNewTrigger ){
486
sqliteDeleteTrigger(pParse->pNewTrigger);
487
pParse->pNewTrigger = 0;
489
if( nErr>0 && (pParse->rc==SQLITE_OK || pParse->rc==SQLITE_DONE) ){
490
pParse->rc = SQLITE_ERROR;
496
** Token types used by the sqlite_complete() routine. See the header
497
** comments on that procedure for additional information.
509
** Return TRUE if the given SQL string ends in a semicolon.
511
** Special handling is require for CREATE TRIGGER statements.
512
** Whenever the CREATE TRIGGER keywords are seen, the statement
513
** must end with ";END;".
515
** This implementation uses a state machine with 7 states:
517
** (0) START At the beginning or end of an SQL statement. This routine
518
** returns 1 if it ends in the START state and 0 if it ends
519
** in any other state.
521
** (1) EXPLAIN The keyword EXPLAIN has been seen at the beginning of
524
** (2) CREATE The keyword CREATE has been seen at the beginning of a
525
** statement, possibly preceeded by EXPLAIN and/or followed by
528
** (3) NORMAL We are in the middle of statement which ends with a single
531
** (4) TRIGGER We are in the middle of a trigger definition that must be
532
** ended by a semicolon, the keyword END, and another semicolon.
534
** (5) SEMI We've seen the first semicolon in the ";END;" that occurs at
535
** the end of a trigger definition.
537
** (6) END We've seen the ";END" of the ";END;" that occurs at the end
538
** of a trigger difinition.
540
** Transitions between states above are determined by tokens extracted
541
** from the input. The following tokens are significant:
543
** (0) tkEXPLAIN The "explain" keyword.
544
** (1) tkCREATE The "create" keyword.
545
** (2) tkTEMP The "temp" or "temporary" keyword.
546
** (3) tkTRIGGER The "trigger" keyword.
547
** (4) tkEND The "end" keyword.
548
** (5) tkSEMI A semicolon.
549
** (6) tkWS Whitespace
550
** (7) tkOTHER Any other SQL token.
552
** Whitespace never causes a state transition and is always ignored.
554
int sqlite_complete(const char *zSql){
555
u8 state = 0; /* Current state, using numbers defined in header comment */
556
u8 token; /* Value of the next token */
558
/* The following matrix defines the transition from one state to another
559
** according to what token is seen. trans[state][token] returns the
562
static const u8 trans[7][8] = {
564
/* State: ** EXPLAIN CREATE TEMP TRIGGER END SEMI WS OTHER */
565
/* 0 START: */ { 1, 2, 3, 3, 3, 0, 0, 3, },
566
/* 1 EXPLAIN: */ { 3, 2, 3, 3, 3, 0, 1, 3, },
567
/* 2 CREATE: */ { 3, 3, 2, 4, 3, 0, 2, 3, },
568
/* 3 NORMAL: */ { 3, 3, 3, 3, 3, 0, 3, 3, },
569
/* 4 TRIGGER: */ { 4, 4, 4, 4, 4, 5, 4, 4, },
570
/* 5 SEMI: */ { 4, 4, 4, 4, 6, 5, 5, 4, },
571
/* 6 END: */ { 4, 4, 4, 4, 4, 0, 6, 4, },
576
case ';': { /* A semicolon */
584
case '\f': { /* White space is ignored */
588
case '/': { /* C-style comments */
594
while( zSql[0] && (zSql[0]!='*' || zSql[1]!='/') ){ zSql++; }
595
if( zSql[0]==0 ) return 0;
600
case '-': { /* SQL-style comments from "--" to end of line */
605
while( *zSql && *zSql!='\n' ){ zSql++; }
606
if( *zSql==0 ) return state==0;
610
case '[': { /* Microsoft-style identifiers in [...] */
612
while( *zSql && *zSql!=']' ){ zSql++; }
613
if( *zSql==0 ) return 0;
617
case '"': /* single- and double-quoted strings */
621
while( *zSql && *zSql!=c ){ zSql++; }
622
if( *zSql==0 ) return 0;
627
if( isIdChar[(u8)*zSql] ){
628
/* Keywords and unquoted identifiers */
630
for(nId=1; isIdChar[(u8)zSql[nId]]; nId++){}
632
case 'c': case 'C': {
633
if( nId==6 && sqliteStrNICmp(zSql, "create", 6)==0 ){
640
case 't': case 'T': {
641
if( nId==7 && sqliteStrNICmp(zSql, "trigger", 7)==0 ){
643
}else if( nId==4 && sqliteStrNICmp(zSql, "temp", 4)==0 ){
645
}else if( nId==9 && sqliteStrNICmp(zSql, "temporary", 9)==0 ){
652
case 'e': case 'E': {
653
if( nId==3 && sqliteStrNICmp(zSql, "end", 3)==0 ){
655
}else if( nId==7 && sqliteStrNICmp(zSql, "explain", 7)==0 ){
669
/* Operators and special symbols */
675
state = trans[state][token];
added
removed
digikam/sqlite/tokenize.c
