~jan-kneschke/mysql-proxy/packet-tracking-assertions
» Revision
1099
(compared to revision 1130)
: lib/sql-tokenizer.l
« back to all changes in this revision
Viewing changes to lib/sql-tokenizer.l
- browse files at revision 1099
- compare with another revision
- download diff
- download tarball
- view history from revision 1099
- Committer: jan at mysql
- Date: 2010-05-27 14:19:43 UTC
- Revision ID: jan@mysql.com-20100527141943-inwv2cohofexa2pb
added entry about #49716 - literals starting with digits
- files removed:
- tests/suite/base/r/bug_55459.result
- files modified:
- CMakeLists.txt
added
removed
lib/sql-tokenizer.l
41
41
42
42
#define YY_DECL int sql_tokenizer_internal(GPtrArray *tokens)
43
43
44
#define GE_STR_LITERAL_WITH_LEN(str) str, sizeof(str) - 1
45
46
static void sql_token_append(GPtrArray *tokens, sql_token_id token_id, const gchar *text) G_GNUC_DEPRECATED;
47
static void sql_token_append_len(GPtrArray *tokens, sql_token_id token_id, const gchar *text, gsize text_len);
48
static void sql_token_append_last_token_len(GPtrArray *tokens, sql_token_id token_id, const gchar *text, size_t text_len);
49
static void sql_token_append_last_token(GPtrArray *tokens, sql_token_id token_id, const gchar *text) G_GNUC_DEPRECATED;
50
sql_token_id sql_token_get_id_len(const gchar *name, gsize name_len);
44
static void sql_token_append(GPtrArray *tokens, sql_token_id token_id, gchar *text);
45
static void sql_token_append_last_token(GPtrArray *tokens, sql_token_id token_id, gchar *text);
51
46
sql_token_id sql_token_get_id(const gchar *name);
52
47
53
48
#include "sql-tokenizer-keywords.h" /* generated, brings in sql_keywords */
66
61
%%
67
62
68
63
/** comments */
69
"--"\r?\n comment_token_id = TK_COMMENT; sql_token_append_len(tokens, comment_token_id, GE_STR_LITERAL_WITH_LEN(""));
70
"/*" comment_token_id = TK_COMMENT; sql_token_append_len(tokens, comment_token_id, GE_STR_LITERAL_WITH_LEN("")); BEGIN(COMMENT);
71
"/*!" comment_token_id = TK_COMMENT_MYSQL; sql_token_append_len(tokens, comment_token_id, GE_STR_LITERAL_WITH_LEN("")); BEGIN(COMMENT);
72
"--"[[:blank:]] comment_token_id = TK_COMMENT; sql_token_append_len(tokens, comment_token_id, GE_STR_LITERAL_WITH_LEN("")); BEGIN(LINECOMMENT);
73
<COMMENT>[^*]* sql_token_append_last_token_len(tokens, comment_token_id, yytext, yyleng);
74
<COMMENT>"*"+[^*/]* sql_token_append_last_token_len(tokens, comment_token_id, yytext, yyleng);
64
"--"\r?\n comment_token_id = TK_COMMENT; sql_token_append(tokens, comment_token_id, "");
65
"/*" comment_token_id = TK_COMMENT; sql_token_append(tokens, comment_token_id, ""); BEGIN(COMMENT);
66
"/*!" comment_token_id = TK_COMMENT_MYSQL; sql_token_append(tokens, comment_token_id, ""); BEGIN(COMMENT);
67
"--"[[:blank:]] comment_token_id = TK_COMMENT; sql_token_append(tokens, comment_token_id, ""); BEGIN(LINECOMMENT);
68
<COMMENT>[^*]* sql_token_append_last_token(tokens, comment_token_id, yytext);
69
<COMMENT>"*"+[^*/]* sql_token_append_last_token(tokens, comment_token_id, yytext);
75
70
<COMMENT>"*"+"/" BEGIN(INITIAL);
76
71
<COMMENT><<EOF>> BEGIN(INITIAL);
77
<LINECOMMENT>[^\n]* sql_token_append_last_token_len(tokens, comment_token_id, yytext, yyleng);
72
<LINECOMMENT>[^\n]* sql_token_append_last_token(tokens, comment_token_id, yytext);
78
73
<LINECOMMENT>\r?\n BEGIN(INITIAL);
79
74
<LINECOMMENT><<EOF>> BEGIN(INITIAL);
80
75
86
81
case '"': quote_token_id = TK_STRING; break;
87
82
case '`': quote_token_id = TK_LITERAL; break;
88
83
}
89
sql_token_append_len(tokens, quote_token_id, GE_STR_LITERAL_WITH_LEN("")); }
90
<QUOTED>[^"'`\\]* sql_token_append_last_token_len(tokens, quote_token_id, yytext, yyleng); /** all non quote or esc chars are passed through */
91
<QUOTED>"\\". sql_token_append_last_token_len(tokens, quote_token_id, yytext, yyleng); /** add escaping */
84
sql_token_append(tokens, quote_token_id, ""); }
85
<QUOTED>[^"'`\\]* sql_token_append_last_token(tokens, quote_token_id, yytext); /** all non quote or esc chars are passed through */
86
<QUOTED>"\\". sql_token_append_last_token(tokens, quote_token_id, yytext); /** add escaping */
92
87
<QUOTED>["'`]{2} { if (yytext[0] == yytext[1] && yytext[1] == quote_char) {
93
sql_token_append_last_token_len(tokens, quote_token_id, yytext + 1, yyleng - 1); /** doubling quotes */
88
sql_token_append_last_token(tokens, quote_token_id, yytext + 1); /** doubling quotes */
94
89
} else {
95
/** pick the first char and put the second back to parsing */
90
/** the pick the first char and put the second back to parsing */
96
91
yyless(1);
97
sql_token_append_last_token_len(tokens, quote_token_id, yytext, yyleng);
98
}
99
}
100
<QUOTED>["'`] if (*yytext == quote_char) { BEGIN(INITIAL); } else { sql_token_append_last_token_len(tokens, quote_token_id, yytext, yyleng); }
92
sql_token_append_last_token(tokens, quote_token_id, yytext);
93
}
94
}
95
<QUOTED>["'`] if (*yytext == quote_char) { BEGIN(INITIAL); } else { sql_token_append_last_token(tokens, quote_token_id, yytext); }
101
96
<QUOTED><<EOF>> BEGIN(INITIAL);
102
97
103
98
/** strings, quoting, literals */
120
115
* 1e+1e is a float ("1e+1") and a literal ("e")
121
116
* compare this to 1.1e which is INVALID (a broken scientific notation)
122
117
*/
123
([[:digit:]]*".")?[[:digit:]]+[eE][-+]?[[:digit:]]+ sql_token_append_len(tokens, TK_FLOAT, yytext, yyleng);
118
([[:digit:]]*".")?[[:digit:]]+[eE][-+]?[[:digit:]]+ sql_token_append(tokens, TK_FLOAT, yytext);
124
119
/* literals
125
120
* - be greedy and capture specifiers made up of up to 3 literals: lit.lit.lit
126
121
* - if it has a dot, split it into 3 tokens: lit dot lit
134
129
*/
135
130
[[:digit:]]*[[:alpha:]_@][[:alnum:]_@]*("."[[:digit:]]*[[:alpha:]_@][[:alnum:]_@]*){0,2} {
136
131
char *cur, *tk_start = yytext;
137
gsize tk_len;
138
139
132
for (cur = yytext; cur < yytext + yyleng; cur++) {
140
133
if (*cur == '.') {
141
tk_len = cur - tk_start;
142
143
sql_token_append_len(tokens, sql_token_get_id_len(tk_start, tk_len), tk_start, tk_len);
144
sql_token_append_len(tokens, TK_DOT, GE_STR_LITERAL_WITH_LEN("."));
134
*cur = '\0';
135
sql_token_append(tokens, sql_token_get_id(tk_start), tk_start);
136
sql_token_append(tokens, TK_DOT, ".");
145
137
tk_start = cur + 1;
146
138
}
147
139
}
148
/* copy the rest */
149
tk_len = yytext + yyleng - tk_start;
150
sql_token_append_len(tokens, sql_token_get_id_len(tk_start, tk_len), tk_start, tk_len);
140
sql_token_append(tokens, sql_token_get_id(tk_start), tk_start);
151
141
}
152
142
/* literals followed by a ( are function names */
153
143
[[:digit:]]*[[:alpha:]_@][[:alnum:]_@]*("."[[:digit:]]*[[:alpha:]_@][[:alnum:]_@]*){0,2}\( {
154
144
char *cur, *tk_start = yytext;
155
gsize tk_len;
156
145
157
146
yyless(yyleng - 1); /* on step back to track the parantheses correctly */
158
147
159
/* split the matched string at the dots */
160
148
for (cur = yytext; cur < yytext + yyleng; cur++) {
161
149
if (*cur == '.') {
162
tk_len = cur - tk_start;
163
164
sql_token_append_len(tokens, sql_token_get_id_len(tk_start, tk_len), tk_start, tk_len);
165
sql_token_append_len(tokens, TK_DOT, GE_STR_LITERAL_WITH_LEN("."));
150
*cur = '\0';
151
sql_token_append(tokens, sql_token_get_id(tk_start), tk_start);
152
sql_token_append(tokens, TK_DOT, ".");
166
153
tk_start = cur + 1;
167
154
}
168
155
}
169
tk_len = yytext + yyleng - tk_start;
170
sql_token_append_len(tokens, TK_FUNCTION, tk_start, tk_len);
156
sql_token_append(tokens, TK_FUNCTION, tk_start);
171
157
}
172
158
173
[[:digit:]]+ sql_token_append_len(tokens, TK_INTEGER, yytext, yyleng);
174
[[:digit:]]*"."[[:digit:]]+ sql_token_append_len(tokens, TK_FLOAT, yytext, yyleng);
175
"," sql_token_append_len(tokens, TK_COMMA, yytext, yyleng);
176
"." sql_token_append_len(tokens, TK_DOT, yytext, yyleng);
177
178
"<" sql_token_append_len(tokens, TK_LT, yytext, yyleng);
179
">" sql_token_append_len(tokens, TK_GT, yytext, yyleng);
180
"<=" sql_token_append_len(tokens, TK_LE, yytext, yyleng);
181
">=" sql_token_append_len(tokens, TK_GE, yytext, yyleng);
182
"=" sql_token_append_len(tokens, TK_EQ, yytext, yyleng);
183
"<>" sql_token_append_len(tokens, TK_NE, yytext, yyleng);
184
"!=" sql_token_append_len(tokens, TK_NE, yytext, yyleng);
185
186
"(" sql_token_append_len(tokens, TK_OBRACE, yytext, yyleng);
187
")" sql_token_append_len(tokens, TK_CBRACE, yytext, yyleng);
188
";" sql_token_append_len(tokens, TK_SEMICOLON, yytext, yyleng);
189
":=" sql_token_append_len(tokens, TK_ASSIGN, yytext, yyleng);
190
191
"*" sql_token_append_len(tokens, TK_STAR, yytext, yyleng);
192
"+" sql_token_append_len(tokens, TK_PLUS, yytext, yyleng);
193
"/" sql_token_append_len(tokens, TK_DIV, yytext, yyleng);
194
"-" sql_token_append_len(tokens, TK_MINUS, yytext, yyleng);
195
196
"&" sql_token_append_len(tokens, TK_BITWISE_AND, yytext, yyleng);
197
"&&" sql_token_append_len(tokens, TK_LOGICAL_AND, yytext, yyleng);
198
"|" sql_token_append_len(tokens, TK_BITWISE_OR, yytext, yyleng);
199
"||" sql_token_append_len(tokens, TK_LOGICAL_OR, yytext, yyleng);
200
201
"^" sql_token_append_len(tokens, TK_BITWISE_XOR, yytext, yyleng);
159
[[:digit:]]+ sql_token_append(tokens, TK_INTEGER, yytext);
160
[[:digit:]]*"."[[:digit:]]+ sql_token_append(tokens, TK_FLOAT, yytext);
161
"," sql_token_append(tokens, TK_COMMA, yytext);
162
"." sql_token_append(tokens, TK_DOT, yytext);
163
164
"<" sql_token_append(tokens, TK_LT, yytext);
165
">" sql_token_append(tokens, TK_GT, yytext);
166
"<=" sql_token_append(tokens, TK_LE, yytext);
167
">=" sql_token_append(tokens, TK_GE, yytext);
168
"=" sql_token_append(tokens, TK_EQ, yytext);
169
"<>" sql_token_append(tokens, TK_NE, yytext);
170
"!=" sql_token_append(tokens, TK_NE, yytext);
171
172
"(" sql_token_append(tokens, TK_OBRACE, yytext);
173
")" sql_token_append(tokens, TK_CBRACE, yytext);
174
";" sql_token_append(tokens, TK_SEMICOLON, yytext);
175
":=" sql_token_append(tokens, TK_ASSIGN, yytext);
176
177
"*" sql_token_append(tokens, TK_STAR, yytext);
178
"+" sql_token_append(tokens, TK_PLUS, yytext);
179
"/" sql_token_append(tokens, TK_DIV, yytext);
180
"-" sql_token_append(tokens, TK_MINUS, yytext);
181
182
"&" sql_token_append(tokens, TK_BITWISE_AND, yytext);
183
"&&" sql_token_append(tokens, TK_LOGICAL_AND, yytext);
184
"|" sql_token_append(tokens, TK_BITWISE_OR, yytext);
185
"||" sql_token_append(tokens, TK_LOGICAL_OR, yytext);
186
187
"^" sql_token_append(tokens, TK_BITWISE_XOR, yytext);
202
188
203
189
/** the default rule */
204
. sql_token_append_len(tokens, TK_UNKNOWN, yytext, yyleng);
190
. sql_token_append(tokens, TK_UNKNOWN, yytext);
205
191
206
192
%%
207
193
sql_token *sql_token_new(void) {
224
210
g_free(token);
225
211
}
226
212
227
228
213
/**
229
214
* append a token to the token-list
230
215
*/
231
static void sql_token_append_len(GPtrArray *tokens, sql_token_id token_id, const gchar *text, gsize text_len) {
216
static void sql_token_append(GPtrArray *tokens, sql_token_id token_id, gchar *text) {
232
217
sql_token *token;
233
218
234
219
token = sql_token_new();
235
220
token->token_id = token_id;
236
g_string_assign_len(token->text, text, text_len);
237
221
g_string_assign(token->text, text);
238
222
g_ptr_array_add(tokens, token);
239
223
}
240
224
241
static void sql_token_append(GPtrArray *tokens, sql_token_id token_id, const gchar *text) {
242
sql_token_append_len(tokens, token_id, text, strlen(text));
243
}
244
245
225
/**
246
226
* append text to the last token in the token-list
247
227
*/
248
static void sql_token_append_last_token_len(GPtrArray *tokens, sql_token_id token_id, const gchar *text, size_t text_len) {
228
static void sql_token_append_last_token(GPtrArray *tokens, sql_token_id token_id, gchar *text) {
249
229
sql_token *token;
250
230
251
231
g_assert(tokens->len > 0);
254
234
g_assert(token);
255
235
g_assert(token->token_id == token_id);
256
236
257
g_string_append_len(token->text, text, text_len);
258
}
259
260
static void sql_token_append_last_token(GPtrArray *tokens, sql_token_id token_id, const gchar *text) {
261
sql_token_append_last_token_len(tokens, token_id, text, strlen(text));
262
}
263
264
typedef struct {
265
const char *name;
266
size_t name_len;
267
} sql_token_cmp_data;
237
g_string_append(token->text, text);
238
}
268
239
269
240
static int sql_token_cmp(const void *_a, const void *_b) {
270
241
int i = *(int *)_b;
271
const sql_token_cmp_data *name = _a;
272
const char *keyword;
273
size_t keyword_len;
242
const char *name = _a;
243
const char *keyword;
274
244
275
keyword = sql_token_get_name(i, &keyword_len);
245
keyword = sql_token_get_name(i) + sizeof("TK_SQL_") - 1;
276
246
g_assert(keyword); /* if this isn't true, we have a internal problem */
277
278
keyword += sizeof("TK_SQL_") - 1;
279
keyword_len -= sizeof("TK_SQL_") - 1;
280
281
for (i = 0; i < keyword_len && i < name->name_len; i++) {
282
int c_diff = g_ascii_tolower(name->name[i]) - g_ascii_tolower(keyword[i]);
283
284
if (0 != c_diff) return c_diff;
285
}
286
287
/* we are still here, up to now they are the same */
288
return name->name_len - keyword_len;
247
248
return g_ascii_strcasecmp(name, keyword);
289
249
}
290
250
291
251
/**
292
252
* get the token_id for a literal
293
253
*/
294
sql_token_id sql_token_get_id_len(const gchar *name, gsize name_len) {
254
sql_token_id sql_token_get_id(const gchar *name) {
295
255
gint *i;
296
sql_token_cmp_data data;
297
256
298
257
/* do a binary search on the sql_keywords */
299
data.name = name;
300
data.name_len = name_len;
301
258
302
i = bsearch(&data,
259
i = bsearch(name,
303
260
sql_keywords_get(),
304
261
sql_keywords_get_count(),
305
262
sizeof(int),
309
266
}
310
267
311
268
/**
312
* get the token_id for a literal
313
*/
314
sql_token_id sql_token_get_id(const gchar *name) {
315
return sql_token_get_id_len(name, strlen(name));
316
}
317
318
/**
319
269
* scan a string into SQL tokens
320
270
*/
321
271
int sql_tokenizer(GPtrArray *tokens, const gchar *str, gsize len) {
341
291
for (i = 0; i < tokens->len; i++) {
342
292
sql_token *token = tokens->pdata[i];
343
293
344
if (token) sql_token_free(token);
294
sql_token_free(token);
345
295
}
346
296
g_ptr_array_free(tokens, TRUE);
347
297
}
Loggerhead is a web-based interface for Breezy
Version: 2.0.1