1 /*
2 ** 2001 September 15
3 **
4 ** The author disclaims copyright to this source code. In place of
5 ** a legal notice, here is a blessing:
6 **
7 ** May you do good and not evil.
8 ** May you find forgiveness for yourself and forgive others.
9 ** May you share freely, never taking more than you give.
10 **
11 *************************************************************************
12 ** An tokenizer for SQL
13 **
14 ** This file contains C code that splits an SQL input string up into
15 ** individual tokens and sends those tokens one-by-one over to the
16 ** parser for analysis.
17 */
18 #include "sqliteInt.h"
19 #include <stdlib.h>
20
21 /* Character classes for tokenizing
22 **
23 ** In the sqlite3GetToken() function, a switch() on aiClass[c] is implemented
24 ** using a lookup table, whereas a switch() directly on c uses a binary search.
25 ** The lookup table is much faster. To maximize speed, and to ensure that
26 ** a lookup table is used, all of the classes need to be small integers and
27 ** all of them need to be used within the switch.
28 */
29 #define CC_X 0 /* The letter 'x', or start of BLOB literal */
30 #define CC_KYWD 1 /* Alphabetics or '_'. Usable in a keyword */
31 #define CC_ID 2 /* unicode characters usable in IDs */
32 #define CC_DIGIT 3 /* Digits */
33 #define CC_DOLLAR 4 /* '$' */
34 #define CC_VARALPHA 5 /* '@', '#', ':'. Alphabetic SQL variables */
35 #define CC_VARNUM 6 /* '?'. Numeric SQL variables */
36 #define CC_SPACE 7 /* Space characters */
37 #define CC_QUOTE 8 /* '"', '\'', or '`'. String literals, quoted ids */
38 #define CC_QUOTE2 9 /* '['. [...] style quoted ids */
39 #define CC_PIPE 10 /* '|'. Bitwise OR or concatenate */
40 #define CC_MINUS 11 /* '-'. Minus or SQL-style comment */
41 #define CC_LT 12 /* '<'. Part of < or <= or <> */
42 #define CC_GT 13 /* '>'. Part of > or >= */
43 #define CC_EQ 14 /* '='. Part of = or == */
44 #define CC_BANG 15 /* '!'. Part of != */
45 #define CC_SLASH 16 /* '/'. / or c-style comment */
46 #define CC_LP 17 /* '(' */
47 #define CC_RP 18 /* ')' */
48 #define CC_SEMI 19 /* ';' */
49 #define CC_PLUS 20 /* '+' */
50 #define CC_STAR 21 /* '*' */
51 #define CC_PERCENT 22 /* '%' */
52 #define CC_COMMA 23 /* ',' */
53 #define CC_AND 24 /* '&' */
54 #define CC_TILDA 25 /* '~' */
55 #define CC_DOT 26 /* '.' */
56 #define CC_ILLEGAL 27 /* Illegal character */
57 #define CC_NUL 28 /* 0x00 */
58
59 static const unsigned char aiClass[] = {
60 #ifdef SQLITE_ASCII
61 /* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xa xb xc xd xe xf */
62 /* 0x */ 28, 27, 27, 27, 27, 27, 27, 27, 27, 7, 7, 27, 7, 7, 27, 27,
63 /* 1x */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
64 /* 2x */ 7, 15, 8, 5, 4, 22, 24, 8, 17, 18, 21, 20, 23, 11, 26, 16,
65 /* 3x */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 5, 19, 12, 14, 13, 6,
66 /* 4x */ 5, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
67 /* 5x */ 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 9, 27, 27, 27, 1,
68 /* 6x */ 8, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
69 /* 7x */ 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 27, 10, 27, 25, 27,
70 /* 8x */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
71 /* 9x */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
72 /* Ax */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
73 /* Bx */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
74 /* Cx */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
75 /* Dx */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
76 /* Ex */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
77 /* Fx */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2
78 #endif
79 #ifdef SQLITE_EBCDIC
80 /* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xa xb xc xd xe xf */
81 /* 0x */ 27, 27, 27, 27, 27, 7, 27, 27, 27, 27, 27, 27, 7, 7, 27, 27,
82 /* 1x */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
83 /* 2x */ 27, 27, 27, 27, 27, 7, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
84 /* 3x */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
85 /* 4x */ 7, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 26, 12, 17, 20, 10,
86 /* 5x */ 24, 27, 27, 27, 27, 27, 27, 27, 27, 27, 15, 4, 21, 18, 19, 27,
87 /* 6x */ 11, 16, 27, 27, 27, 27, 27, 27, 27, 27, 27, 23, 22, 1, 13, 6,
88 /* 7x */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 8, 5, 5, 5, 8, 14, 8,
89 /* 8x */ 27, 1, 1, 1, 1, 1, 1, 1, 1, 1, 27, 27, 27, 27, 27, 27,
90 /* 9x */ 27, 1, 1, 1, 1, 1, 1, 1, 1, 1, 27, 27, 27, 27, 27, 27,
91 /* Ax */ 27, 25, 1, 1, 1, 1, 1, 0, 1, 1, 27, 27, 27, 27, 27, 27,
92 /* Bx */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 9, 27, 27, 27, 27, 27,
93 /* Cx */ 27, 1, 1, 1, 1, 1, 1, 1, 1, 1, 27, 27, 27, 27, 27, 27,
94 /* Dx */ 27, 1, 1, 1, 1, 1, 1, 1, 1, 1, 27, 27, 27, 27, 27, 27,
95 /* Ex */ 27, 27, 1, 1, 1, 1, 1, 0, 1, 1, 27, 27, 27, 27, 27, 27,
96 /* Fx */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 27, 27, 27, 27, 27, 27,
97 #endif
98 };
99
100 /*
101 ** The charMap() macro maps alphabetic characters (only) into their
102 ** lower-case ASCII equivalent. On ASCII machines, this is just
103 ** an upper-to-lower case map. On EBCDIC machines we also need
104 ** to adjust the encoding. The mapping is only valid for alphabetics
105 ** which are the only characters for which this feature is used.
106 **
107 ** Used by keywordhash.h
108 */
109 #ifdef SQLITE_ASCII
110 # define charMap(X) sqlite3UpperToLower[(unsigned char)X]
111 #endif
112 #ifdef SQLITE_EBCDIC
113 # define charMap(X) ebcdicToAscii[(unsigned char)X]
114 const unsigned char ebcdicToAscii[] = {
115 /* 0 1 2 3 4 5 6 7 8 9 A B C D E F */
116 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x */
117 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 1x */
118 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 2x */
119 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 3x */
120 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 4x */
121 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 5x */
122 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 0, 0, /* 6x */
123 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 7x */
124 0, 97, 98, 99,100,101,102,103,104,105, 0, 0, 0, 0, 0, 0, /* 8x */
125 0,106,107,108,109,110,111,112,113,114, 0, 0, 0, 0, 0, 0, /* 9x */
126 0, 0,115,116,117,118,119,120,121,122, 0, 0, 0, 0, 0, 0, /* Ax */
127 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* Bx */
128 0, 97, 98, 99,100,101,102,103,104,105, 0, 0, 0, 0, 0, 0, /* Cx */
129 0,106,107,108,109,110,111,112,113,114, 0, 0, 0, 0, 0, 0, /* Dx */
130 0, 0,115,116,117,118,119,120,121,122, 0, 0, 0, 0, 0, 0, /* Ex */
131 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* Fx */
132 };
133 #endif
134
135 /*
136 ** The sqlite3KeywordCode function looks up an identifier to determine if
137 ** it is a keyword. If it is a keyword, the token code of that keyword is
138 ** returned. If the input is not a keyword, TK_ID is returned.
139 **
140 ** The implementation of this routine was generated by a program,
141 ** mkkeywordhash.c, located in the tool subdirectory of the distribution.
142 ** The output of the mkkeywordhash.c program is written into a file
143 ** named keywordhash.h and then included into this source file by
144 ** the #include below.
145 */
146 #include "keywordhash.h"
147
148
149 /*
150 ** If X is a character that can be used in an identifier then
151 ** IdChar(X) will be true. Otherwise it is false.
152 **
153 ** For ASCII, any character with the high-order bit set is
154 ** allowed in an identifier. For 7-bit characters,
155 ** sqlite3IsIdChar[X] must be 1.
156 **
157 ** For EBCDIC, the rules are more complex but have the same
158 ** end result.
159 **
160 ** Ticket #1066. the SQL standard does not allow '$' in the
161 ** middle of identifiers. But many SQL implementations do.
162 ** SQLite will allow '$' in identifiers for compatibility.
163 ** But the feature is undocumented.
164 */
165 #ifdef SQLITE_ASCII
166 #define IdChar(C) ((sqlite3CtypeMap[(unsigned char)C]&0x46)!=0)
167 #endif
168 #ifdef SQLITE_EBCDIC
169 const char sqlite3IsEbcdicIdChar[] = {
170 /* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xA xB xC xD xE xF */
171 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 4x */
172 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 0, 0, 0, /* 5x */
173 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 0, 0, /* 6x */
174 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, /* 7x */
175 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 1, 0, /* 8x */
176 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 1, 0, /* 9x */
177 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 0, /* Ax */
178 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* Bx */
179 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, /* Cx */
180 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, /* Dx */
181 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, /* Ex */
182 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 0, /* Fx */
183 };
184 #define IdChar(C) (((c=C)>=0x42 && sqlite3IsEbcdicIdChar[c-0x40]))
185 #endif
186
187 /* Make the IdChar function accessible from ctime.c and alter.c */
sqlite3IsIdChar(u8 c)188 int sqlite3IsIdChar(u8 c){ return IdChar(c); }
189
190 #ifndef SQLITE_OMIT_WINDOWFUNC
191 /*
192 ** Return the id of the next token in string (*pz). Before returning, set
193 ** (*pz) to point to the byte following the parsed token.
194 */
getToken(const unsigned char ** pz)195 static int getToken(const unsigned char **pz){
196 const unsigned char *z = *pz;
197 int t; /* Token type to return */
198 do {
199 z += sqlite3GetToken(z, &t);
200 }while( t==TK_SPACE );
201 if( t==TK_ID
202 || t==TK_STRING
203 || t==TK_JOIN_KW
204 || t==TK_WINDOW
205 || t==TK_OVER
206 || sqlite3ParserFallback(t)==TK_ID
207 ){
208 t = TK_ID;
209 }
210 *pz = z;
211 return t;
212 }
213
214 /*
215 ** The following three functions are called immediately after the tokenizer
216 ** reads the keywords WINDOW, OVER and FILTER, respectively, to determine
217 ** whether the token should be treated as a keyword or an SQL identifier.
218 ** This cannot be handled by the usual lemon %fallback method, due to
219 ** the ambiguity in some constructions. e.g.
220 **
221 ** SELECT sum(x) OVER ...
222 **
223 ** In the above, "OVER" might be a keyword, or it might be an alias for the
224 ** sum(x) expression. If a "%fallback ID OVER" directive were added to
225 ** grammar, then SQLite would always treat "OVER" as an alias, making it
226 ** impossible to call a window-function without a FILTER clause.
227 **
228 ** WINDOW is treated as a keyword if:
229 **
230 ** * the following token is an identifier, or a keyword that can fallback
231 ** to being an identifier, and
232 ** * the token after than one is TK_AS.
233 **
234 ** OVER is a keyword if:
235 **
236 ** * the previous token was TK_RP, and
237 ** * the next token is either TK_LP or an identifier.
238 **
239 ** FILTER is a keyword if:
240 **
241 ** * the previous token was TK_RP, and
242 ** * the next token is TK_LP.
243 */
analyzeWindowKeyword(const unsigned char * z)244 static int analyzeWindowKeyword(const unsigned char *z){
245 int t;
246 t = getToken(&z);
247 if( t!=TK_ID ) return TK_ID;
248 t = getToken(&z);
249 if( t!=TK_AS ) return TK_ID;
250 return TK_WINDOW;
251 }
analyzeOverKeyword(const unsigned char * z,int lastToken)252 static int analyzeOverKeyword(const unsigned char *z, int lastToken){
253 if( lastToken==TK_RP ){
254 int t = getToken(&z);
255 if( t==TK_LP || t==TK_ID ) return TK_OVER;
256 }
257 return TK_ID;
258 }
analyzeFilterKeyword(const unsigned char * z,int lastToken)259 static int analyzeFilterKeyword(const unsigned char *z, int lastToken){
260 if( lastToken==TK_RP && getToken(&z)==TK_LP ){
261 return TK_FILTER;
262 }
263 return TK_ID;
264 }
265 #endif /* SQLITE_OMIT_WINDOWFUNC */
266
267 /*
268 ** Return the length (in bytes) of the token that begins at z[0].
269 ** Store the token type in *tokenType before returning.
270 */
sqlite3GetToken(const unsigned char * z,int * tokenType)271 int sqlite3GetToken(const unsigned char *z, int *tokenType){
272 int i, c;
273 switch( aiClass[*z] ){ /* Switch on the character-class of the first byte
274 ** of the token. See the comment on the CC_ defines
275 ** above. */
276 case CC_SPACE: {
277 testcase( z[0]==' ' );
278 testcase( z[0]=='\t' );
279 testcase( z[0]=='\n' );
280 testcase( z[0]=='\f' );
281 testcase( z[0]=='\r' );
282 for(i=1; sqlite3Isspace(z[i]); i++){}
283 *tokenType = TK_SPACE;
284 return i;
285 }
286 case CC_MINUS: {
287 if( z[1]=='-' ){
288 for(i=2; (c=z[i])!=0 && c!='\n'; i++){}
289 *tokenType = TK_SPACE; /* IMP: R-22934-25134 */
290 return i;
291 }
292 *tokenType = TK_MINUS;
293 return 1;
294 }
295 case CC_LP: {
296 *tokenType = TK_LP;
297 return 1;
298 }
299 case CC_RP: {
300 *tokenType = TK_RP;
301 return 1;
302 }
303 case CC_SEMI: {
304 *tokenType = TK_SEMI;
305 return 1;
306 }
307 case CC_PLUS: {
308 *tokenType = TK_PLUS;
309 return 1;
310 }
311 case CC_STAR: {
312 *tokenType = TK_STAR;
313 return 1;
314 }
315 case CC_SLASH: {
316 if( z[1]!='*' || z[2]==0 ){
317 *tokenType = TK_SLASH;
318 return 1;
319 }
320 for(i=3, c=z[2]; (c!='*' || z[i]!='/') && (c=z[i])!=0; i++){}
321 if( c ) i++;
322 *tokenType = TK_SPACE; /* IMP: R-22934-25134 */
323 return i;
324 }
325 case CC_PERCENT: {
326 *tokenType = TK_REM;
327 return 1;
328 }
329 case CC_EQ: {
330 *tokenType = TK_EQ;
331 return 1 + (z[1]=='=');
332 }
333 case CC_LT: {
334 if( (c=z[1])=='=' ){
335 *tokenType = TK_LE;
336 return 2;
337 }else if( c=='>' ){
338 *tokenType = TK_NE;
339 return 2;
340 }else if( c=='<' ){
341 *tokenType = TK_LSHIFT;
342 return 2;
343 }else{
344 *tokenType = TK_LT;
345 return 1;
346 }
347 }
348 case CC_GT: {
349 if( (c=z[1])=='=' ){
350 *tokenType = TK_GE;
351 return 2;
352 }else if( c=='>' ){
353 *tokenType = TK_RSHIFT;
354 return 2;
355 }else{
356 *tokenType = TK_GT;
357 return 1;
358 }
359 }
360 case CC_BANG: {
361 if( z[1]!='=' ){
362 *tokenType = TK_ILLEGAL;
363 return 1;
364 }else{
365 *tokenType = TK_NE;
366 return 2;
367 }
368 }
369 case CC_PIPE: {
370 if( z[1]!='|' ){
371 *tokenType = TK_BITOR;
372 return 1;
373 }else{
374 *tokenType = TK_CONCAT;
375 return 2;
376 }
377 }
378 case CC_COMMA: {
379 *tokenType = TK_COMMA;
380 return 1;
381 }
382 case CC_AND: {
383 *tokenType = TK_BITAND;
384 return 1;
385 }
386 case CC_TILDA: {
387 *tokenType = TK_BITNOT;
388 return 1;
389 }
390 case CC_QUOTE: {
391 int delim = z[0];
392 testcase( delim=='`' );
393 testcase( delim=='\'' );
394 testcase( delim=='"' );
395 for(i=1; (c=z[i])!=0; i++){
396 if( c==delim ){
397 if( z[i+1]==delim ){
398 i++;
399 }else{
400 break;
401 }
402 }
403 }
404 if( c=='\'' ){
405 *tokenType = TK_STRING;
406 return i+1;
407 }else if( c!=0 ){
408 *tokenType = TK_ID;
409 return i+1;
410 }else{
411 *tokenType = TK_ILLEGAL;
412 return i;
413 }
414 }
415 case CC_DOT: {
416 #ifndef SQLITE_OMIT_FLOATING_POINT
417 if( !sqlite3Isdigit(z[1]) )
418 #endif
419 {
420 *tokenType = TK_DOT;
421 return 1;
422 }
423 /* If the next character is a digit, this is a floating point
424 ** number that begins with ".". Fall thru into the next case */
425 /* no break */ deliberate_fall_through
426 }
427 case CC_DIGIT: {
428 testcase( z[0]=='0' ); testcase( z[0]=='1' ); testcase( z[0]=='2' );
429 testcase( z[0]=='3' ); testcase( z[0]=='4' ); testcase( z[0]=='5' );
430 testcase( z[0]=='6' ); testcase( z[0]=='7' ); testcase( z[0]=='8' );
431 testcase( z[0]=='9' );
432 *tokenType = TK_INTEGER;
433 #ifndef SQLITE_OMIT_HEX_INTEGER
434 if( z[0]=='0' && (z[1]=='x' || z[1]=='X') && sqlite3Isxdigit(z[2]) ){
435 for(i=3; sqlite3Isxdigit(z[i]); i++){}
436 return i;
437 }
438 #endif
439 for(i=0; sqlite3Isdigit(z[i]); i++){}
440 #ifndef SQLITE_OMIT_FLOATING_POINT
441 if( z[i]=='.' ){
442 i++;
443 while( sqlite3Isdigit(z[i]) ){ i++; }
444 *tokenType = TK_FLOAT;
445 }
446 if( (z[i]=='e' || z[i]=='E') &&
447 ( sqlite3Isdigit(z[i+1])
448 || ((z[i+1]=='+' || z[i+1]=='-') && sqlite3Isdigit(z[i+2]))
449 )
450 ){
451 i += 2;
452 while( sqlite3Isdigit(z[i]) ){ i++; }
453 *tokenType = TK_FLOAT;
454 }
455 #endif
456 while( IdChar(z[i]) ){
457 *tokenType = TK_ILLEGAL;
458 i++;
459 }
460 return i;
461 }
462 case CC_QUOTE2: {
463 for(i=1, c=z[0]; c!=']' && (c=z[i])!=0; i++){}
464 *tokenType = c==']' ? TK_ID : TK_ILLEGAL;
465 return i;
466 }
467 case CC_VARNUM: {
468 *tokenType = TK_VARIABLE;
469 for(i=1; sqlite3Isdigit(z[i]); i++){}
470 return i;
471 }
472 case CC_DOLLAR:
473 case CC_VARALPHA: {
474 int n = 0;
475 testcase( z[0]=='$' ); testcase( z[0]=='@' );
476 testcase( z[0]==':' ); testcase( z[0]=='#' );
477 *tokenType = TK_VARIABLE;
478 for(i=1; (c=z[i])!=0; i++){
479 if( IdChar(c) ){
480 n++;
481 #ifndef SQLITE_OMIT_TCL_VARIABLE
482 }else if( c=='(' && n>0 ){
483 do{
484 i++;
485 }while( (c=z[i])!=0 && !sqlite3Isspace(c) && c!=')' );
486 if( c==')' ){
487 i++;
488 }else{
489 *tokenType = TK_ILLEGAL;
490 }
491 break;
492 }else if( c==':' && z[i+1]==':' ){
493 i++;
494 #endif
495 }else{
496 break;
497 }
498 }
499 if( n==0 ) *tokenType = TK_ILLEGAL;
500 return i;
501 }
502 case CC_KYWD: {
503 for(i=1; aiClass[z[i]]<=CC_KYWD; i++){}
504 if( IdChar(z[i]) ){
505 /* This token started out using characters that can appear in keywords,
506 ** but z[i] is a character not allowed within keywords, so this must
507 ** be an identifier instead */
508 i++;
509 break;
510 }
511 *tokenType = TK_ID;
512 return keywordCode((char*)z, i, tokenType);
513 }
514 case CC_X: {
515 #ifndef SQLITE_OMIT_BLOB_LITERAL
516 testcase( z[0]=='x' ); testcase( z[0]=='X' );
517 if( z[1]=='\'' ){
518 *tokenType = TK_BLOB;
519 for(i=2; sqlite3Isxdigit(z[i]); i++){}
520 if( z[i]!='\'' || i%2 ){
521 *tokenType = TK_ILLEGAL;
522 while( z[i] && z[i]!='\'' ){ i++; }
523 }
524 if( z[i] ) i++;
525 return i;
526 }
527 #endif
528 /* If it is not a BLOB literal, then it must be an ID, since no
529 ** SQL keywords start with the letter 'x'. Fall through */
530 /* no break */ deliberate_fall_through
531 }
532 case CC_ID: {
533 i = 1;
534 break;
535 }
536 case CC_NUL: {
537 *tokenType = TK_ILLEGAL;
538 return 0;
539 }
540 default: {
541 *tokenType = TK_ILLEGAL;
542 return 1;
543 }
544 }
545 while( IdChar(z[i]) ){ i++; }
546 *tokenType = TK_ID;
547 return i;
548 }
549
550 /*
551 ** Run the parser on the given SQL string. The parser structure is
552 ** passed in. An SQLITE_ status code is returned. If an error occurs
553 ** then an and attempt is made to write an error message into
554 ** memory obtained from sqlite3_malloc() and to make *pzErrMsg point to that
555 ** error message.
556 */
sqlite3RunParser(Parse * pParse,const char * zSql,char ** pzErrMsg)557 int sqlite3RunParser(Parse *pParse, const char *zSql, char **pzErrMsg){
558 int nErr = 0; /* Number of errors encountered */
559 void *pEngine; /* The LEMON-generated LALR(1) parser */
560 int n = 0; /* Length of the next token token */
561 int tokenType; /* type of the next token */
562 int lastTokenParsed = -1; /* type of the previous token */
563 sqlite3 *db = pParse->db; /* The database connection */
564 int mxSqlLen; /* Max length of an SQL string */
565 #ifdef sqlite3Parser_ENGINEALWAYSONSTACK
566 yyParser sEngine; /* Space to hold the Lemon-generated Parser object */
567 #endif
568 VVA_ONLY( u8 startedWithOom = db->mallocFailed );
569
570 assert( zSql!=0 );
571 mxSqlLen = db->aLimit[SQLITE_LIMIT_SQL_LENGTH];
572 if( db->nVdbeActive==0 ){
573 AtomicStore(&db->u1.isInterrupted, 0);
574 }
575 pParse->rc = SQLITE_OK;
576 pParse->zTail = zSql;
577 assert( pzErrMsg!=0 );
578 #ifdef SQLITE_DEBUG
579 if( db->flags & SQLITE_ParserTrace ){
580 printf("parser: [[[%s]]]\n", zSql);
581 sqlite3ParserTrace(stdout, "parser: ");
582 }else{
583 sqlite3ParserTrace(0, 0);
584 }
585 #endif
586 #ifdef sqlite3Parser_ENGINEALWAYSONSTACK
587 pEngine = &sEngine;
588 sqlite3ParserInit(pEngine, pParse);
589 #else
590 pEngine = sqlite3ParserAlloc(sqlite3Malloc, pParse);
591 if( pEngine==0 ){
592 sqlite3OomFault(db);
593 return SQLITE_NOMEM_BKPT;
594 }
595 #endif
596 assert( pParse->pNewTable==0 );
597 assert( pParse->pNewTrigger==0 );
598 assert( pParse->nVar==0 );
599 assert( pParse->pVList==0 );
600 pParse->pParentParse = db->pParse;
601 db->pParse = pParse;
602 while( 1 ){
603 n = sqlite3GetToken((u8*)zSql, &tokenType);
604 mxSqlLen -= n;
605 if( mxSqlLen<0 ){
606 pParse->rc = SQLITE_TOOBIG;
607 break;
608 }
609 #ifndef SQLITE_OMIT_WINDOWFUNC
610 if( tokenType>=TK_WINDOW ){
611 assert( tokenType==TK_SPACE || tokenType==TK_OVER || tokenType==TK_FILTER
612 || tokenType==TK_ILLEGAL || tokenType==TK_WINDOW
613 );
614 #else
615 if( tokenType>=TK_SPACE ){
616 assert( tokenType==TK_SPACE || tokenType==TK_ILLEGAL );
617 #endif /* SQLITE_OMIT_WINDOWFUNC */
618 if( AtomicLoad(&db->u1.isInterrupted) ){
619 pParse->rc = SQLITE_INTERRUPT;
620 break;
621 }
622 if( tokenType==TK_SPACE ){
623 zSql += n;
624 continue;
625 }
626 if( zSql[0]==0 ){
627 /* Upon reaching the end of input, call the parser two more times
628 ** with tokens TK_SEMI and 0, in that order. */
629 if( lastTokenParsed==TK_SEMI ){
630 tokenType = 0;
631 }else if( lastTokenParsed==0 ){
632 break;
633 }else{
634 tokenType = TK_SEMI;
635 }
636 n = 0;
637 #ifndef SQLITE_OMIT_WINDOWFUNC
638 }else if( tokenType==TK_WINDOW ){
639 assert( n==6 );
640 tokenType = analyzeWindowKeyword((const u8*)&zSql[6]);
641 }else if( tokenType==TK_OVER ){
642 assert( n==4 );
643 tokenType = analyzeOverKeyword((const u8*)&zSql[4], lastTokenParsed);
644 }else if( tokenType==TK_FILTER ){
645 assert( n==6 );
646 tokenType = analyzeFilterKeyword((const u8*)&zSql[6], lastTokenParsed);
647 #endif /* SQLITE_OMIT_WINDOWFUNC */
648 }else{
649 sqlite3ErrorMsg(pParse, "unrecognized token: \"%.*s\"", n, zSql);
650 break;
651 }
652 }
653 pParse->sLastToken.z = zSql;
654 pParse->sLastToken.n = n;
655 sqlite3Parser(pEngine, tokenType, pParse->sLastToken);
656 lastTokenParsed = tokenType;
657 zSql += n;
658 assert( db->mallocFailed==0 || pParse->rc!=SQLITE_OK || startedWithOom );
659 if( pParse->rc!=SQLITE_OK ) break;
660 }
661 assert( nErr==0 );
662 #ifdef YYTRACKMAXSTACKDEPTH
663 sqlite3_mutex_enter(sqlite3MallocMutex());
664 sqlite3StatusHighwater(SQLITE_STATUS_PARSER_STACK,
665 sqlite3ParserStackPeak(pEngine)
666 );
667 sqlite3_mutex_leave(sqlite3MallocMutex());
668 #endif /* YYDEBUG */
669 #ifdef sqlite3Parser_ENGINEALWAYSONSTACK
670 sqlite3ParserFinalize(pEngine);
671 #else
672 sqlite3ParserFree(pEngine, sqlite3_free);
673 #endif
674 if( db->mallocFailed ){
675 pParse->rc = SQLITE_NOMEM_BKPT;
676 }
677 if( pParse->rc!=SQLITE_OK && pParse->rc!=SQLITE_DONE && pParse->zErrMsg==0 ){
678 pParse->zErrMsg = sqlite3MPrintf(db, "%s", sqlite3ErrStr(pParse->rc));
679 }
680 assert( pzErrMsg!=0 );
681 if( pParse->zErrMsg ){
682 *pzErrMsg = pParse->zErrMsg;
683 sqlite3_log(pParse->rc, "%s in \"%s\"",
684 *pzErrMsg, pParse->zTail);
685 pParse->zErrMsg = 0;
686 nErr++;
687 }
688 pParse->zTail = zSql;
689 if( pParse->pVdbe && pParse->nErr>0 && pParse->nested==0 ){
690 sqlite3VdbeDelete(pParse->pVdbe);
691 pParse->pVdbe = 0;
692 }
693 #ifndef SQLITE_OMIT_SHARED_CACHE
694 if( pParse->nested==0 ){
695 sqlite3DbFree(db, pParse->aTableLock);
696 pParse->aTableLock = 0;
697 pParse->nTableLock = 0;
698 }
699 #endif
700 #ifndef SQLITE_OMIT_VIRTUALTABLE
701 sqlite3_free(pParse->apVtabLock);
702 #endif
703
704 if( !IN_SPECIAL_PARSE ){
705 /* If the pParse->declareVtab flag is set, do not delete any table
706 ** structure built up in pParse->pNewTable. The calling code (see vtab.c)
707 ** will take responsibility for freeing the Table structure.
708 */
709 sqlite3DeleteTable(db, pParse->pNewTable);
710 }
711 if( !IN_RENAME_OBJECT ){
712 sqlite3DeleteTrigger(db, pParse->pNewTrigger);
713 }
714
715 if( pParse->pWithToFree ) sqlite3WithDelete(db, pParse->pWithToFree);
716 sqlite3DbFree(db, pParse->pVList);
717 while( pParse->pAinc ){
718 AutoincInfo *p = pParse->pAinc;
719 pParse->pAinc = p->pNext;
720 sqlite3DbFreeNN(db, p);
721 }
722 while( pParse->pZombieTab ){
723 Table *p = pParse->pZombieTab;
724 pParse->pZombieTab = p->pNextZombie;
725 sqlite3DeleteTable(db, p);
726 }
727 db->pParse = pParse->pParentParse;
728 pParse->pParentParse = 0;
729 assert( nErr==0 || pParse->rc!=SQLITE_OK );
730 return nErr;
731 }
732
733
734 #ifdef SQLITE_ENABLE_NORMALIZE
735 /*
736 ** Insert a single space character into pStr if the current string
737 ** ends with an identifier
738 */
739 static void addSpaceSeparator(sqlite3_str *pStr){
740 if( pStr->nChar && sqlite3IsIdChar(pStr->zText[pStr->nChar-1]) ){
741 sqlite3_str_append(pStr, " ", 1);
742 }
743 }
744
745 /*
746 ** Compute a normalization of the SQL given by zSql[0..nSql-1]. Return
747 ** the normalization in space obtained from sqlite3DbMalloc(). Or return
748 ** NULL if anything goes wrong or if zSql is NULL.
749 */
750 char *sqlite3Normalize(
751 Vdbe *pVdbe, /* VM being reprepared */
752 const char *zSql /* The original SQL string */
753 ){
754 sqlite3 *db; /* The database connection */
755 int i; /* Next unread byte of zSql[] */
756 int n; /* length of current token */
757 int tokenType; /* type of current token */
758 int prevType = 0; /* Previous non-whitespace token */
759 int nParen; /* Number of nested levels of parentheses */
760 int iStartIN; /* Start of RHS of IN operator in z[] */
761 int nParenAtIN; /* Value of nParent at start of RHS of IN operator */
762 u32 j; /* Bytes of normalized SQL generated so far */
763 sqlite3_str *pStr; /* The normalized SQL string under construction */
764
765 db = sqlite3VdbeDb(pVdbe);
766 tokenType = -1;
767 nParen = iStartIN = nParenAtIN = 0;
768 pStr = sqlite3_str_new(db);
769 assert( pStr!=0 ); /* sqlite3_str_new() never returns NULL */
770 for(i=0; zSql[i] && pStr->accError==0; i+=n){
771 if( tokenType!=TK_SPACE ){
772 prevType = tokenType;
773 }
774 n = sqlite3GetToken((unsigned char*)zSql+i, &tokenType);
775 if( NEVER(n<=0) ) break;
776 switch( tokenType ){
777 case TK_SPACE: {
778 break;
779 }
780 case TK_NULL: {
781 if( prevType==TK_IS || prevType==TK_NOT ){
782 sqlite3_str_append(pStr, " NULL", 5);
783 break;
784 }
785 /* Fall through */
786 }
787 case TK_STRING:
788 case TK_INTEGER:
789 case TK_FLOAT:
790 case TK_VARIABLE:
791 case TK_BLOB: {
792 sqlite3_str_append(pStr, "?", 1);
793 break;
794 }
795 case TK_LP: {
796 nParen++;
797 if( prevType==TK_IN ){
798 iStartIN = pStr->nChar;
799 nParenAtIN = nParen;
800 }
801 sqlite3_str_append(pStr, "(", 1);
802 break;
803 }
804 case TK_RP: {
805 if( iStartIN>0 && nParen==nParenAtIN ){
806 assert( pStr->nChar>=(u32)iStartIN );
807 pStr->nChar = iStartIN+1;
808 sqlite3_str_append(pStr, "?,?,?", 5);
809 iStartIN = 0;
810 }
811 nParen--;
812 sqlite3_str_append(pStr, ")", 1);
813 break;
814 }
815 case TK_ID: {
816 iStartIN = 0;
817 j = pStr->nChar;
818 if( sqlite3Isquote(zSql[i]) ){
819 char *zId = sqlite3DbStrNDup(db, zSql+i, n);
820 int nId;
821 int eType = 0;
822 if( zId==0 ) break;
823 sqlite3Dequote(zId);
824 if( zSql[i]=='"' && sqlite3VdbeUsesDoubleQuotedString(pVdbe, zId) ){
825 sqlite3_str_append(pStr, "?", 1);
826 sqlite3DbFree(db, zId);
827 break;
828 }
829 nId = sqlite3Strlen30(zId);
830 if( sqlite3GetToken((u8*)zId, &eType)==nId && eType==TK_ID ){
831 addSpaceSeparator(pStr);
832 sqlite3_str_append(pStr, zId, nId);
833 }else{
834 sqlite3_str_appendf(pStr, "\"%w\"", zId);
835 }
836 sqlite3DbFree(db, zId);
837 }else{
838 addSpaceSeparator(pStr);
839 sqlite3_str_append(pStr, zSql+i, n);
840 }
841 while( j<pStr->nChar ){
842 pStr->zText[j] = sqlite3Tolower(pStr->zText[j]);
843 j++;
844 }
845 break;
846 }
847 case TK_SELECT: {
848 iStartIN = 0;
849 /* fall through */
850 }
851 default: {
852 if( sqlite3IsIdChar(zSql[i]) ) addSpaceSeparator(pStr);
853 j = pStr->nChar;
854 sqlite3_str_append(pStr, zSql+i, n);
855 while( j<pStr->nChar ){
856 pStr->zText[j] = sqlite3Toupper(pStr->zText[j]);
857 j++;
858 }
859 break;
860 }
861 }
862 }
863 if( tokenType!=TK_SEMI ) sqlite3_str_append(pStr, ";", 1);
864 return sqlite3_str_finish(pStr);
865 }
866 #endif /* SQLITE_ENABLE_NORMALIZE */
867