1 /*
2 ** 2001 September 15
3 **
4 ** The author disclaims copyright to this source code. In place of
5 ** a legal notice, here is a blessing:
6 **
7 ** May you do good and not evil.
8 ** May you find forgiveness for yourself and forgive others.
9 ** May you share freely, never taking more than you give.
10 **
11 *************************************************************************
12 ** An tokenizer for SQL
13 **
14 ** This file contains C code that splits an SQL input string up into
15 ** individual tokens and sends those tokens one-by-one over to the
16 ** parser for analysis.
17 */
18 #include "src/trace_processor/perfetto_sql/tokenizer/tokenize_internal_helper.h"
19 #include <stdlib.h>
20
21 /* Character classes for tokenizing
22 **
23 ** In the sqlite3GetToken() function, a switch() on aiClass[c] is implemented
24 ** using a lookup table, whereas a switch() directly on c uses a binary search.
25 ** The lookup table is much faster. To maximize speed, and to ensure that
26 ** a lookup table is used, all of the classes need to be small integers and
27 ** all of them need to be used within the switch.
28 */
29 #define CC_X 0 /* The letter 'x', or start of BLOB literal */
30 #define CC_KYWD0 1 /* First letter of a keyword */
31 #define CC_KYWD 2 /* Alphabetics or '_'. Usable in a keyword */
32 #define CC_DIGIT 3 /* Digits */
33 #define CC_DOLLAR 4 /* '$' */
34 #define CC_VARALPHA 5 /* '@', '#', ':'. Alphabetic SQL variables */
35 #define CC_VARNUM 6 /* '?'. Numeric SQL variables */
36 #define CC_SPACE 7 /* Space characters */
37 #define CC_QUOTE 8 /* '"', '\'', or '`'. String literals, quoted ids */
38 #define CC_QUOTE2 9 /* '['. [...] style quoted ids */
39 #define CC_PIPE 10 /* '|'. Bitwise OR or concatenate */
40 #define CC_MINUS 11 /* '-'. Minus or SQL-style comment */
41 #define CC_LT 12 /* '<'. Part of < or <= or <> */
42 #define CC_GT 13 /* '>'. Part of > or >= */
43 #define CC_EQ 14 /* '='. Part of = or == */
44 #define CC_BANG 15 /* '!'. Part of != */
45 #define CC_SLASH 16 /* '/'. / or c-style comment */
46 #define CC_LP 17 /* '(' */
47 #define CC_RP 18 /* ')' */
48 #define CC_SEMI 19 /* ';' */
49 #define CC_PLUS 20 /* '+' */
50 #define CC_STAR 21 /* '*' */
51 #define CC_PERCENT 22 /* '%' */
52 #define CC_COMMA 23 /* ',' */
53 #define CC_AND 24 /* '&' */
54 #define CC_TILDA 25 /* '~' */
55 #define CC_DOT 26 /* '.' */
56 #define CC_ID 27 /* unicode characters usable in IDs */
57 #define CC_ILLEGAL 28 /* Illegal character */
58 #define CC_NUL 29 /* 0x00 */
59 #define CC_BOM 30 /* First byte of UTF8 BOM: 0xEF 0xBB 0xBF */
60
61 static const unsigned char aiClass[] = {
62 #ifdef SQLITE_ASCII
63 /* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xa xb xc xd xe xf */
64 /* 0x */ 29, 28, 28, 28, 28, 28, 28, 28, 28, 7, 7, 28, 7, 7, 28, 28,
65 /* 1x */ 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28,
66 /* 2x */ 7, 15, 8, 5, 4, 22, 24, 8, 17, 18, 21, 20, 23, 11, 26, 16,
67 /* 3x */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 5, 19, 12, 14, 13, 6,
68 /* 4x */ 5, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
69 /* 5x */ 1, 1, 1, 1, 1, 1, 1, 1, 0, 2, 2, 9, 28, 28, 28, 2,
70 /* 6x */ 8, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
71 /* 7x */ 1, 1, 1, 1, 1, 1, 1, 1, 0, 2, 2, 28, 10, 28, 25, 28,
72 /* 8x */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
73 /* 9x */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
74 /* Ax */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
75 /* Bx */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
76 /* Cx */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
77 /* Dx */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
78 /* Ex */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 30,
79 /* Fx */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27
80 #endif
81 #ifdef SQLITE_EBCDIC
82 /* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xa xb xc xd xe xf */
83 /* 0x */ 29, 28, 28, 28, 28, 7, 28, 28, 28, 28, 28, 28, 7, 7, 28, 28,
84 /* 1x */ 28, 28, 28, 28, 28, 7, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28,
85 /* 2x */ 28, 28, 28, 28, 28, 7, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28,
86 /* 3x */ 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28,
87 /* 4x */ 7, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 26, 12, 17, 20, 10,
88 /* 5x */ 24, 28, 28, 28, 28, 28, 28, 28, 28, 28, 15, 4, 21, 18, 19, 28,
89 /* 6x */ 11, 16, 28, 28, 28, 28, 28, 28, 28, 28, 28, 23, 22, 2, 13, 6,
90 /* 7x */ 28, 28, 28, 28, 28, 28, 28, 28, 28, 8, 5, 5, 5, 8, 14, 8,
91 /* 8x */ 28, 1, 1, 1, 1, 1, 1, 1, 1, 1, 28, 28, 28, 28, 28, 28,
92 /* 9x */ 28, 1, 1, 1, 1, 1, 1, 1, 1, 1, 28, 28, 28, 28, 28, 28,
93 /* Ax */ 28, 25, 1, 1, 1, 1, 1, 0, 2, 2, 28, 28, 28, 28, 28, 28,
94 /* Bx */ 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 9, 28, 28, 28, 28, 28,
95 /* Cx */ 28, 1, 1, 1, 1, 1, 1, 1, 1, 1, 28, 28, 28, 28, 28, 28,
96 /* Dx */ 28, 1, 1, 1, 1, 1, 1, 1, 1, 1, 28, 28, 28, 28, 28, 28,
97 /* Ex */ 28, 28, 1, 1, 1, 1, 1, 0, 2, 2, 28, 28, 28, 28, 28, 28,
98 /* Fx */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 28, 28, 28, 28, 28, 28,
99 #endif
100 };
101
102 /*
103 ** The charMap() macro maps alphabetic characters (only) into their
104 ** lower-case ASCII equivalent. On ASCII machines, this is just
105 ** an upper-to-lower case map. On EBCDIC machines we also need
106 ** to adjust the encoding. The mapping is only valid for alphabetics
107 ** which are the only characters for which this feature is used.
108 **
109 ** Used by keywordhash.h
110 */
111 #ifdef SQLITE_ASCII
112 # define charMap(X) sqlite3UpperToLower[(unsigned char)X]
113 #endif
114 #ifdef SQLITE_EBCDIC
115 # define charMap(X) ebcdicToAscii[(unsigned char)X]
116 const unsigned char ebcdicToAscii[] = {
117 /* 0 1 2 3 4 5 6 7 8 9 A B C D E F */
118 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x */
119 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 1x */
120 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 2x */
121 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 3x */
122 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 4x */
123 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 5x */
124 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 0, 0, /* 6x */
125 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 7x */
126 0, 97, 98, 99,100,101,102,103,104,105, 0, 0, 0, 0, 0, 0, /* 8x */
127 0,106,107,108,109,110,111,112,113,114, 0, 0, 0, 0, 0, 0, /* 9x */
128 0, 0,115,116,117,118,119,120,121,122, 0, 0, 0, 0, 0, 0, /* Ax */
129 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* Bx */
130 0, 97, 98, 99,100,101,102,103,104,105, 0, 0, 0, 0, 0, 0, /* Cx */
131 0,106,107,108,109,110,111,112,113,114, 0, 0, 0, 0, 0, 0, /* Dx */
132 0, 0,115,116,117,118,119,120,121,122, 0, 0, 0, 0, 0, 0, /* Ex */
133 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* Fx */
134 };
135 #endif
136
137 /*
138 ** The sqlite3KeywordCode function looks up an identifier to determine if
139 ** it is a keyword. If it is a keyword, the token code of that keyword is
140 ** returned. If the input is not a keyword, TK_ID is returned.
141 **
142 ** The implementation of this routine was generated by a program,
143 ** mkkeywordhash.c, located in the tool subdirectory of the distribution.
144 ** The output of the mkkeywordhash.c program is written into a file
145 ** named keywordhash.h and then included into this source file by
146 ** the #include below.
147 */
148
149
150 /*
151 ** If X is a character that can be used in an identifier then
152 ** IdChar(X) will be true. Otherwise it is false.
153 **
154 ** For ASCII, any character with the high-order bit set is
155 ** allowed in an identifier. For 7-bit characters,
156 ** sqlite3IsIdChar[X] must be 1.
157 **
158 ** For EBCDIC, the rules are more complex but have the same
159 ** end result.
160 **
161 ** Ticket #1066. the SQL standard does not allow '$' in the
162 ** middle of identifiers. But many SQL implementations do.
163 ** SQLite will allow '$' in identifiers for compatibility.
164 ** But the feature is undocumented.
165 */
166 #ifdef SQLITE_ASCII
167 #define IdChar(C) ((sqlite3CtypeMap[(unsigned char)C]&0x46)!=0)
168 #endif
169 #ifdef SQLITE_EBCDIC
170 const char sqlite3IsEbcdicIdChar[] = {
171 /* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xA xB xC xD xE xF */
172 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 4x */
173 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 0, 0, 0, /* 5x */
174 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 0, 0, /* 6x */
175 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, /* 7x */
176 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 1, 0, /* 8x */
177 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 1, 0, /* 9x */
178 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 0, /* Ax */
179 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* Bx */
180 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, /* Cx */
181 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, /* Dx */
182 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, /* Ex */
183 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 0, /* Fx */
184 };
185 #define IdChar(C) (((c=C)>=0x42 && sqlite3IsEbcdicIdChar[c-0x40]))
186 #endif
187
188 /* Make the IdChar function accessible from ctime.c and alter.c */
sqlite3IsIdChar(u8 c)189 int sqlite3IsIdChar(u8 c){ return IdChar(c); }
190
191 #ifndef SQLITE_OMIT_WINDOWFUNC
192 /*
193 ** Return the id of the next token in string (*pz). Before returning, set
194 ** (*pz) to point to the byte following the parsed token.
195 */
getToken(const unsigned char ** pz)196 static int getToken(const unsigned char **pz){
197 const unsigned char *z = *pz;
198 int t; /* Token type to return */
199 do {
200 z += sqlite3GetToken(z, &t);
201 }while( t==TK_SPACE );
202 if( t==TK_ID
203 || t==TK_STRING
204 || t==TK_JOIN_KW
205 || t==TK_WINDOW
206 || t==TK_OVER
207 || sqlite3ParserFallback(t)==TK_ID
208 ){
209 t = TK_ID;
210 }
211 *pz = z;
212 return t;
213 }
214
215 /*
216 ** The following three functions are called immediately after the tokenizer
217 ** reads the keywords WINDOW, OVER and FILTER, respectively, to determine
218 ** whether the token should be treated as a keyword or an SQL identifier.
219 ** This cannot be handled by the usual lemon %fallback method, due to
220 ** the ambiguity in some constructions. e.g.
221 **
222 ** SELECT sum(x) OVER ...
223 **
224 ** In the above, "OVER" might be a keyword, or it might be an alias for the
225 ** sum(x) expression. If a "%fallback ID OVER" directive were added to
226 ** grammar, then SQLite would always treat "OVER" as an alias, making it
227 ** impossible to call a window-function without a FILTER clause.
228 **
229 ** WINDOW is treated as a keyword if:
230 **
231 ** * the following token is an identifier, or a keyword that can fallback
232 ** to being an identifier, and
233 ** * the token after than one is TK_AS.
234 **
235 ** OVER is a keyword if:
236 **
237 ** * the previous token was TK_RP, and
238 ** * the next token is either TK_LP or an identifier.
239 **
240 ** FILTER is a keyword if:
241 **
242 ** * the previous token was TK_RP, and
243 ** * the next token is TK_LP.
244 */
analyzeWindowKeyword(const unsigned char * z)245 static int analyzeWindowKeyword(const unsigned char *z){
246 int t;
247 t = getToken(&z);
248 if( t!=TK_ID ) return TK_ID;
249 t = getToken(&z);
250 if( t!=TK_AS ) return TK_ID;
251 return TK_WINDOW;
252 }
analyzeOverKeyword(const unsigned char * z,int lastToken)253 static int analyzeOverKeyword(const unsigned char *z, int lastToken){
254 if( lastToken==TK_RP ){
255 int t = getToken(&z);
256 if( t==TK_LP || t==TK_ID ) return TK_OVER;
257 }
258 return TK_ID;
259 }
analyzeFilterKeyword(const unsigned char * z,int lastToken)260 static int analyzeFilterKeyword(const unsigned char *z, int lastToken){
261 if( lastToken==TK_RP && getToken(&z)==TK_LP ){
262 return TK_FILTER;
263 }
264 return TK_ID;
265 }
266 #endif /* SQLITE_OMIT_WINDOWFUNC */
267
268 /*
269 ** Return the length (in bytes) of the token that begins at z[0].
270 ** Store the token type in *tokenType before returning.
271 */
sqlite3GetToken(const unsigned char * z,int * tokenType)272 int sqlite3GetToken(const unsigned char *z, int *tokenType){
273 int i, c;
274 switch( aiClass[*z] ){ /* Switch on the character-class of the first byte
275 ** of the token. See the comment on the CC_ defines
276 ** above. */
277 case CC_SPACE: {
278 testcase( z[0]==' ' );
279 testcase( z[0]=='\t' );
280 testcase( z[0]=='\n' );
281 testcase( z[0]=='\f' );
282 testcase( z[0]=='\r' );
283 for(i=1; sqlite3Isspace(z[i]); i++){}
284 *tokenType = TK_SPACE;
285 return i;
286 }
287 case CC_MINUS: {
288 if( z[1]=='-' ){
289 for(i=2; (c=z[i])!=0 && c!='\n'; i++){}
290 *tokenType = TK_SPACE; /* IMP: R-22934-25134 */
291 return i;
292 }else if( z[1]=='>' ){
293 *tokenType = TK_PTR;
294 return 2 + (z[2]=='>');
295 }
296 *tokenType = TK_MINUS;
297 return 1;
298 }
299 case CC_LP: {
300 *tokenType = TK_LP;
301 return 1;
302 }
303 case CC_RP: {
304 *tokenType = TK_RP;
305 return 1;
306 }
307 case CC_SEMI: {
308 *tokenType = TK_SEMI;
309 return 1;
310 }
311 case CC_PLUS: {
312 *tokenType = TK_PLUS;
313 return 1;
314 }
315 case CC_STAR: {
316 *tokenType = TK_STAR;
317 return 1;
318 }
319 case CC_SLASH: {
320 if( z[1]!='*' || z[2]==0 ){
321 *tokenType = TK_SLASH;
322 return 1;
323 }
324 for(i=3, c=z[2]; (c!='*' || z[i]!='/') && (c=z[i])!=0; i++){}
325 if( c ) i++;
326 *tokenType = TK_SPACE; /* IMP: R-22934-25134 */
327 return i;
328 }
329 case CC_PERCENT: {
330 *tokenType = TK_REM;
331 return 1;
332 }
333 case CC_EQ: {
334 *tokenType = TK_EQ;
335 return 1 + (z[1]=='=');
336 }
337 case CC_LT: {
338 if( (c=z[1])=='=' ){
339 *tokenType = TK_LE;
340 return 2;
341 }else if( c=='>' ){
342 *tokenType = TK_NE;
343 return 2;
344 }else if( c=='<' ){
345 *tokenType = TK_LSHIFT;
346 return 2;
347 }else{
348 *tokenType = TK_LT;
349 return 1;
350 }
351 }
352 case CC_GT: {
353 if( (c=z[1])=='=' ){
354 *tokenType = TK_GE;
355 return 2;
356 }else if( c=='>' ){
357 *tokenType = TK_RSHIFT;
358 return 2;
359 }else{
360 *tokenType = TK_GT;
361 return 1;
362 }
363 }
364 case CC_BANG: {
365 if( z[1]!='=' ){
366 *tokenType = TK_ILLEGAL;
367 return 1;
368 }else{
369 *tokenType = TK_NE;
370 return 2;
371 }
372 }
373 case CC_PIPE: {
374 if( z[1]!='|' ){
375 *tokenType = TK_BITOR;
376 return 1;
377 }else{
378 *tokenType = TK_CONCAT;
379 return 2;
380 }
381 }
382 case CC_COMMA: {
383 *tokenType = TK_COMMA;
384 return 1;
385 }
386 case CC_AND: {
387 *tokenType = TK_BITAND;
388 return 1;
389 }
390 case CC_TILDA: {
391 *tokenType = TK_BITNOT;
392 return 1;
393 }
394 case CC_QUOTE: {
395 int delim = z[0];
396 testcase( delim=='`' );
397 testcase( delim=='\'' );
398 testcase( delim=='"' );
399 for(i=1; (c=z[i])!=0; i++){
400 if( c==delim ){
401 if( z[i+1]==delim ){
402 i++;
403 }else{
404 break;
405 }
406 }
407 }
408 if( c=='\'' ){
409 *tokenType = TK_STRING;
410 return i+1;
411 }else if( c!=0 ){
412 *tokenType = TK_ID;
413 return i+1;
414 }else{
415 *tokenType = TK_ILLEGAL;
416 return i;
417 }
418 }
419 case CC_DOT: {
420 #ifndef SQLITE_OMIT_FLOATING_POINT
421 if( !sqlite3Isdigit(z[1]) )
422 #endif
423 {
424 *tokenType = TK_DOT;
425 return 1;
426 }
427 /* If the next character is a digit, this is a floating point
428 ** number that begins with ".". Fall thru into the next case */
429 /* no break */ deliberate_fall_through
430 }
431 case CC_DIGIT: {
432 testcase( z[0]=='0' ); testcase( z[0]=='1' ); testcase( z[0]=='2' );
433 testcase( z[0]=='3' ); testcase( z[0]=='4' ); testcase( z[0]=='5' );
434 testcase( z[0]=='6' ); testcase( z[0]=='7' ); testcase( z[0]=='8' );
435 testcase( z[0]=='9' ); testcase( z[0]=='.' );
436 *tokenType = TK_INTEGER;
437 #ifndef SQLITE_OMIT_HEX_INTEGER
438 if( z[0]=='0' && (z[1]=='x' || z[1]=='X') && sqlite3Isxdigit(z[2]) ){
439 for(i=3; sqlite3Isxdigit(z[i]); i++){}
440 return i;
441 }
442 #endif
443 for(i=0; sqlite3Isdigit(z[i]); i++){}
444 #ifndef SQLITE_OMIT_FLOATING_POINT
445 if( z[i]=='.' ){
446 i++;
447 while( sqlite3Isdigit(z[i]) ){ i++; }
448 *tokenType = TK_FLOAT;
449 }
450 if( (z[i]=='e' || z[i]=='E') &&
451 ( sqlite3Isdigit(z[i+1])
452 || ((z[i+1]=='+' || z[i+1]=='-') && sqlite3Isdigit(z[i+2]))
453 )
454 ){
455 i += 2;
456 while( sqlite3Isdigit(z[i]) ){ i++; }
457 *tokenType = TK_FLOAT;
458 }
459 #endif
460 while( IdChar(z[i]) ){
461 *tokenType = TK_ILLEGAL;
462 i++;
463 }
464 return i;
465 }
466 case CC_QUOTE2: {
467 for(i=1, c=z[0]; c!=']' && (c=z[i])!=0; i++){}
468 *tokenType = c==']' ? TK_ID : TK_ILLEGAL;
469 return i;
470 }
471 case CC_VARNUM: {
472 *tokenType = TK_VARIABLE;
473 for(i=1; sqlite3Isdigit(z[i]); i++){}
474 return i;
475 }
476 case CC_DOLLAR:
477 case CC_VARALPHA: {
478 int n = 0;
479 testcase( z[0]=='$' ); testcase( z[0]=='@' );
480 testcase( z[0]==':' ); testcase( z[0]=='#' );
481 *tokenType = TK_VARIABLE;
482 for(i=1; (c=z[i])!=0; i++){
483 if( IdChar(c) ){
484 n++;
485 #ifndef SQLITE_OMIT_TCL_VARIABLE
486 }else if( c=='(' && n>0 ){
487 do{
488 i++;
489 }while( (c=z[i])!=0 && !sqlite3Isspace(c) && c!=')' );
490 if( c==')' ){
491 i++;
492 }else{
493 *tokenType = TK_ILLEGAL;
494 }
495 break;
496 }else if( c==':' && z[i+1]==':' ){
497 i++;
498 #endif
499 }else{
500 break;
501 }
502 }
503 if( n==0 ) *tokenType = TK_ILLEGAL;
504 return i;
505 }
506 case CC_KYWD0: {
507 if( aiClass[z[1]]>CC_KYWD ){ i = 1; break; }
508 for(i=2; aiClass[z[i]]<=CC_KYWD; i++){}
509 if( IdChar(z[i]) ){
510 /* This token started out using characters that can appear in keywords,
511 ** but z[i] is a character not allowed within keywords, so this must
512 ** be an identifier instead */
513 i++;
514 break;
515 }
516 *tokenType = TK_ID;
517 return keywordCode((char*)z, i, tokenType);
518 }
519 case CC_X: {
520 #ifndef SQLITE_OMIT_BLOB_LITERAL
521 testcase( z[0]=='x' ); testcase( z[0]=='X' );
522 if( z[1]=='\'' ){
523 *tokenType = TK_BLOB;
524 for(i=2; sqlite3Isxdigit(z[i]); i++){}
525 if( z[i]!='\'' || i%2 ){
526 *tokenType = TK_ILLEGAL;
527 while( z[i] && z[i]!='\'' ){ i++; }
528 }
529 if( z[i] ) i++;
530 return i;
531 }
532 #endif
533 /* If it is not a BLOB literal, then it must be an ID, since no
534 ** SQL keywords start with the letter 'x'. Fall through */
535 /* no break */ deliberate_fall_through
536 }
537 case CC_KYWD:
538 case CC_ID: {
539 i = 1;
540 break;
541 }
542 case CC_BOM: {
543 if( z[1]==0xbb && z[2]==0xbf ){
544 *tokenType = TK_SPACE;
545 return 3;
546 }
547 i = 1;
548 break;
549 }
550 case CC_NUL: {
551 *tokenType = TK_ILLEGAL;
552 return 0;
553 }
554 default: {
555 *tokenType = TK_ILLEGAL;
556 return 1;
557 }
558 }
559 while( IdChar(z[i]) ){ i++; }
560 *tokenType = TK_ID;
561 return i;
562 }
563
564