1 /*
2 ** Lexical analyzer.
3 ** Copyright (C) 2005-2021 Mike Pall. See Copyright Notice in luajit.h
4 **
5 ** Major portions taken verbatim or adapted from the Lua interpreter.
6 ** Copyright (C) 1994-2008 Lua.org, PUC-Rio. See Copyright Notice in lua.h
7 */
8
9 #define lj_lex_c
10 #define LUA_CORE
11
12 #include "lj_obj.h"
13 #include "lj_gc.h"
14 #include "lj_err.h"
15 #include "lj_buf.h"
16 #include "lj_str.h"
17 #if LJ_HASFFI
18 #include "lj_tab.h"
19 #include "lj_ctype.h"
20 #include "lj_cdata.h"
21 #include "lualib.h"
22 #endif
23 #include "lj_state.h"
24 #include "lj_lex.h"
25 #include "lj_parse.h"
26 #include "lj_char.h"
27 #include "lj_strscan.h"
28 #include "lj_strfmt.h"
29
30 /* Lua lexer token names. */
31 static const char *const tokennames[] = {
32 #define TKSTR1(name) #name,
33 #define TKSTR2(name, sym) #sym,
34 TKDEF(TKSTR1, TKSTR2)
35 #undef TKSTR1
36 #undef TKSTR2
37 NULL
38 };
39
40 /* -- Buffer handling ----------------------------------------------------- */
41
42 #define LEX_EOF (-1)
43 #define lex_iseol(ls) (ls->c == '\n' || ls->c == '\r')
44
45 /* Get more input from reader. */
lex_more(LexState * ls)46 static LJ_NOINLINE LexChar lex_more(LexState *ls)
47 {
48 size_t sz;
49 const char *p = ls->rfunc(ls->L, ls->rdata, &sz);
50 if (p == NULL || sz == 0) return LEX_EOF;
51 if (sz >= LJ_MAX_BUF) {
52 if (sz != ~(size_t)0) lj_err_mem(ls->L);
53 sz = ~(uintptr_t)0 - (uintptr_t)p;
54 if (sz >= LJ_MAX_BUF) sz = LJ_MAX_BUF-1;
55 ls->endmark = 1;
56 }
57 ls->pe = p + sz;
58 ls->p = p + 1;
59 return (LexChar)(uint8_t)p[0];
60 }
61
62 /* Get next character. */
lex_next(LexState * ls)63 static LJ_AINLINE LexChar lex_next(LexState *ls)
64 {
65 return (ls->c = ls->p < ls->pe ? (LexChar)(uint8_t)*ls->p++ : lex_more(ls));
66 }
67
68 /* Save character. */
lex_save(LexState * ls,LexChar c)69 static LJ_AINLINE void lex_save(LexState *ls, LexChar c)
70 {
71 lj_buf_putb(&ls->sb, c);
72 }
73
74 /* Save previous character and get next character. */
lex_savenext(LexState * ls)75 static LJ_AINLINE LexChar lex_savenext(LexState *ls)
76 {
77 lex_save(ls, ls->c);
78 return lex_next(ls);
79 }
80
81 /* Skip line break. Handles "\n", "\r", "\r\n" or "\n\r". */
lex_newline(LexState * ls)82 static void lex_newline(LexState *ls)
83 {
84 LexChar old = ls->c;
85 lj_assertLS(lex_iseol(ls), "bad usage");
86 lex_next(ls); /* Skip "\n" or "\r". */
87 if (lex_iseol(ls) && ls->c != old) lex_next(ls); /* Skip "\n\r" or "\r\n". */
88 if (++ls->linenumber >= LJ_MAX_LINE)
89 lj_lex_error(ls, ls->tok, LJ_ERR_XLINES);
90 }
91
92 /* -- Scanner for terminals ----------------------------------------------- */
93
94 /* Parse a number literal. */
lex_number(LexState * ls,TValue * tv)95 static void lex_number(LexState *ls, TValue *tv)
96 {
97 StrScanFmt fmt;
98 LexChar c, xp = 'e';
99 lj_assertLS(lj_char_isdigit(ls->c), "bad usage");
100 if ((c = ls->c) == '0' && (lex_savenext(ls) | 0x20) == 'x')
101 xp = 'p';
102 while (lj_char_isident(ls->c) || ls->c == '.' ||
103 ((ls->c == '-' || ls->c == '+') && (c | 0x20) == xp)) {
104 c = ls->c;
105 lex_savenext(ls);
106 }
107 lex_save(ls, '\0');
108 fmt = lj_strscan_scan((const uint8_t *)ls->sb.b, sbuflen(&ls->sb)-1, tv,
109 (LJ_DUALNUM ? STRSCAN_OPT_TOINT : STRSCAN_OPT_TONUM) |
110 (LJ_HASFFI ? (STRSCAN_OPT_LL|STRSCAN_OPT_IMAG) : 0));
111 if (LJ_DUALNUM && fmt == STRSCAN_INT) {
112 setitype(tv, LJ_TISNUM);
113 } else if (fmt == STRSCAN_NUM) {
114 /* Already in correct format. */
115 #if LJ_HASFFI
116 } else if (fmt != STRSCAN_ERROR) {
117 lua_State *L = ls->L;
118 GCcdata *cd;
119 lj_assertLS(fmt == STRSCAN_I64 || fmt == STRSCAN_U64 || fmt == STRSCAN_IMAG,
120 "unexpected number format %d", fmt);
121 ctype_loadffi(L);
122 if (fmt == STRSCAN_IMAG) {
123 cd = lj_cdata_new_(L, CTID_COMPLEX_DOUBLE, 2*sizeof(double));
124 ((double *)cdataptr(cd))[0] = 0;
125 ((double *)cdataptr(cd))[1] = numV(tv);
126 } else {
127 cd = lj_cdata_new_(L, fmt==STRSCAN_I64 ? CTID_INT64 : CTID_UINT64, 8);
128 *(uint64_t *)cdataptr(cd) = tv->u64;
129 }
130 lj_parse_keepcdata(ls, tv, cd);
131 #endif
132 } else {
133 lj_assertLS(fmt == STRSCAN_ERROR,
134 "unexpected number format %d", fmt);
135 lj_lex_error(ls, TK_number, LJ_ERR_XNUMBER);
136 }
137 }
138
139 /* Skip equal signs for "[=...=[" and "]=...=]" and return their count. */
lex_skipeq(LexState * ls)140 static int lex_skipeq(LexState *ls)
141 {
142 int count = 0;
143 LexChar s = ls->c;
144 lj_assertLS(s == '[' || s == ']', "bad usage");
145 while (lex_savenext(ls) == '=' && count < 0x20000000)
146 count++;
147 return (ls->c == s) ? count : (-count) - 1;
148 }
149
150 /* Parse a long string or long comment (tv set to NULL). */
lex_longstring(LexState * ls,TValue * tv,int sep)151 static void lex_longstring(LexState *ls, TValue *tv, int sep)
152 {
153 lex_savenext(ls); /* Skip second '['. */
154 if (lex_iseol(ls)) /* Skip initial newline. */
155 lex_newline(ls);
156 for (;;) {
157 switch (ls->c) {
158 case LEX_EOF:
159 lj_lex_error(ls, TK_eof, tv ? LJ_ERR_XLSTR : LJ_ERR_XLCOM);
160 break;
161 case ']':
162 if (lex_skipeq(ls) == sep) {
163 lex_savenext(ls); /* Skip second ']'. */
164 goto endloop;
165 }
166 break;
167 case '\n':
168 case '\r':
169 lex_save(ls, '\n');
170 lex_newline(ls);
171 if (!tv) lj_buf_reset(&ls->sb); /* Don't waste space for comments. */
172 break;
173 default:
174 lex_savenext(ls);
175 break;
176 }
177 } endloop:
178 if (tv) {
179 GCstr *str = lj_parse_keepstr(ls, ls->sb.b + (2 + (MSize)sep),
180 sbuflen(&ls->sb) - 2*(2 + (MSize)sep));
181 setstrV(ls->L, tv, str);
182 }
183 }
184
185 /* Parse a string. */
lex_string(LexState * ls,TValue * tv)186 static void lex_string(LexState *ls, TValue *tv)
187 {
188 LexChar delim = ls->c; /* Delimiter is '\'' or '"'. */
189 lex_savenext(ls);
190 while (ls->c != delim) {
191 switch (ls->c) {
192 case LEX_EOF:
193 lj_lex_error(ls, TK_eof, LJ_ERR_XSTR);
194 continue;
195 case '\n':
196 case '\r':
197 lj_lex_error(ls, TK_string, LJ_ERR_XSTR);
198 continue;
199 case '\\': {
200 LexChar c = lex_next(ls); /* Skip the '\\'. */
201 switch (c) {
202 case 'a': c = '\a'; break;
203 case 'b': c = '\b'; break;
204 case 'f': c = '\f'; break;
205 case 'n': c = '\n'; break;
206 case 'r': c = '\r'; break;
207 case 't': c = '\t'; break;
208 case 'v': c = '\v'; break;
209 case 'x': /* Hexadecimal escape '\xXX'. */
210 c = (lex_next(ls) & 15u) << 4;
211 if (!lj_char_isdigit(ls->c)) {
212 if (!lj_char_isxdigit(ls->c)) goto err_xesc;
213 c += 9 << 4;
214 }
215 c += (lex_next(ls) & 15u);
216 if (!lj_char_isdigit(ls->c)) {
217 if (!lj_char_isxdigit(ls->c)) goto err_xesc;
218 c += 9;
219 }
220 break;
221 case 'u': /* Unicode escape '\u{XX...}'. */
222 if (lex_next(ls) != '{') goto err_xesc;
223 lex_next(ls);
224 c = 0;
225 do {
226 c = (c << 4) | (ls->c & 15u);
227 if (!lj_char_isdigit(ls->c)) {
228 if (!lj_char_isxdigit(ls->c)) goto err_xesc;
229 c += 9;
230 }
231 if (c >= 0x110000) goto err_xesc; /* Out of Unicode range. */
232 } while (lex_next(ls) != '}');
233 if (c < 0x800) {
234 if (c < 0x80) break;
235 lex_save(ls, 0xc0 | (c >> 6));
236 } else {
237 if (c >= 0x10000) {
238 lex_save(ls, 0xf0 | (c >> 18));
239 lex_save(ls, 0x80 | ((c >> 12) & 0x3f));
240 } else {
241 if (c >= 0xd800 && c < 0xe000) goto err_xesc; /* No surrogates. */
242 lex_save(ls, 0xe0 | (c >> 12));
243 }
244 lex_save(ls, 0x80 | ((c >> 6) & 0x3f));
245 }
246 c = 0x80 | (c & 0x3f);
247 break;
248 case 'z': /* Skip whitespace. */
249 lex_next(ls);
250 while (lj_char_isspace(ls->c))
251 if (lex_iseol(ls)) lex_newline(ls); else lex_next(ls);
252 continue;
253 case '\n': case '\r': lex_save(ls, '\n'); lex_newline(ls); continue;
254 case '\\': case '\"': case '\'': break;
255 case LEX_EOF: continue;
256 default:
257 if (!lj_char_isdigit(c))
258 goto err_xesc;
259 c -= '0'; /* Decimal escape '\ddd'. */
260 if (lj_char_isdigit(lex_next(ls))) {
261 c = c*10 + (ls->c - '0');
262 if (lj_char_isdigit(lex_next(ls))) {
263 c = c*10 + (ls->c - '0');
264 if (c > 255) {
265 err_xesc:
266 lj_lex_error(ls, TK_string, LJ_ERR_XESC);
267 }
268 lex_next(ls);
269 }
270 }
271 lex_save(ls, c);
272 continue;
273 }
274 lex_save(ls, c);
275 lex_next(ls);
276 continue;
277 }
278 default:
279 lex_savenext(ls);
280 break;
281 }
282 }
283 lex_savenext(ls); /* Skip trailing delimiter. */
284 setstrV(ls->L, tv,
285 lj_parse_keepstr(ls, ls->sb.b+1, sbuflen(&ls->sb)-2));
286 }
287
288 /* -- Main lexical scanner ------------------------------------------------ */
289
290 /* Get next lexical token. */
lex_scan(LexState * ls,TValue * tv)291 static LexToken lex_scan(LexState *ls, TValue *tv)
292 {
293 lj_buf_reset(&ls->sb);
294 for (;;) {
295 if (lj_char_isident(ls->c)) {
296 GCstr *s;
297 if (lj_char_isdigit(ls->c)) { /* Numeric literal. */
298 lex_number(ls, tv);
299 return TK_number;
300 }
301 /* Identifier or reserved word. */
302 do {
303 lex_savenext(ls);
304 } while (lj_char_isident(ls->c));
305 s = lj_parse_keepstr(ls, ls->sb.b, sbuflen(&ls->sb));
306 setstrV(ls->L, tv, s);
307 if (s->reserved > 0) /* Reserved word? */
308 return TK_OFS + s->reserved;
309 return TK_name;
310 }
311 switch (ls->c) {
312 case '\n':
313 case '\r':
314 lex_newline(ls);
315 continue;
316 case ' ':
317 case '\t':
318 case '\v':
319 case '\f':
320 lex_next(ls);
321 continue;
322 case '-':
323 lex_next(ls);
324 if (ls->c != '-') return '-';
325 lex_next(ls);
326 if (ls->c == '[') { /* Long comment "--[=*[...]=*]". */
327 int sep = lex_skipeq(ls);
328 lj_buf_reset(&ls->sb); /* `lex_skipeq' may dirty the buffer */
329 if (sep >= 0) {
330 lex_longstring(ls, NULL, sep);
331 lj_buf_reset(&ls->sb);
332 continue;
333 }
334 }
335 /* Short comment "--.*\n". */
336 while (!lex_iseol(ls) && ls->c != LEX_EOF)
337 lex_next(ls);
338 continue;
339 case '[': {
340 int sep = lex_skipeq(ls);
341 if (sep >= 0) {
342 lex_longstring(ls, tv, sep);
343 return TK_string;
344 } else if (sep == -1) {
345 return '[';
346 } else {
347 lj_lex_error(ls, TK_string, LJ_ERR_XLDELIM);
348 continue;
349 }
350 }
351 case '=':
352 lex_next(ls);
353 if (ls->c != '=') return '='; else { lex_next(ls); return TK_eq; }
354 case '<':
355 lex_next(ls);
356 if (ls->c != '=') return '<'; else { lex_next(ls); return TK_le; }
357 case '>':
358 lex_next(ls);
359 if (ls->c != '=') return '>'; else { lex_next(ls); return TK_ge; }
360 case '~':
361 lex_next(ls);
362 if (ls->c != '=') return '~'; else { lex_next(ls); return TK_ne; }
363 case ':':
364 lex_next(ls);
365 if (ls->c != ':') return ':'; else { lex_next(ls); return TK_label; }
366 case '"':
367 case '\'':
368 lex_string(ls, tv);
369 return TK_string;
370 case '.':
371 if (lex_savenext(ls) == '.') {
372 lex_next(ls);
373 if (ls->c == '.') {
374 lex_next(ls);
375 return TK_dots; /* ... */
376 }
377 return TK_concat; /* .. */
378 } else if (!lj_char_isdigit(ls->c)) {
379 return '.';
380 } else {
381 lex_number(ls, tv);
382 return TK_number;
383 }
384 case LEX_EOF:
385 return TK_eof;
386 default: {
387 LexChar c = ls->c;
388 lex_next(ls);
389 return c; /* Single-char tokens (+ - / ...). */
390 }
391 }
392 }
393 }
394
395 /* -- Lexer API ----------------------------------------------------------- */
396
397 /* Setup lexer state. */
lj_lex_setup(lua_State * L,LexState * ls)398 int lj_lex_setup(lua_State *L, LexState *ls)
399 {
400 int header = 0;
401 ls->L = L;
402 ls->fs = NULL;
403 ls->pe = ls->p = NULL;
404 ls->vstack = NULL;
405 ls->sizevstack = 0;
406 ls->vtop = 0;
407 ls->bcstack = NULL;
408 ls->sizebcstack = 0;
409 ls->tok = 0;
410 ls->lookahead = TK_eof; /* No look-ahead token. */
411 ls->linenumber = 1;
412 ls->lastline = 1;
413 ls->endmark = 0;
414 lex_next(ls); /* Read-ahead first char. */
415 if (ls->c == 0xef && ls->p + 2 <= ls->pe && (uint8_t)ls->p[0] == 0xbb &&
416 (uint8_t)ls->p[1] == 0xbf) { /* Skip UTF-8 BOM (if buffered). */
417 ls->p += 2;
418 lex_next(ls);
419 header = 1;
420 }
421 if (ls->c == '#') { /* Skip POSIX #! header line. */
422 do {
423 lex_next(ls);
424 if (ls->c == LEX_EOF) return 0;
425 } while (!lex_iseol(ls));
426 lex_newline(ls);
427 header = 1;
428 }
429 if (ls->c == LUA_SIGNATURE[0]) { /* Bytecode dump. */
430 if (header) {
431 /*
432 ** Loading bytecode with an extra header is disabled for security
433 ** reasons. This may circumvent the usual check for bytecode vs.
434 ** Lua code by looking at the first char. Since this is a potential
435 ** security violation no attempt is made to echo the chunkname either.
436 */
437 setstrV(L, L->top++, lj_err_str(L, LJ_ERR_BCBAD));
438 lj_err_throw(L, LUA_ERRSYNTAX);
439 }
440 return 1;
441 }
442 return 0;
443 }
444
445 /* Cleanup lexer state. */
lj_lex_cleanup(lua_State * L,LexState * ls)446 void lj_lex_cleanup(lua_State *L, LexState *ls)
447 {
448 global_State *g = G(L);
449 lj_mem_freevec(g, ls->bcstack, ls->sizebcstack, BCInsLine);
450 lj_mem_freevec(g, ls->vstack, ls->sizevstack, VarInfo);
451 lj_buf_free(g, &ls->sb);
452 }
453
454 /* Return next lexical token. */
lj_lex_next(LexState * ls)455 void lj_lex_next(LexState *ls)
456 {
457 ls->lastline = ls->linenumber;
458 if (LJ_LIKELY(ls->lookahead == TK_eof)) { /* No lookahead token? */
459 ls->tok = lex_scan(ls, &ls->tokval); /* Get next token. */
460 } else { /* Otherwise return lookahead token. */
461 ls->tok = ls->lookahead;
462 ls->lookahead = TK_eof;
463 ls->tokval = ls->lookaheadval;
464 }
465 }
466
467 /* Look ahead for the next token. */
lj_lex_lookahead(LexState * ls)468 LexToken lj_lex_lookahead(LexState *ls)
469 {
470 lj_assertLS(ls->lookahead == TK_eof, "double lookahead");
471 ls->lookahead = lex_scan(ls, &ls->lookaheadval);
472 return ls->lookahead;
473 }
474
475 /* Convert token to string. */
lj_lex_token2str(LexState * ls,LexToken tok)476 const char *lj_lex_token2str(LexState *ls, LexToken tok)
477 {
478 if (tok > TK_OFS)
479 return tokennames[tok-TK_OFS-1];
480 else if (!lj_char_iscntrl(tok))
481 return lj_strfmt_pushf(ls->L, "%c", tok);
482 else
483 return lj_strfmt_pushf(ls->L, "char(%d)", tok);
484 }
485
486 /* Lexer error. */
lj_lex_error(LexState * ls,LexToken tok,ErrMsg em,...)487 void lj_lex_error(LexState *ls, LexToken tok, ErrMsg em, ...)
488 {
489 const char *tokstr;
490 va_list argp;
491 if (tok == 0) {
492 tokstr = NULL;
493 } else if (tok == TK_name || tok == TK_string || tok == TK_number) {
494 lex_save(ls, '\0');
495 tokstr = ls->sb.b;
496 } else {
497 tokstr = lj_lex_token2str(ls, tok);
498 }
499 va_start(argp, em);
500 lj_err_lex(ls->L, ls->chunkname, tokstr, ls->linenumber, em, argp);
501 va_end(argp);
502 }
503
504 /* Initialize strings for reserved words. */
lj_lex_init(lua_State * L)505 void lj_lex_init(lua_State *L)
506 {
507 uint32_t i;
508 for (i = 0; i < TK_RESERVED; i++) {
509 GCstr *s = lj_str_newz(L, tokennames[i]);
510 fixstring(s); /* Reserved words are never collected. */
511 s->reserved = (uint8_t)(i+1);
512 }
513 }
514
515