1 /* 2 ** $Id: llex.c,v 2.63.1.3 2015/02/09 17:56:34 roberto Exp $ 3 ** Lexical Analyzer 4 ** See Copyright Notice in lua.h 5 */ 6 7 #define llex_c 8 #define LUA_CORE 9 10 #include <sys/lua/lua.h> 11 12 #include "lctype.h" 13 #include "ldo.h" 14 #include "llex.h" 15 #include "lobject.h" 16 #include "lparser.h" 17 #include "lstate.h" 18 #include "lstring.h" 19 #include "ltable.h" 20 #include "lzio.h" 21 22 23 24 #define next(ls) (ls->current = zgetc(ls->z)) 25 26 27 28 #define currIsNewline(ls) (ls->current == '\n' || ls->current == '\r') 29 30 31 /* ORDER RESERVED */ 32 static const char *const luaX_tokens [] = { 33 "and", "break", "do", "else", "elseif", 34 "end", "false", "for", "function", "goto", "if", 35 "in", "local", "nil", "not", "or", "repeat", 36 "return", "then", "true", "until", "while", 37 "..", "...", "==", ">=", "<=", "~=", "::", "<eof>", 38 "<number>", "<name>", "<string>" 39 }; 40 41 42 #define save_and_next(ls) (save(ls, ls->current), next(ls)) 43 44 45 static l_noret lexerror (LexState *ls, const char *msg, int token); 46 47 48 static void save (LexState *ls, int c) { 49 Mbuffer *b = ls->buff; 50 if (luaZ_bufflen(b) + 1 > luaZ_sizebuffer(b)) { 51 size_t newsize; 52 if (luaZ_sizebuffer(b) >= MAX_SIZET/2) 53 lexerror(ls, "lexical element too long", 0); 54 newsize = luaZ_sizebuffer(b) * 2; 55 luaZ_resizebuffer(ls->L, b, newsize); 56 } 57 b->buffer[luaZ_bufflen(b)++] = cast(char, c); 58 } 59 60 61 void luaX_init (lua_State *L) { 62 int i; 63 for (i=0; i<NUM_RESERVED; i++) { 64 TString *ts = luaS_new(L, luaX_tokens[i]); 65 luaS_fix(ts); /* reserved words are never collected */ 66 ts->tsv.extra = cast_byte(i+1); /* reserved word */ 67 } 68 } 69 70 71 const char *luaX_token2str (LexState *ls, int token) { 72 if (token < FIRST_RESERVED) { /* single-byte symbols? */ 73 lua_assert(token == cast(unsigned char, token)); 74 return (lisprint(token)) ? luaO_pushfstring(ls->L, LUA_QL("%c"), token) : 75 luaO_pushfstring(ls->L, "char(%d)", token); 76 } 77 else { 78 const char *s = luaX_tokens[token - FIRST_RESERVED]; 79 if (token < TK_EOS) /* fixed format (symbols and reserved words)? */ 80 return luaO_pushfstring(ls->L, LUA_QS, s); 81 else /* names, strings, and numerals */ 82 return s; 83 } 84 } 85 86 87 static const char *txtToken (LexState *ls, int token) { 88 switch (token) { 89 case TK_NAME: 90 case TK_STRING: 91 case TK_NUMBER: 92 save(ls, '\0'); 93 return luaO_pushfstring(ls->L, LUA_QS, luaZ_buffer(ls->buff)); 94 default: 95 return luaX_token2str(ls, token); 96 } 97 } 98 99 100 static l_noret lexerror (LexState *ls, const char *msg, int token) { 101 char buff[LUA_IDSIZE]; 102 luaO_chunkid(buff, getstr(ls->source), LUA_IDSIZE); 103 msg = luaO_pushfstring(ls->L, "%s:%d: %s", buff, ls->linenumber, msg); 104 if (token) 105 luaO_pushfstring(ls->L, "%s near %s", msg, txtToken(ls, token)); 106 luaD_throw(ls->L, LUA_ERRSYNTAX); 107 } 108 109 110 l_noret luaX_syntaxerror (LexState *ls, const char *msg) { 111 lexerror(ls, msg, ls->t.token); 112 } 113 114 115 /* 116 ** creates a new string and anchors it in function's table so that 117 ** it will not be collected until the end of the function's compilation 118 ** (by that time it should be anchored in function's prototype) 119 */ 120 TString *luaX_newstring (LexState *ls, const char *str, size_t l) { 121 lua_State *L = ls->L; 122 TValue *o; /* entry for `str' */ 123 TString *ts = luaS_newlstr(L, str, l); /* create new string */ 124 setsvalue2s(L, L->top++, ts); /* temporarily anchor it in stack */ 125 o = luaH_set(L, ls->fs->h, L->top - 1); 126 if (ttisnil(o)) { /* not in use yet? (see 'addK') */ 127 /* boolean value does not need GC barrier; 128 table has no metatable, so it does not need to invalidate cache */ 129 setbvalue(o, 1); /* t[string] = true */ 130 luaC_checkGC(L); 131 } 132 else { /* string already present */ 133 ts = rawtsvalue(keyfromval(o)); /* re-use value previously stored */ 134 } 135 L->top--; /* remove string from stack */ 136 return ts; 137 } 138 139 140 /* 141 ** increment line number and skips newline sequence (any of 142 ** \n, \r, \n\r, or \r\n) 143 */ 144 static void inclinenumber (LexState *ls) { 145 int old = ls->current; 146 lua_assert(currIsNewline(ls)); 147 next(ls); /* skip `\n' or `\r' */ 148 if (currIsNewline(ls) && ls->current != old) 149 next(ls); /* skip `\n\r' or `\r\n' */ 150 if (++ls->linenumber >= MAX_INT) 151 lexerror(ls, "chunk has too many lines", 0); 152 } 153 154 155 void luaX_setinput (lua_State *L, LexState *ls, ZIO *z, TString *source, 156 int firstchar) { 157 ls->decpoint = '.'; 158 ls->L = L; 159 ls->current = firstchar; 160 ls->lookahead.token = TK_EOS; /* no look-ahead token */ 161 ls->z = z; 162 ls->fs = NULL; 163 ls->linenumber = 1; 164 ls->lastline = 1; 165 ls->source = source; 166 ls->envn = luaS_new(L, LUA_ENV); /* create env name */ 167 luaS_fix(ls->envn); /* never collect this name */ 168 luaZ_resizebuffer(ls->L, ls->buff, LUA_MINBUFFER); /* initialize buffer */ 169 } 170 171 172 173 /* 174 ** ======================================================= 175 ** LEXICAL ANALYZER 176 ** ======================================================= 177 */ 178 179 180 181 static int check_next (LexState *ls, const char *set) { 182 if (ls->current == '\0' || !strchr(set, ls->current)) 183 return 0; 184 save_and_next(ls); 185 return 1; 186 } 187 188 189 /* 190 ** change all characters 'from' in buffer to 'to' 191 */ 192 static void buffreplace (LexState *ls, char from, char to) { 193 size_t n = luaZ_bufflen(ls->buff); 194 char *p = luaZ_buffer(ls->buff); 195 while (n--) 196 if (p[n] == from) p[n] = to; 197 } 198 199 200 #if !defined(getlocaledecpoint) 201 #define getlocaledecpoint() (localeconv()->decimal_point[0]) 202 #endif 203 204 205 #define buff2d(b,e) luaO_str2d(luaZ_buffer(b), luaZ_bufflen(b) - 1, e) 206 207 /* 208 ** in case of format error, try to change decimal point separator to 209 ** the one defined in the current locale and check again 210 */ 211 static void trydecpoint (LexState *ls, SemInfo *seminfo) { 212 char old = ls->decpoint; 213 ls->decpoint = getlocaledecpoint(); 214 buffreplace(ls, old, ls->decpoint); /* try new decimal separator */ 215 if (!buff2d(ls->buff, &seminfo->r)) { 216 /* format error with correct decimal point: no more options */ 217 buffreplace(ls, ls->decpoint, '.'); /* undo change (for error message) */ 218 lexerror(ls, "malformed number", TK_NUMBER); 219 } 220 } 221 222 223 /* LUA_NUMBER */ 224 /* 225 ** this function is quite liberal in what it accepts, as 'luaO_str2d' 226 ** will reject ill-formed numerals. 227 */ 228 static void read_numeral (LexState *ls, SemInfo *seminfo) { 229 const char *expo = "Ee"; 230 int first = ls->current; 231 lua_assert(lisdigit(ls->current)); 232 save_and_next(ls); 233 if (first == '0' && check_next(ls, "Xx")) /* hexadecimal? */ 234 expo = "Pp"; 235 for (;;) { 236 if (check_next(ls, expo)) /* exponent part? */ 237 (void) check_next(ls, "+-"); /* optional exponent sign */ 238 if (lisxdigit(ls->current) || ls->current == '.') 239 save_and_next(ls); 240 else break; 241 } 242 save(ls, '\0'); 243 buffreplace(ls, '.', ls->decpoint); /* follow locale for decimal point */ 244 if (!buff2d(ls->buff, &seminfo->r)) /* format error? */ 245 trydecpoint(ls, seminfo); /* try to update decimal point separator */ 246 } 247 248 249 /* 250 ** skip a sequence '[=*[' or ']=*]' and return its number of '='s or 251 ** -1 if sequence is malformed 252 */ 253 static int skip_sep (LexState *ls) { 254 int count = 0; 255 int s = ls->current; 256 lua_assert(s == '[' || s == ']'); 257 save_and_next(ls); 258 while (ls->current == '=') { 259 save_and_next(ls); 260 count++; 261 } 262 return (ls->current == s) ? count : (-count) - 1; 263 } 264 265 266 static void read_long_string (LexState *ls, SemInfo *seminfo, int sep) { 267 save_and_next(ls); /* skip 2nd `[' */ 268 if (currIsNewline(ls)) /* string starts with a newline? */ 269 inclinenumber(ls); /* skip it */ 270 for (;;) { 271 switch (ls->current) { 272 case EOZ: 273 lexerror(ls, (seminfo) ? "unfinished long string" : 274 "unfinished long comment", TK_EOS); 275 break; /* to avoid warnings */ 276 case ']': { 277 if (skip_sep(ls) == sep) { 278 save_and_next(ls); /* skip 2nd `]' */ 279 goto endloop; 280 } 281 break; 282 } 283 case '\n': case '\r': { 284 save(ls, '\n'); 285 inclinenumber(ls); 286 if (!seminfo) luaZ_resetbuffer(ls->buff); /* avoid wasting space */ 287 break; 288 } 289 default: { 290 if (seminfo) save_and_next(ls); 291 else next(ls); 292 } 293 } 294 } endloop: 295 if (seminfo) 296 seminfo->ts = luaX_newstring(ls, luaZ_buffer(ls->buff) + (2 + sep), 297 luaZ_bufflen(ls->buff) - 2*(2 + sep)); 298 } 299 300 301 static void escerror (LexState *ls, int *c, int n, const char *msg) { 302 int i; 303 luaZ_resetbuffer(ls->buff); /* prepare error message */ 304 save(ls, '\\'); 305 for (i = 0; i < n && c[i] != EOZ; i++) 306 save(ls, c[i]); 307 lexerror(ls, msg, TK_STRING); 308 } 309 310 311 static int readhexaesc (LexState *ls) { 312 int c[3], i; /* keep input for error message */ 313 int r = 0; /* result accumulator */ 314 c[0] = 'x'; /* for error message */ 315 for (i = 1; i < 3; i++) { /* read two hexadecimal digits */ 316 c[i] = next(ls); 317 if (!lisxdigit(c[i])) 318 escerror(ls, c, i + 1, "hexadecimal digit expected"); 319 r = (r << 4) + luaO_hexavalue(c[i]); 320 } 321 return r; 322 } 323 324 325 static int readdecesc (LexState *ls) { 326 int c[3], i; 327 int r = 0; /* result accumulator */ 328 for (i = 0; i < 3 && lisdigit(ls->current); i++) { /* read up to 3 digits */ 329 c[i] = ls->current; 330 r = 10*r + c[i] - '0'; 331 next(ls); 332 } 333 if (r > UCHAR_MAX) 334 escerror(ls, c, i, "decimal escape too large"); 335 return r; 336 } 337 338 339 static void read_string (LexState *ls, int del, SemInfo *seminfo) { 340 save_and_next(ls); /* keep delimiter (for error messages) */ 341 while (ls->current != del) { 342 switch (ls->current) { 343 case EOZ: 344 lexerror(ls, "unfinished string", TK_EOS); 345 break; /* to avoid warnings */ 346 case '\n': 347 case '\r': 348 lexerror(ls, "unfinished string", TK_STRING); 349 break; /* to avoid warnings */ 350 case '\\': { /* escape sequences */ 351 int c; /* final character to be saved */ 352 next(ls); /* do not save the `\' */ 353 switch (ls->current) { 354 case 'a': c = '\a'; goto read_save; 355 case 'b': c = '\b'; goto read_save; 356 case 'f': c = '\f'; goto read_save; 357 case 'n': c = '\n'; goto read_save; 358 case 'r': c = '\r'; goto read_save; 359 case 't': c = '\t'; goto read_save; 360 case 'v': c = '\v'; goto read_save; 361 case 'x': c = readhexaesc(ls); goto read_save; 362 case '\n': case '\r': 363 inclinenumber(ls); c = '\n'; goto only_save; 364 case '\\': case '\"': case '\'': 365 c = ls->current; goto read_save; 366 case EOZ: goto no_save; /* will raise an error next loop */ 367 case 'z': { /* zap following span of spaces */ 368 next(ls); /* skip the 'z' */ 369 while (lisspace(ls->current)) { 370 if (currIsNewline(ls)) inclinenumber(ls); 371 else next(ls); 372 } 373 goto no_save; 374 } 375 default: { 376 if (!lisdigit(ls->current)) 377 escerror(ls, &ls->current, 1, "invalid escape sequence"); 378 /* digital escape \ddd */ 379 c = readdecesc(ls); 380 goto only_save; 381 } 382 } 383 read_save: next(ls); /* read next character */ 384 only_save: save(ls, c); /* save 'c' */ 385 no_save: break; 386 } 387 default: 388 save_and_next(ls); 389 } 390 } 391 save_and_next(ls); /* skip delimiter */ 392 seminfo->ts = luaX_newstring(ls, luaZ_buffer(ls->buff) + 1, 393 luaZ_bufflen(ls->buff) - 2); 394 } 395 396 397 static int llex (LexState *ls, SemInfo *seminfo) { 398 luaZ_resetbuffer(ls->buff); 399 for (;;) { 400 switch (ls->current) { 401 case '\n': case '\r': { /* line breaks */ 402 inclinenumber(ls); 403 break; 404 } 405 case ' ': case '\f': case '\t': case '\v': { /* spaces */ 406 next(ls); 407 break; 408 } 409 case '-': { /* '-' or '--' (comment) */ 410 next(ls); 411 if (ls->current != '-') return '-'; 412 /* else is a comment */ 413 next(ls); 414 if (ls->current == '[') { /* long comment? */ 415 int sep = skip_sep(ls); 416 luaZ_resetbuffer(ls->buff); /* `skip_sep' may dirty the buffer */ 417 if (sep >= 0) { 418 read_long_string(ls, NULL, sep); /* skip long comment */ 419 luaZ_resetbuffer(ls->buff); /* previous call may dirty the buff. */ 420 break; 421 } 422 } 423 /* else short comment */ 424 while (!currIsNewline(ls) && ls->current != EOZ) 425 next(ls); /* skip until end of line (or end of file) */ 426 break; 427 } 428 case '[': { /* long string or simply '[' */ 429 int sep = skip_sep(ls); 430 if (sep >= 0) { 431 read_long_string(ls, seminfo, sep); 432 return TK_STRING; 433 } else if (sep == -1) { 434 return '['; 435 } else { 436 lexerror(ls, "invalid long string delimiter", TK_STRING); 437 break; 438 } 439 } 440 case '=': { 441 next(ls); 442 if (ls->current != '=') return '='; 443 else { next(ls); return TK_EQ; } 444 } 445 case '<': { 446 next(ls); 447 if (ls->current != '=') return '<'; 448 else { next(ls); return TK_LE; } 449 } 450 case '>': { 451 next(ls); 452 if (ls->current != '=') return '>'; 453 else { next(ls); return TK_GE; } 454 } 455 case '~': { 456 next(ls); 457 if (ls->current != '=') return '~'; 458 else { next(ls); return TK_NE; } 459 } 460 case ':': { 461 next(ls); 462 if (ls->current != ':') return ':'; 463 else { next(ls); return TK_DBCOLON; } 464 } 465 case '"': case '\'': { /* short literal strings */ 466 read_string(ls, ls->current, seminfo); 467 return TK_STRING; 468 } 469 case '.': { /* '.', '..', '...', or number */ 470 save_and_next(ls); 471 if (check_next(ls, ".")) { 472 if (check_next(ls, ".")) 473 return TK_DOTS; /* '...' */ 474 else return TK_CONCAT; /* '..' */ 475 } 476 else if (!lisdigit(ls->current)) return '.'; 477 /* else go through */ 478 } 479 zfs_fallthrough; 480 case '0': case '1': case '2': case '3': case '4': 481 case '5': case '6': case '7': case '8': case '9': { 482 read_numeral(ls, seminfo); 483 return TK_NUMBER; 484 } 485 case EOZ: { 486 return TK_EOS; 487 } 488 default: { 489 if (lislalpha(ls->current)) { /* identifier or reserved word? */ 490 TString *ts; 491 do { 492 save_and_next(ls); 493 } while (lislalnum(ls->current)); 494 ts = luaX_newstring(ls, luaZ_buffer(ls->buff), 495 luaZ_bufflen(ls->buff)); 496 seminfo->ts = ts; 497 if (isreserved(ts)) /* reserved word? */ 498 return ts->tsv.extra - 1 + FIRST_RESERVED; 499 else { 500 return TK_NAME; 501 } 502 } 503 else { /* single-char tokens (+ - / ...) */ 504 int c = ls->current; 505 next(ls); 506 return c; 507 } 508 } 509 } 510 } 511 } 512 513 514 void luaX_next (LexState *ls) { 515 ls->lastline = ls->linenumber; 516 if (ls->lookahead.token != TK_EOS) { /* is there a look-ahead token? */ 517 ls->t = ls->lookahead; /* use this one */ 518 ls->lookahead.token = TK_EOS; /* and discharge it */ 519 } 520 else 521 ls->t.token = llex(ls, &ls->t.seminfo); /* read next token */ 522 } 523 524 525 int luaX_lookahead (LexState *ls) { 526 lua_assert(ls->lookahead.token == TK_EOS); 527 ls->lookahead.token = llex(ls, &ls->lookahead.seminfo); 528 return ls->lookahead.token; 529 } 530