1 /* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- 2 * 3 * ***** BEGIN LICENSE BLOCK ***** 4 * Version: MPL 1.1/GPL 2.0/LGPL 2.1 5 * 6 * The contents of this file are subject to the Mozilla Public License Version 7 * 1.1 (the "License"); you may not use this file except in compliance with 8 * the License. You may obtain a copy of the License at 9 * http://www.mozilla.org/MPL/ 10 * 11 * Software distributed under the License is distributed on an "AS IS" basis, 12 * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License 13 * for the specific language governing rights and limitations under the 14 * License. 15 * 16 * The Original Code is Mozilla Communicator client code, released 17 * March 31, 1998. 18 * 19 * The Initial Developer of the Original Code is 20 * Netscape Communications Corporation. 21 * Portions created by the Initial Developer are Copyright (C) 1998 22 * the Initial Developer. All Rights Reserved. 23 * 24 * Contributor(s): 25 * 26 * Alternatively, the contents of this file may be used under the terms of 27 * either of the GNU General Public License Version 2 or later (the "GPL"), 28 * or the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), 29 * in which case the provisions of the GPL or the LGPL are applicable instead 30 * of those above. If you wish to allow use of your version of this file only 31 * under the terms of either the GPL or the LGPL, and not to allow others to 32 * use your version of this file under the terms of the MPL, indicate your 33 * decision by deleting the provisions above and replace them with the notice 34 * and other provisions required by the GPL or the LGPL. If you do not delete 35 * the provisions above, a recipient may use your version of this file under 36 * the terms of any one of the MPL, the GPL or the LGPL. 37 * 38 * ***** END LICENSE BLOCK ***** */ 39 40 #ifndef jsscan_h___ 41 #define jsscan_h___ 42 /* 43 * JS lexical scanner interface. 44 */ 45 #include <stddef.h> 46 #include <stdio.h> 47 #include "jsconfig.h" 48 #include "jsopcode.h" 49 #include "jsprvtd.h" 50 #include "jspubtd.h" 51 52 JS_BEGIN_EXTERN_C 53 54 #define JS_KEYWORD(keyword, type, op, version) \ 55 extern const char js_##keyword##_str[]; 56 #include "jskeyword.tbl" 57 #undef JS_KEYWORD 58 59 typedef enum JSTokenType { 60 TOK_ERROR = -1, /* well-known as the only code < EOF */ 61 TOK_EOF = 0, /* end of file */ 62 TOK_EOL = 1, /* end of line */ 63 TOK_SEMI = 2, /* semicolon */ 64 TOK_COMMA = 3, /* comma operator */ 65 TOK_ASSIGN = 4, /* assignment ops (= += -= etc.) */ 66 TOK_HOOK = 5, TOK_COLON = 6, /* conditional (?:) */ 67 TOK_OR = 7, /* logical or (||) */ 68 TOK_AND = 8, /* logical and (&&) */ 69 TOK_BITOR = 9, /* bitwise-or (|) */ 70 TOK_BITXOR = 10, /* bitwise-xor (^) */ 71 TOK_BITAND = 11, /* bitwise-and (&) */ 72 TOK_EQOP = 12, /* equality ops (== !=) */ 73 TOK_RELOP = 13, /* relational ops (< <= > >=) */ 74 TOK_SHOP = 14, /* shift ops (<< >> >>>) */ 75 TOK_PLUS = 15, /* plus */ 76 TOK_MINUS = 16, /* minus */ 77 TOK_STAR = 17, TOK_DIVOP = 18, /* multiply/divide ops (* / %) */ 78 TOK_UNARYOP = 19, /* unary prefix operator */ 79 TOK_INC = 20, TOK_DEC = 21, /* increment/decrement (++ --) */ 80 TOK_DOT = 22, /* member operator (.) */ 81 TOK_LB = 23, TOK_RB = 24, /* left and right brackets */ 82 TOK_LC = 25, TOK_RC = 26, /* left and right curlies (braces) */ 83 TOK_LP = 27, TOK_RP = 28, /* left and right parentheses */ 84 TOK_NAME = 29, /* identifier */ 85 TOK_NUMBER = 30, /* numeric constant */ 86 TOK_STRING = 31, /* string constant */ 87 TOK_OBJECT = 32, /* RegExp or other object constant */ 88 TOK_PRIMARY = 33, /* true, false, null, this, super */ 89 TOK_FUNCTION = 34, /* function keyword */ 90 TOK_EXPORT = 35, /* export keyword */ 91 TOK_IMPORT = 36, /* import keyword */ 92 TOK_IF = 37, /* if keyword */ 93 TOK_ELSE = 38, /* else keyword */ 94 TOK_SWITCH = 39, /* switch keyword */ 95 TOK_CASE = 40, /* case keyword */ 96 TOK_DEFAULT = 41, /* default keyword */ 97 TOK_WHILE = 42, /* while keyword */ 98 TOK_DO = 43, /* do keyword */ 99 TOK_FOR = 44, /* for keyword */ 100 TOK_BREAK = 45, /* break keyword */ 101 TOK_CONTINUE = 46, /* continue keyword */ 102 TOK_IN = 47, /* in keyword */ 103 TOK_VAR = 48, /* var keyword */ 104 TOK_WITH = 49, /* with keyword */ 105 TOK_RETURN = 50, /* return keyword */ 106 TOK_NEW = 51, /* new keyword */ 107 TOK_DELETE = 52, /* delete keyword */ 108 TOK_DEFSHARP = 53, /* #n= for object/array initializers */ 109 TOK_USESHARP = 54, /* #n# for object/array initializers */ 110 TOK_TRY = 55, /* try keyword */ 111 TOK_CATCH = 56, /* catch keyword */ 112 TOK_FINALLY = 57, /* finally keyword */ 113 TOK_THROW = 58, /* throw keyword */ 114 TOK_INSTANCEOF = 59, /* instanceof keyword */ 115 TOK_DEBUGGER = 60, /* debugger keyword */ 116 TOK_XMLSTAGO = 61, /* XML start tag open (<) */ 117 TOK_XMLETAGO = 62, /* XML end tag open (</) */ 118 TOK_XMLPTAGC = 63, /* XML point tag close (/>) */ 119 TOK_XMLTAGC = 64, /* XML start or end tag close (>) */ 120 TOK_XMLNAME = 65, /* XML start-tag non-final fragment */ 121 TOK_XMLATTR = 66, /* XML quoted attribute value */ 122 TOK_XMLSPACE = 67, /* XML whitespace */ 123 TOK_XMLTEXT = 68, /* XML text */ 124 TOK_XMLCOMMENT = 69, /* XML comment */ 125 TOK_XMLCDATA = 70, /* XML CDATA section */ 126 TOK_XMLPI = 71, /* XML processing instruction */ 127 TOK_AT = 72, /* XML attribute op (@) */ 128 TOK_DBLCOLON = 73, /* namespace qualified name op (::) */ 129 TOK_ANYNAME = 74, /* XML AnyName singleton (*) */ 130 TOK_DBLDOT = 75, /* XML descendant op (..) */ 131 TOK_FILTER = 76, /* XML filtering predicate op (.()) */ 132 TOK_XMLELEM = 77, /* XML element node type (no token) */ 133 TOK_XMLLIST = 78, /* XML list node type (no token) */ 134 TOK_YIELD = 79, /* yield from generator function */ 135 TOK_ARRAYCOMP = 80, /* array comprehension initialiser */ 136 TOK_ARRAYPUSH = 81, /* array push within comprehension */ 137 TOK_LEXICALSCOPE = 82, /* block scope AST node label */ 138 TOK_LET = 83, /* let keyword */ 139 TOK_BODY = 84, /* synthetic body of function with 140 destructuring formal parameters */ 141 TOK_RESERVED, /* reserved keywords */ 142 TOK_LIMIT /* domain size */ 143 } JSTokenType; 144 145 #define IS_PRIMARY_TOKEN(tt) \ 146 ((uintN)((tt) - TOK_NAME) <= (uintN)(TOK_PRIMARY - TOK_NAME)) 147 148 #define TOKEN_TYPE_IS_XML(tt) \ 149 (tt == TOK_AT || tt == TOK_DBLCOLON || tt == TOK_ANYNAME) 150 151 #if JS_HAS_BLOCK_SCOPE 152 # define TOKEN_TYPE_IS_DECL(tt) ((tt) == TOK_VAR || (tt) == TOK_LET) 153 #else 154 # define TOKEN_TYPE_IS_DECL(tt) ((tt) == TOK_VAR) 155 #endif 156 157 struct JSStringBuffer { 158 jschar *base; 159 jschar *limit; /* length limit for quick bounds check */ 160 jschar *ptr; /* slot for next non-NUL char to store */ 161 void *data; 162 JSBool (*grow)(JSStringBuffer *sb, size_t newlength); 163 void (*free)(JSStringBuffer *sb); 164 }; 165 166 #define STRING_BUFFER_ERROR_BASE ((jschar *) 1) 167 #define STRING_BUFFER_OK(sb) ((sb)->base != STRING_BUFFER_ERROR_BASE) 168 #define STRING_BUFFER_OFFSET(sb) ((sb)->ptr -(sb)->base) 169 170 extern void 171 js_InitStringBuffer(JSStringBuffer *sb); 172 173 extern void 174 js_FinishStringBuffer(JSStringBuffer *sb); 175 176 extern void 177 js_AppendChar(JSStringBuffer *sb, jschar c); 178 179 extern void 180 js_RepeatChar(JSStringBuffer *sb, jschar c, uintN count); 181 182 extern void 183 js_AppendCString(JSStringBuffer *sb, const char *asciiz); 184 185 extern void 186 js_AppendJSString(JSStringBuffer *sb, JSString *str); 187 188 struct JSTokenPtr { 189 uint16 index; /* index of char in physical line */ 190 uint16 lineno; /* physical line number */ 191 }; 192 193 struct JSTokenPos { 194 JSTokenPtr begin; /* first character and line of token */ 195 JSTokenPtr end; /* index 1 past last char, last line */ 196 }; 197 198 struct JSToken { 199 JSTokenType type; /* char value or above enumerator */ 200 JSTokenPos pos; /* token position in file */ 201 jschar *ptr; /* beginning of token in line buffer */ 202 union { 203 struct { /* non-numeric literal */ 204 JSOp op; /* operator, for minimal parser */ 205 JSAtom *atom; /* atom table entry */ 206 } s; 207 struct { /* atom pair, for XML PIs */ 208 JSAtom *atom2; /* auxiliary atom table entry */ 209 JSAtom *atom; /* main atom table entry */ 210 } p; 211 jsdouble dval; /* floating point number */ 212 } u; 213 }; 214 215 #define t_op u.s.op 216 #define t_atom u.s.atom 217 #define t_atom2 u.p.atom2 218 #define t_dval u.dval 219 220 typedef struct JSTokenBuf { 221 jschar *base; /* base of line or stream buffer */ 222 jschar *limit; /* limit for quick bounds check */ 223 jschar *ptr; /* next char to get, or slot to use */ 224 } JSTokenBuf; 225 226 #define JS_LINE_LIMIT 256 /* logical line buffer size limit -- 227 physical line length is unlimited */ 228 #define NTOKENS 4 /* 1 current + 2 lookahead, rounded */ 229 #define NTOKENS_MASK (NTOKENS-1) /* to power of 2 to avoid divmod by 3 */ 230 231 struct JSTokenStream { 232 JSToken tokens[NTOKENS];/* circular token buffer */ 233 uintN cursor; /* index of last parsed token */ 234 uintN lookahead; /* count of lookahead tokens */ 235 uintN lineno; /* current line number */ 236 uintN ungetpos; /* next free char slot in ungetbuf */ 237 jschar ungetbuf[6]; /* at most 6, for \uXXXX lookahead */ 238 uintN flags; /* flags -- see below */ 239 ptrdiff_t linelen; /* physical linebuf segment length */ 240 ptrdiff_t linepos; /* linebuf offset in physical line */ 241 JSTokenBuf linebuf; /* line buffer for diagnostics */ 242 JSTokenBuf userbuf; /* user input buffer if !file */ 243 JSStringBuffer tokenbuf; /* current token string buffer */ 244 const char *filename; /* input filename or null */ 245 FILE *file; /* stdio stream if reading from file */ 246 JSPrincipals *principals; /* principals associated with source */ 247 JSSourceHandler listener; /* callback for source; eg debugger */ 248 void *listenerData; /* listener 'this' data */ 249 void *listenerTSData;/* listener data for this TokenStream */ 250 jschar *saveEOL; /* save next end of line in userbuf, to 251 optimize for very long lines */ 252 }; 253 254 #define CURRENT_TOKEN(ts) ((ts)->tokens[(ts)->cursor]) 255 #define ON_CURRENT_LINE(ts,pos) ((uint16)(ts)->lineno == (pos).end.lineno) 256 257 /* JSTokenStream flags */ 258 #define TSF_ERROR 0x01 /* fatal error while compiling */ 259 #define TSF_EOF 0x02 /* hit end of file */ 260 #define TSF_NEWLINES 0x04 /* tokenize newlines */ 261 #define TSF_OPERAND 0x08 /* looking for operand, not operator */ 262 #define TSF_NLFLAG 0x20 /* last linebuf ended with \n */ 263 #define TSF_CRFLAG 0x40 /* linebuf would have ended with \r */ 264 #define TSF_DIRTYLINE 0x80 /* non-whitespace since start of line */ 265 #define TSF_OWNFILENAME 0x100 /* ts->filename is malloc'd */ 266 #define TSF_XMLTAGMODE 0x200 /* scanning within an XML tag in E4X */ 267 #define TSF_XMLTEXTMODE 0x400 /* scanning XMLText terminal from E4X */ 268 #define TSF_XMLONLYMODE 0x800 /* don't scan {expr} within text/tag */ 269 270 /* Flag indicating unexpected end of input, i.e. TOK_EOF not at top-level. */ 271 #define TSF_UNEXPECTED_EOF 0x1000 272 273 /* 274 * To handle the hard case of contiguous HTML comments, we want to clear the 275 * TSF_DIRTYINPUT flag at the end of each such comment. But we'd rather not 276 * scan for --> within every //-style comment unless we have to. So we set 277 * TSF_IN_HTML_COMMENT when a <!-- is scanned as an HTML begin-comment, and 278 * clear it (and TSF_DIRTYINPUT) when we scan --> either on a clean line, or 279 * only if (ts->flags & TSF_IN_HTML_COMMENT), in a //-style comment. 280 * 281 * This still works as before given a malformed comment hiding hack such as: 282 * 283 * <script> 284 * <!-- comment hiding hack #1 285 * code goes here 286 * // --> oops, markup for script-unaware browsers goes here! 287 * </script> 288 * 289 * It does not cope with malformed comment hiding hacks where --> is hidden 290 * by C-style comments, or on a dirty line. Such cases are already broken. 291 */ 292 #define TSF_IN_HTML_COMMENT 0x2000 293 294 /* Ignore keywords and return TOK_NAME instead to the parser. */ 295 #define TSF_KEYWORD_IS_NAME 0x4000 296 297 /* Unicode separators that are treated as line terminators, in addition to \n, \r */ 298 #define LINE_SEPARATOR 0x2028 299 #define PARA_SEPARATOR 0x2029 300 301 /* 302 * Create a new token stream, either from an input buffer or from a file. 303 * Return null on file-open or memory-allocation failure. 304 * 305 * NB: All of js_New{,Buffer,File}TokenStream() return a pointer to transient 306 * memory in the current context's temp pool. This memory is deallocated via 307 * JS_ARENA_RELEASE() after parsing is finished. 308 */ 309 extern JSTokenStream * 310 js_NewTokenStream(JSContext *cx, const jschar *base, size_t length, 311 const char *filename, uintN lineno, JSPrincipals *principals); 312 313 extern JS_FRIEND_API(JSTokenStream *) 314 js_NewBufferTokenStream(JSContext *cx, const jschar *base, size_t length); 315 316 extern JS_FRIEND_API(JSTokenStream *) 317 js_NewFileTokenStream(JSContext *cx, const char *filename, FILE *defaultfp); 318 319 extern JS_FRIEND_API(JSBool) 320 js_CloseTokenStream(JSContext *cx, JSTokenStream *ts); 321 322 extern JS_FRIEND_API(int) 323 js_fgets(char *buf, int size, FILE *file); 324 325 /* 326 * If the given char array forms JavaScript keyword, return corresponding 327 * token. Otherwise return TOK_EOF. 328 */ 329 extern JSTokenType 330 js_CheckKeyword(const jschar *chars, size_t length); 331 332 #define js_IsKeyword(chars, length) \ 333 (js_CheckKeyword(chars, length) != TOK_EOF) 334 335 /* 336 * Friend-exported API entry point to call a mapping function on each reserved 337 * identifier in the scanner's keyword table. 338 */ 339 extern JS_FRIEND_API(void) 340 js_MapKeywords(void (*mapfun)(const char *)); 341 342 /* 343 * Report a compile-time error by its number, using ts or cg to show context. 344 * Return true for a warning, false for an error. 345 */ 346 extern JSBool 347 js_ReportCompileErrorNumber(JSContext *cx, void *handle, uintN flags, 348 uintN errorNumber, ...); 349 350 extern JSBool 351 js_ReportCompileErrorNumberUC(JSContext *cx, void *handle, uintN flags, 352 uintN errorNumber, ...); 353 354 /* Steal some JSREPORT_* bits (see jsapi.h) to tell handle's type. */ 355 #define JSREPORT_HANDLE 0x300 356 #define JSREPORT_TS 0x000 357 #define JSREPORT_CG 0x100 358 #define JSREPORT_PN 0x200 359 360 /* 361 * Look ahead one token and return its type. 362 */ 363 extern JSTokenType 364 js_PeekToken(JSContext *cx, JSTokenStream *ts); 365 366 extern JSTokenType 367 js_PeekTokenSameLine(JSContext *cx, JSTokenStream *ts); 368 369 /* 370 * Get the next token from ts. 371 */ 372 extern JSTokenType 373 js_GetToken(JSContext *cx, JSTokenStream *ts); 374 375 /* 376 * Push back the last scanned token onto ts. 377 */ 378 extern void 379 js_UngetToken(JSTokenStream *ts); 380 381 /* 382 * Get the next token from ts if its type is tt. 383 */ 384 extern JSBool 385 js_MatchToken(JSContext *cx, JSTokenStream *ts, JSTokenType tt); 386 387 JS_END_EXTERN_C 388 389 #endif /* jsscan_h___ */ 390