1 // Copyright 2012 the V8 project authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #ifndef V8_PARSING_TOKEN_H_ 6 #define V8_PARSING_TOKEN_H_ 7 8 #include "src/base/bit-field.h" 9 #include "src/base/bounds.h" 10 #include "src/base/logging.h" 11 #include "src/common/globals.h" 12 13 namespace v8 { 14 namespace internal { 15 16 // TOKEN_LIST takes a list of 3 macros M, all of which satisfy the 17 // same signature M(name, string, precedence), where name is the 18 // symbolic token name, string is the corresponding syntactic symbol 19 // (or nullptr, for literals), and precedence is the precedence (or 0). 20 // The parameters are invoked for token categories as follows: 21 // 22 // T: Non-keyword tokens 23 // K: Keyword tokens 24 25 // IGNORE_TOKEN is a convenience macro that can be supplied as 26 // an argument (at any position) for a TOKEN_LIST call. It does 27 // nothing with tokens belonging to the respective category. 28 29 #define IGNORE_TOKEN(name, string, precedence) 30 31 /* Binary operators */ 32 /* ADD and SUB are at the end since they are UnaryOp */ 33 #define BINARY_OP_TOKEN_LIST(T, E) \ 34 E(T, NULLISH, "??", 3) \ 35 E(T, OR, "||", 4) \ 36 E(T, AND, "&&", 5) \ 37 E(T, BIT_OR, "|", 6) \ 38 E(T, BIT_XOR, "^", 7) \ 39 E(T, BIT_AND, "&", 8) \ 40 E(T, SHL, "<<", 11) \ 41 E(T, SAR, ">>", 11) \ 42 E(T, SHR, ">>>", 11) \ 43 E(T, MUL, "*", 13) \ 44 E(T, DIV, "/", 13) \ 45 E(T, MOD, "%", 13) \ 46 E(T, EXP, "**", 14) \ 47 E(T, ADD, "+", 12) \ 48 E(T, SUB, "-", 12) 49 50 #define EXPAND_BINOP_ASSIGN_TOKEN(T, name, string, precedence) \ 51 T(ASSIGN_##name, string "=", 2) 52 53 #define EXPAND_BINOP_TOKEN(T, name, string, precedence) \ 54 T(name, string, precedence) 55 56 #define TOKEN_LIST(T, K) \ 57 \ 58 /* BEGIN PropertyOrCall */ \ 59 /* BEGIN Member */ \ 60 /* BEGIN Template */ \ 61 /* ES6 Template Literals */ \ 62 T(TEMPLATE_SPAN, nullptr, 0) \ 63 T(TEMPLATE_TAIL, nullptr, 0) \ 64 /* END Template */ \ 65 \ 66 /* Punctuators (ECMA-262, section 7.7, page 15). */ \ 67 /* BEGIN Property */ \ 68 T(PERIOD, ".", 0) \ 69 T(LBRACK, "[", 0) \ 70 /* END Property */ \ 71 /* END Member */ \ 72 T(QUESTION_PERIOD, "?.", 0) \ 73 T(LPAREN, "(", 0) \ 74 /* END PropertyOrCall */ \ 75 T(RPAREN, ")", 0) \ 76 T(RBRACK, "]", 0) \ 77 T(LBRACE, "{", 0) \ 78 T(COLON, ":", 0) \ 79 T(ELLIPSIS, "...", 0) \ 80 T(CONDITIONAL, "?", 3) \ 81 /* BEGIN AutoSemicolon */ \ 82 T(SEMICOLON, ";", 0) \ 83 T(RBRACE, "}", 0) \ 84 /* End of source indicator. */ \ 85 T(EOS, "EOS", 0) \ 86 /* END AutoSemicolon */ \ 87 \ 88 /* BEGIN ArrowOrAssignmentOp */ \ 89 T(ARROW, "=>", 0) \ 90 /* BEGIN AssignmentOp */ \ 91 /* IsAssignmentOp() relies on this block of enum values being */ \ 92 /* contiguous and sorted in the same order! */ \ 93 T(INIT, "=init", 2) /* AST-use only. */ \ 94 T(ASSIGN, "=", 2) \ 95 BINARY_OP_TOKEN_LIST(T, EXPAND_BINOP_ASSIGN_TOKEN) \ 96 /* END AssignmentOp */ \ 97 /* END ArrowOrAssignmentOp */ \ 98 \ 99 /* Binary operators sorted by precedence. */ \ 100 /* IsBinaryOp() relies on this block of enum values */ \ 101 /* being contiguous and sorted in the same order! */ \ 102 T(COMMA, ",", 1) \ 103 \ 104 /* Unary operators, starting at ADD in BINARY_OP_TOKEN_LIST */ \ 105 /* IsUnaryOp() relies on this block of enum values */ \ 106 /* being contiguous and sorted in the same order! */ \ 107 BINARY_OP_TOKEN_LIST(T, EXPAND_BINOP_TOKEN) \ 108 \ 109 T(NOT, "!", 0) \ 110 T(BIT_NOT, "~", 0) \ 111 K(DELETE, "delete", 0) \ 112 K(TYPEOF, "typeof", 0) \ 113 K(VOID, "void", 0) \ 114 \ 115 /* BEGIN IsCountOp */ \ 116 T(INC, "++", 0) \ 117 T(DEC, "--", 0) \ 118 /* END IsCountOp */ \ 119 /* END IsUnaryOrCountOp */ \ 120 \ 121 /* Compare operators sorted by precedence. */ \ 122 /* IsCompareOp() relies on this block of enum values */ \ 123 /* being contiguous and sorted in the same order! */ \ 124 T(EQ, "==", 9) \ 125 T(EQ_STRICT, "===", 9) \ 126 T(NE, "!=", 9) \ 127 T(NE_STRICT, "!==", 9) \ 128 T(LT, "<", 10) \ 129 T(GT, ">", 10) \ 130 T(LTE, "<=", 10) \ 131 T(GTE, ">=", 10) \ 132 K(INSTANCEOF, "instanceof", 10) \ 133 K(IN, "in", 10) \ 134 \ 135 /* Keywords (ECMA-262, section 7.5.2, page 13). */ \ 136 K(BREAK, "break", 0) \ 137 K(CASE, "case", 0) \ 138 K(CATCH, "catch", 0) \ 139 K(CONTINUE, "continue", 0) \ 140 K(DEBUGGER, "debugger", 0) \ 141 K(DEFAULT, "default", 0) \ 142 /* DELETE */ \ 143 K(DO, "do", 0) \ 144 K(ELSE, "else", 0) \ 145 K(FINALLY, "finally", 0) \ 146 K(FOR, "for", 0) \ 147 K(FUNCTION, "function", 0) \ 148 K(IF, "if", 0) \ 149 /* IN */ \ 150 /* INSTANCEOF */ \ 151 K(NEW, "new", 0) \ 152 K(RETURN, "return", 0) \ 153 K(SWITCH, "switch", 0) \ 154 K(THROW, "throw", 0) \ 155 K(TRY, "try", 0) \ 156 /* TYPEOF */ \ 157 K(VAR, "var", 0) \ 158 /* VOID */ \ 159 K(WHILE, "while", 0) \ 160 K(WITH, "with", 0) \ 161 K(THIS, "this", 0) \ 162 \ 163 /* Literals (ECMA-262, section 7.8, page 16). */ \ 164 K(NULL_LITERAL, "null", 0) \ 165 K(TRUE_LITERAL, "true", 0) \ 166 K(FALSE_LITERAL, "false", 0) \ 167 T(NUMBER, nullptr, 0) \ 168 T(SMI, nullptr, 0) \ 169 T(BIGINT, nullptr, 0) \ 170 T(STRING, nullptr, 0) \ 171 \ 172 /* BEGIN Callable */ \ 173 K(SUPER, "super", 0) \ 174 /* BEGIN AnyIdentifier */ \ 175 /* Identifiers (not keywords or future reserved words). */ \ 176 T(IDENTIFIER, nullptr, 0) \ 177 K(GET, "get", 0) \ 178 K(SET, "set", 0) \ 179 K(ASYNC, "async", 0) \ 180 /* `await` is a reserved word in module code only */ \ 181 K(AWAIT, "await", 0) \ 182 K(YIELD, "yield", 0) \ 183 K(LET, "let", 0) \ 184 K(STATIC, "static", 0) \ 185 /* Future reserved words (ECMA-262, section 7.6.1.2). */ \ 186 T(FUTURE_STRICT_RESERVED_WORD, nullptr, 0) \ 187 T(ESCAPED_STRICT_RESERVED_WORD, nullptr, 0) \ 188 /* END AnyIdentifier */ \ 189 /* END Callable */ \ 190 K(ENUM, "enum", 0) \ 191 K(CLASS, "class", 0) \ 192 K(CONST, "const", 0) \ 193 K(EXPORT, "export", 0) \ 194 K(EXTENDS, "extends", 0) \ 195 K(IMPORT, "import", 0) \ 196 T(PRIVATE_NAME, nullptr, 0) \ 197 \ 198 /* Illegal token - not able to scan. */ \ 199 T(ILLEGAL, "ILLEGAL", 0) \ 200 T(ESCAPED_KEYWORD, nullptr, 0) \ 201 \ 202 /* Scanner-internal use only. */ \ 203 T(WHITESPACE, nullptr, 0) \ 204 T(UNINITIALIZED, nullptr, 0) \ 205 T(REGEXP_LITERAL, nullptr, 0) 206 207 class V8_EXPORT_PRIVATE Token { 208 public: 209 // All token values. 210 #define T(name, string, precedence) name, 211 enum Value : uint8_t { TOKEN_LIST(T, T) NUM_TOKENS }; 212 #undef T 213 214 // Returns a string corresponding to the C++ token name 215 // (e.g. "LT" for the token LT). Name(Value token)216 static const char* Name(Value token) { 217 DCHECK_GT(NUM_TOKENS, token); // token is unsigned 218 return name_[token]; 219 } 220 221 using IsKeywordBits = base::BitField8<bool, 0, 1>; 222 using IsPropertyNameBits = IsKeywordBits::Next<bool, 1>; 223 224 // Predicates IsKeyword(Value token)225 static bool IsKeyword(Value token) { 226 return IsKeywordBits::decode(token_flags[token]); 227 } 228 IsPropertyName(Value token)229 static bool IsPropertyName(Value token) { 230 return IsPropertyNameBits::decode(token_flags[token]); 231 } 232 IsValidIdentifier(Value token,LanguageMode language_mode,bool is_generator,bool disallow_await)233 V8_INLINE static bool IsValidIdentifier(Value token, 234 LanguageMode language_mode, 235 bool is_generator, 236 bool disallow_await) { 237 if (V8_LIKELY(base::IsInRange(token, IDENTIFIER, ASYNC))) return true; 238 if (token == AWAIT) return !disallow_await; 239 if (token == YIELD) return !is_generator && is_sloppy(language_mode); 240 return IsStrictReservedWord(token) && is_sloppy(language_mode); 241 } 242 IsCallable(Value token)243 static bool IsCallable(Value token) { 244 return base::IsInRange(token, SUPER, ESCAPED_STRICT_RESERVED_WORD); 245 } 246 IsAutoSemicolon(Value token)247 static bool IsAutoSemicolon(Value token) { 248 return base::IsInRange(token, SEMICOLON, EOS); 249 } 250 IsAnyIdentifier(Value token)251 static bool IsAnyIdentifier(Value token) { 252 return base::IsInRange(token, IDENTIFIER, ESCAPED_STRICT_RESERVED_WORD); 253 } 254 IsStrictReservedWord(Value token)255 static bool IsStrictReservedWord(Value token) { 256 return base::IsInRange(token, YIELD, ESCAPED_STRICT_RESERVED_WORD); 257 } 258 IsLiteral(Value token)259 static bool IsLiteral(Value token) { 260 return base::IsInRange(token, NULL_LITERAL, STRING); 261 } 262 IsTemplate(Value token)263 static bool IsTemplate(Value token) { 264 return base::IsInRange(token, TEMPLATE_SPAN, TEMPLATE_TAIL); 265 } 266 IsMember(Value token)267 static bool IsMember(Value token) { 268 return base::IsInRange(token, TEMPLATE_SPAN, LBRACK); 269 } 270 IsProperty(Value token)271 static bool IsProperty(Value token) { 272 return base::IsInRange(token, PERIOD, LBRACK); 273 } 274 IsPropertyOrCall(Value token)275 static bool IsPropertyOrCall(Value token) { 276 return base::IsInRange(token, TEMPLATE_SPAN, LPAREN); 277 } 278 IsArrowOrAssignmentOp(Value token)279 static bool IsArrowOrAssignmentOp(Value token) { 280 return base::IsInRange(token, ARROW, ASSIGN_SUB); 281 } 282 IsAssignmentOp(Value token)283 static bool IsAssignmentOp(Value token) { 284 return base::IsInRange(token, INIT, ASSIGN_SUB); 285 } 286 IsLogicalAssignmentOp(Value token)287 static bool IsLogicalAssignmentOp(Value token) { 288 return base::IsInRange(token, ASSIGN_NULLISH, ASSIGN_AND); 289 } 290 IsBinaryOp(Value op)291 static bool IsBinaryOp(Value op) { return base::IsInRange(op, COMMA, SUB); } 292 IsCompareOp(Value op)293 static bool IsCompareOp(Value op) { return base::IsInRange(op, EQ, IN); } 294 IsOrderedRelationalCompareOp(Value op)295 static bool IsOrderedRelationalCompareOp(Value op) { 296 return base::IsInRange(op, LT, GTE); 297 } 298 IsEqualityOp(Value op)299 static bool IsEqualityOp(Value op) { 300 return base::IsInRange(op, EQ, EQ_STRICT); 301 } 302 BinaryOpForAssignment(Value op)303 static Value BinaryOpForAssignment(Value op) { 304 DCHECK(base::IsInRange(op, ASSIGN_NULLISH, ASSIGN_SUB)); 305 Value result = static_cast<Value>(op - ASSIGN_NULLISH + NULLISH); 306 DCHECK(IsBinaryOp(result)); 307 return result; 308 } 309 IsBitOp(Value op)310 static bool IsBitOp(Value op) { 311 return base::IsInRange(op, BIT_OR, SHR) || op == BIT_NOT; 312 } 313 IsUnaryOp(Value op)314 static bool IsUnaryOp(Value op) { return base::IsInRange(op, ADD, VOID); } IsCountOp(Value op)315 static bool IsCountOp(Value op) { return base::IsInRange(op, INC, DEC); } IsUnaryOrCountOp(Value op)316 static bool IsUnaryOrCountOp(Value op) { 317 return base::IsInRange(op, ADD, DEC); 318 } IsShiftOp(Value op)319 static bool IsShiftOp(Value op) { return base::IsInRange(op, SHL, SHR); } 320 321 // Returns a string corresponding to the JS token string 322 // (.e., "<" for the token LT) or nullptr if the token doesn't 323 // have a (unique) string (e.g. an IDENTIFIER). String(Value token)324 static const char* String(Value token) { 325 DCHECK_GT(NUM_TOKENS, token); // token is unsigned 326 return string_[token]; 327 } 328 StringLength(Value token)329 static uint8_t StringLength(Value token) { 330 DCHECK_GT(NUM_TOKENS, token); // token is unsigned 331 return string_length_[token]; 332 } 333 334 // Returns the precedence > 0 for binary and compare 335 // operators; returns 0 otherwise. Precedence(Value token,bool accept_IN)336 static int Precedence(Value token, bool accept_IN) { 337 DCHECK_GT(NUM_TOKENS, token); // token is unsigned 338 return precedence_[accept_IN][token]; 339 } 340 341 private: 342 static const char* const name_[NUM_TOKENS]; 343 static const char* const string_[NUM_TOKENS]; 344 static const uint8_t string_length_[NUM_TOKENS]; 345 static const int8_t precedence_[2][NUM_TOKENS]; 346 static const uint8_t token_flags[NUM_TOKENS]; 347 }; 348 349 } // namespace internal 350 } // namespace v8 351 352 #endif // V8_PARSING_TOKEN_H_ 353