1 // Copyright 2011 the V8 project authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 // Features shared by parsing and pre-parsing scanners. 6 7 #ifndef V8_PARSING_SCANNER_H_ 8 #define V8_PARSING_SCANNER_H_ 9 10 #include <algorithm> 11 #include <memory> 12 13 #include "include/v8.h" 14 #include "src/base/logging.h" 15 #include "src/common/globals.h" 16 #include "src/common/message-template.h" 17 #include "src/parsing/literal-buffer.h" 18 #include "src/parsing/token.h" 19 #include "src/strings/char-predicates.h" 20 #include "src/strings/unicode.h" 21 #include "src/utils/allocation.h" 22 #include "src/utils/pointer-with-payload.h" 23 24 namespace v8 { 25 namespace internal { 26 27 class AstRawString; 28 class AstValueFactory; 29 class ExternalOneByteString; 30 class ExternalTwoByteString; 31 class ParserRecorder; 32 class RuntimeCallStats; 33 class Zone; 34 35 // --------------------------------------------------------------------- 36 // Buffered stream of UTF-16 code units, using an internal UTF-16 buffer. 37 // A code unit is a 16 bit value representing either a 16 bit code point 38 // or one part of a surrogate pair that make a single 21 bit code point. 39 class Utf16CharacterStream { 40 public: 41 static const uc32 kEndOfInput = -1; 42 43 virtual ~Utf16CharacterStream() = default; 44 set_parser_error()45 V8_INLINE void set_parser_error() { 46 buffer_cursor_ = buffer_end_; 47 has_parser_error_ = true; 48 } reset_parser_error_flag()49 V8_INLINE void reset_parser_error_flag() { has_parser_error_ = false; } has_parser_error()50 V8_INLINE bool has_parser_error() const { return has_parser_error_; } 51 Peek()52 inline uc32 Peek() { 53 if (V8_LIKELY(buffer_cursor_ < buffer_end_)) { 54 return static_cast<uc32>(*buffer_cursor_); 55 } else if (ReadBlockChecked()) { 56 return static_cast<uc32>(*buffer_cursor_); 57 } else { 58 return kEndOfInput; 59 } 60 } 61 62 // Returns and advances past the next UTF-16 code unit in the input 63 // stream. If there are no more code units it returns kEndOfInput. Advance()64 inline uc32 Advance() { 65 uc32 result = Peek(); 66 buffer_cursor_++; 67 return result; 68 } 69 70 // Returns and advances past the next UTF-16 code unit in the input stream 71 // that meets the checks requirement. If there are no more code units it 72 // returns kEndOfInput. 73 template <typename FunctionType> AdvanceUntil(FunctionType check)74 V8_INLINE uc32 AdvanceUntil(FunctionType check) { 75 while (true) { 76 auto next_cursor_pos = 77 std::find_if(buffer_cursor_, buffer_end_, [&check](uint16_t raw_c0_) { 78 uc32 c0_ = static_cast<uc32>(raw_c0_); 79 return check(c0_); 80 }); 81 82 if (next_cursor_pos == buffer_end_) { 83 buffer_cursor_ = buffer_end_; 84 if (!ReadBlockChecked()) { 85 buffer_cursor_++; 86 return kEndOfInput; 87 } 88 } else { 89 buffer_cursor_ = next_cursor_pos + 1; 90 return static_cast<uc32>(*next_cursor_pos); 91 } 92 } 93 } 94 95 // Go back one by one character in the input stream. 96 // This undoes the most recent Advance(). Back()97 inline void Back() { 98 // The common case - if the previous character is within 99 // buffer_start_ .. buffer_end_ will be handles locally. 100 // Otherwise, a new block is requested. 101 if (V8_LIKELY(buffer_cursor_ > buffer_start_)) { 102 buffer_cursor_--; 103 } else { 104 ReadBlockAt(pos() - 1); 105 } 106 } 107 pos()108 inline size_t pos() const { 109 return buffer_pos_ + (buffer_cursor_ - buffer_start_); 110 } 111 Seek(size_t pos)112 inline void Seek(size_t pos) { 113 if (V8_LIKELY(pos >= buffer_pos_ && 114 pos < (buffer_pos_ + (buffer_end_ - buffer_start_)))) { 115 buffer_cursor_ = buffer_start_ + (pos - buffer_pos_); 116 } else { 117 ReadBlockAt(pos); 118 } 119 } 120 121 // Returns true if the stream could access the V8 heap after construction. can_be_cloned_for_parallel_access()122 bool can_be_cloned_for_parallel_access() const { 123 return can_be_cloned() && !can_access_heap(); 124 } 125 126 // Returns true if the stream can be cloned with Clone. 127 // TODO(rmcilroy): Remove this once ChunkedStreams can be cloned. 128 virtual bool can_be_cloned() const = 0; 129 130 // Clones the character stream to enable another independent scanner to access 131 // the same underlying stream. 132 virtual std::unique_ptr<Utf16CharacterStream> Clone() const = 0; 133 134 // Returns true if the stream could access the V8 heap after construction. 135 virtual bool can_access_heap() const = 0; 136 runtime_call_stats()137 RuntimeCallStats* runtime_call_stats() const { return runtime_call_stats_; } set_runtime_call_stats(RuntimeCallStats * runtime_call_stats)138 void set_runtime_call_stats(RuntimeCallStats* runtime_call_stats) { 139 runtime_call_stats_ = runtime_call_stats; 140 } 141 142 protected: Utf16CharacterStream(const uint16_t * buffer_start,const uint16_t * buffer_cursor,const uint16_t * buffer_end,size_t buffer_pos)143 Utf16CharacterStream(const uint16_t* buffer_start, 144 const uint16_t* buffer_cursor, 145 const uint16_t* buffer_end, size_t buffer_pos) 146 : buffer_start_(buffer_start), 147 buffer_cursor_(buffer_cursor), 148 buffer_end_(buffer_end), 149 buffer_pos_(buffer_pos) {} Utf16CharacterStream()150 Utf16CharacterStream() : Utf16CharacterStream(nullptr, nullptr, nullptr, 0) {} 151 ReadBlockChecked()152 bool ReadBlockChecked() { 153 size_t position = pos(); 154 USE(position); 155 bool success = !has_parser_error() && ReadBlock(); 156 157 // Post-conditions: 1, We should always be at the right position. 158 // 2, Cursor should be inside the buffer. 159 // 3, We should have more characters available iff success. 160 DCHECK_EQ(pos(), position); 161 DCHECK_LE(buffer_cursor_, buffer_end_); 162 DCHECK_LE(buffer_start_, buffer_cursor_); 163 DCHECK_EQ(success, buffer_cursor_ < buffer_end_); 164 return success; 165 } 166 ReadBlockAt(size_t new_pos)167 void ReadBlockAt(size_t new_pos) { 168 // The callers of this method (Back/Back2/Seek) should handle the easy 169 // case (seeking within the current buffer), and we should only get here 170 // if we actually require new data. 171 // (This is really an efficiency check, not a correctness invariant.) 172 DCHECK(new_pos < buffer_pos_ || 173 new_pos >= buffer_pos_ + (buffer_end_ - buffer_start_)); 174 175 // Change pos() to point to new_pos. 176 buffer_pos_ = new_pos; 177 buffer_cursor_ = buffer_start_; 178 DCHECK_EQ(pos(), new_pos); 179 ReadBlockChecked(); 180 } 181 182 // Read more data, and update buffer_*_ to point to it. 183 // Returns true if more data was available. 184 // 185 // ReadBlock() may modify any of the buffer_*_ members, but must sure that 186 // the result of pos() remains unaffected. 187 // 188 // Examples: 189 // - a stream could either fill a separate buffer. Then buffer_start_ and 190 // buffer_cursor_ would point to the beginning of the buffer, and 191 // buffer_pos would be the old pos(). 192 // - a stream with existing buffer chunks would set buffer_start_ and 193 // buffer_end_ to cover the full chunk, and then buffer_cursor_ would 194 // point into the middle of the buffer, while buffer_pos_ would describe 195 // the start of the buffer. 196 virtual bool ReadBlock() = 0; 197 198 const uint16_t* buffer_start_; 199 const uint16_t* buffer_cursor_; 200 const uint16_t* buffer_end_; 201 size_t buffer_pos_; 202 RuntimeCallStats* runtime_call_stats_; 203 bool has_parser_error_ = false; 204 }; 205 206 // ---------------------------------------------------------------------------- 207 // JavaScript Scanner. 208 209 class V8_EXPORT_PRIVATE Scanner { 210 public: 211 // Scoped helper for a re-settable bookmark. 212 class V8_EXPORT_PRIVATE BookmarkScope { 213 public: BookmarkScope(Scanner * scanner)214 explicit BookmarkScope(Scanner* scanner) 215 : scanner_(scanner), 216 bookmark_(kNoBookmark), 217 had_parser_error_(scanner->has_parser_error()) { 218 DCHECK_NOT_NULL(scanner_); 219 } 220 ~BookmarkScope() = default; 221 222 void Set(size_t bookmark); 223 void Apply(); 224 bool HasBeenSet() const; 225 bool HasBeenApplied() const; 226 227 private: 228 static const size_t kNoBookmark; 229 static const size_t kBookmarkWasApplied; 230 231 Scanner* scanner_; 232 size_t bookmark_; 233 bool had_parser_error_; 234 235 DISALLOW_COPY_AND_ASSIGN(BookmarkScope); 236 }; 237 238 // Sets the Scanner into an error state to stop further scanning and terminate 239 // the parsing by only returning ILLEGAL tokens after that. set_parser_error()240 V8_INLINE void set_parser_error() { 241 if (!has_parser_error()) { 242 c0_ = kEndOfInput; 243 source_->set_parser_error(); 244 for (TokenDesc& desc : token_storage_) desc.token = Token::ILLEGAL; 245 } 246 } reset_parser_error_flag()247 V8_INLINE void reset_parser_error_flag() { 248 source_->reset_parser_error_flag(); 249 } has_parser_error()250 V8_INLINE bool has_parser_error() const { 251 return source_->has_parser_error(); 252 } 253 254 // Representation of an interval of source positions. 255 struct Location { LocationLocation256 Location(int b, int e) : beg_pos(b), end_pos(e) { } LocationLocation257 Location() : beg_pos(0), end_pos(0) { } 258 lengthLocation259 int length() const { return end_pos - beg_pos; } IsValidLocation260 bool IsValid() const { return base::IsInRange(beg_pos, 0, end_pos); } 261 invalidLocation262 static Location invalid() { return Location(-1, 0); } 263 264 int beg_pos; 265 int end_pos; 266 }; 267 268 // -1 is outside of the range of any real source code. 269 static const int kNoOctalLocation = -1; 270 static const uc32 kEndOfInput = Utf16CharacterStream::kEndOfInput; 271 272 explicit Scanner(Utf16CharacterStream* source, bool is_module); 273 274 void Initialize(); 275 276 // Returns the next token and advances input. 277 Token::Value Next(); 278 // Returns the token following peek() 279 Token::Value PeekAhead(); 280 // Returns the current token again. current_token()281 Token::Value current_token() const { return current().token; } 282 283 // Returns the location information for the current token 284 // (the token last returned by Next()). location()285 const Location& location() const { return current().location; } 286 287 // This error is specifically an invalid hex or unicode escape sequence. has_error()288 bool has_error() const { return scanner_error_ != MessageTemplate::kNone; } error()289 MessageTemplate error() const { return scanner_error_; } error_location()290 const Location& error_location() const { return scanner_error_location_; } 291 has_invalid_template_escape()292 bool has_invalid_template_escape() const { 293 return current().invalid_template_escape_message != MessageTemplate::kNone; 294 } invalid_template_escape_message()295 MessageTemplate invalid_template_escape_message() const { 296 DCHECK(has_invalid_template_escape()); 297 return current().invalid_template_escape_message; 298 } 299 clear_invalid_template_escape_message()300 void clear_invalid_template_escape_message() { 301 DCHECK(has_invalid_template_escape()); 302 current_->invalid_template_escape_message = MessageTemplate::kNone; 303 } 304 invalid_template_escape_location()305 Location invalid_template_escape_location() const { 306 DCHECK(has_invalid_template_escape()); 307 return current().invalid_template_escape_location; 308 } 309 310 // Similar functions for the upcoming token. 311 312 // One token look-ahead (past the token returned by Next()). peek()313 Token::Value peek() const { return next().token; } 314 peek_location()315 const Location& peek_location() const { return next().location; } 316 literal_contains_escapes()317 bool literal_contains_escapes() const { 318 return LiteralContainsEscapes(current()); 319 } 320 next_literal_contains_escapes()321 bool next_literal_contains_escapes() const { 322 return LiteralContainsEscapes(next()); 323 } 324 325 const AstRawString* CurrentSymbol(AstValueFactory* ast_value_factory) const; 326 327 const AstRawString* NextSymbol(AstValueFactory* ast_value_factory) const; 328 const AstRawString* CurrentRawSymbol( 329 AstValueFactory* ast_value_factory) const; 330 331 double DoubleValue(); 332 333 const char* CurrentLiteralAsCString(Zone* zone) const; 334 CurrentMatches(Token::Value token)335 inline bool CurrentMatches(Token::Value token) const { 336 DCHECK(Token::IsKeyword(token)); 337 return current().token == token; 338 } 339 340 template <size_t N> NextLiteralExactlyEquals(const char (& s)[N])341 bool NextLiteralExactlyEquals(const char (&s)[N]) { 342 DCHECK(next().CanAccessLiteral()); 343 // The length of the token is used to make sure the literal equals without 344 // taking escape sequences (e.g., "use \x73trict") or line continuations 345 // (e.g., "use \(newline) strict") into account. 346 if (!is_next_literal_one_byte()) return false; 347 if (peek_location().length() != N + 1) return false; 348 349 Vector<const uint8_t> next = next_literal_one_byte_string(); 350 const char* chars = reinterpret_cast<const char*>(next.begin()); 351 return next.length() == N - 1 && strncmp(s, chars, N - 1) == 0; 352 } 353 354 template <size_t N> CurrentLiteralEquals(const char (& s)[N])355 bool CurrentLiteralEquals(const char (&s)[N]) { 356 DCHECK(current().CanAccessLiteral()); 357 if (!is_literal_one_byte()) return false; 358 359 Vector<const uint8_t> current = literal_one_byte_string(); 360 const char* chars = reinterpret_cast<const char*>(current.begin()); 361 return current.length() == N - 1 && strncmp(s, chars, N - 1) == 0; 362 } 363 364 // Returns the location of the last seen octal literal. octal_position()365 Location octal_position() const { return octal_pos_; } clear_octal_position()366 void clear_octal_position() { 367 octal_pos_ = Location::invalid(); 368 octal_message_ = MessageTemplate::kNone; 369 } octal_message()370 MessageTemplate octal_message() const { return octal_message_; } 371 372 // Returns the value of the last smi that was scanned. smi_value()373 uint32_t smi_value() const { return current().smi_value_; } 374 375 // Seek forward to the given position. This operation does not 376 // work in general, for instance when there are pushed back 377 // characters, but works for seeking forward until simple delimiter 378 // tokens, which is what it is used for. 379 void SeekForward(int pos); 380 381 // Returns true if there was a line terminator before the peek'ed token, 382 // possibly inside a multi-line comment. HasLineTerminatorBeforeNext()383 bool HasLineTerminatorBeforeNext() const { 384 return next().after_line_terminator; 385 } 386 HasLineTerminatorAfterNext()387 bool HasLineTerminatorAfterNext() { 388 Token::Value ensure_next_next = PeekAhead(); 389 USE(ensure_next_next); 390 return next_next().after_line_terminator; 391 } 392 393 // Scans the input as a regular expression pattern, next token must be /(=). 394 // Returns true if a pattern is scanned. 395 bool ScanRegExpPattern(); 396 // Scans the input as regular expression flags. Returns the flags on success. 397 Maybe<int> ScanRegExpFlags(); 398 399 // Scans the input as a template literal ScanTemplateContinuation()400 Token::Value ScanTemplateContinuation() { 401 DCHECK_EQ(next().token, Token::RBRACE); 402 DCHECK_EQ(source_pos() - 1, next().location.beg_pos); 403 return ScanTemplateSpan(); 404 } 405 406 template <typename LocalIsolate> 407 Handle<String> SourceUrl(LocalIsolate* isolate) const; 408 template <typename LocalIsolate> 409 Handle<String> SourceMappingUrl(LocalIsolate* isolate) const; 410 FoundHtmlComment()411 bool FoundHtmlComment() const { return found_html_comment_; } 412 allow_harmony_optional_chaining()413 bool allow_harmony_optional_chaining() const { 414 return allow_harmony_optional_chaining_; 415 } 416 set_allow_harmony_optional_chaining(bool allow)417 void set_allow_harmony_optional_chaining(bool allow) { 418 allow_harmony_optional_chaining_ = allow; 419 } 420 allow_harmony_nullish()421 bool allow_harmony_nullish() const { return allow_harmony_nullish_; } 422 set_allow_harmony_nullish(bool allow)423 void set_allow_harmony_nullish(bool allow) { allow_harmony_nullish_ = allow; } 424 stream()425 const Utf16CharacterStream* stream() const { return source_; } 426 427 private: 428 // Scoped helper for saving & restoring scanner error state. 429 // This is used for tagged template literals, in which normally forbidden 430 // escape sequences are allowed. 431 class ErrorState; 432 433 // The current and look-ahead token. 434 struct TokenDesc { 435 Location location = {0, 0}; 436 LiteralBuffer literal_chars; 437 LiteralBuffer raw_literal_chars; 438 Token::Value token = Token::UNINITIALIZED; 439 MessageTemplate invalid_template_escape_message = MessageTemplate::kNone; 440 Location invalid_template_escape_location; 441 uint32_t smi_value_ = 0; 442 bool after_line_terminator = false; 443 444 #ifdef DEBUG CanAccessLiteralTokenDesc445 bool CanAccessLiteral() const { 446 return token == Token::PRIVATE_NAME || token == Token::ILLEGAL || 447 token == Token::ESCAPED_KEYWORD || token == Token::UNINITIALIZED || 448 token == Token::REGEXP_LITERAL || 449 base::IsInRange(token, Token::NUMBER, Token::STRING) || 450 Token::IsAnyIdentifier(token) || Token::IsKeyword(token) || 451 base::IsInRange(token, Token::TEMPLATE_SPAN, Token::TEMPLATE_TAIL); 452 } CanAccessRawLiteralTokenDesc453 bool CanAccessRawLiteral() const { 454 return token == Token::ILLEGAL || token == Token::UNINITIALIZED || 455 base::IsInRange(token, Token::TEMPLATE_SPAN, Token::TEMPLATE_TAIL); 456 } 457 #endif // DEBUG 458 }; 459 460 enum NumberKind { 461 IMPLICIT_OCTAL, 462 BINARY, 463 OCTAL, 464 HEX, 465 DECIMAL, 466 DECIMAL_WITH_LEADING_ZERO 467 }; 468 IsValidBigIntKind(NumberKind kind)469 inline bool IsValidBigIntKind(NumberKind kind) { 470 return base::IsInRange(kind, BINARY, DECIMAL); 471 } 472 IsDecimalNumberKind(NumberKind kind)473 inline bool IsDecimalNumberKind(NumberKind kind) { 474 return base::IsInRange(kind, DECIMAL, DECIMAL_WITH_LEADING_ZERO); 475 } 476 477 static const int kCharacterLookaheadBufferSize = 1; 478 static const int kMaxAscii = 127; 479 480 // Scans octal escape sequence. Also accepts "\0" decimal escape sequence. 481 template <bool capture_raw> 482 uc32 ScanOctalEscape(uc32 c, int length); 483 484 // Call this after setting source_ to the input. Init()485 void Init() { 486 // Set c0_ (one character ahead) 487 STATIC_ASSERT(kCharacterLookaheadBufferSize == 1); 488 Advance(); 489 490 current_ = &token_storage_[0]; 491 next_ = &token_storage_[1]; 492 next_next_ = &token_storage_[2]; 493 494 found_html_comment_ = false; 495 scanner_error_ = MessageTemplate::kNone; 496 } 497 ReportScannerError(const Location & location,MessageTemplate error)498 void ReportScannerError(const Location& location, MessageTemplate error) { 499 if (has_error()) return; 500 scanner_error_ = error; 501 scanner_error_location_ = location; 502 } 503 ReportScannerError(int pos,MessageTemplate error)504 void ReportScannerError(int pos, MessageTemplate error) { 505 if (has_error()) return; 506 scanner_error_ = error; 507 scanner_error_location_ = Location(pos, pos + 1); 508 } 509 510 // Seek to the next_ token at the given position. 511 void SeekNext(size_t position); 512 AddLiteralChar(uc32 c)513 V8_INLINE void AddLiteralChar(uc32 c) { next().literal_chars.AddChar(c); } 514 AddLiteralChar(char c)515 V8_INLINE void AddLiteralChar(char c) { next().literal_chars.AddChar(c); } 516 AddRawLiteralChar(uc32 c)517 V8_INLINE void AddRawLiteralChar(uc32 c) { 518 next().raw_literal_chars.AddChar(c); 519 } 520 AddLiteralCharAdvance()521 V8_INLINE void AddLiteralCharAdvance() { 522 AddLiteralChar(c0_); 523 Advance(); 524 } 525 526 // Low-level scanning support. 527 template <bool capture_raw = false> Advance()528 void Advance() { 529 if (capture_raw) { 530 AddRawLiteralChar(c0_); 531 } 532 c0_ = source_->Advance(); 533 } 534 535 template <typename FunctionType> AdvanceUntil(FunctionType check)536 V8_INLINE void AdvanceUntil(FunctionType check) { 537 c0_ = source_->AdvanceUntil(check); 538 } 539 CombineSurrogatePair()540 bool CombineSurrogatePair() { 541 DCHECK(!unibrow::Utf16::IsLeadSurrogate(kEndOfInput)); 542 if (unibrow::Utf16::IsLeadSurrogate(c0_)) { 543 uc32 c1 = source_->Advance(); 544 DCHECK(!unibrow::Utf16::IsTrailSurrogate(kEndOfInput)); 545 if (unibrow::Utf16::IsTrailSurrogate(c1)) { 546 c0_ = unibrow::Utf16::CombineSurrogatePair(c0_, c1); 547 return true; 548 } 549 source_->Back(); 550 } 551 return false; 552 } 553 PushBack(uc32 ch)554 void PushBack(uc32 ch) { 555 DCHECK_LE(c0_, static_cast<uc32>(unibrow::Utf16::kMaxNonSurrogateCharCode)); 556 source_->Back(); 557 c0_ = ch; 558 } 559 Peek()560 uc32 Peek() const { return source_->Peek(); } 561 Select(Token::Value tok)562 inline Token::Value Select(Token::Value tok) { 563 Advance(); 564 return tok; 565 } 566 Select(uc32 next,Token::Value then,Token::Value else_)567 inline Token::Value Select(uc32 next, Token::Value then, Token::Value else_) { 568 Advance(); 569 if (c0_ == next) { 570 Advance(); 571 return then; 572 } else { 573 return else_; 574 } 575 } 576 // Returns the literal string, if any, for the current token (the 577 // token last returned by Next()). The string is 0-terminated. 578 // Literal strings are collected for identifiers, strings, numbers as well 579 // as for template literals. For template literals we also collect the raw 580 // form. 581 // These functions only give the correct result if the literal was scanned 582 // when a LiteralScope object is alive. 583 // 584 // Current usage of these functions is unfortunately a little undisciplined, 585 // and is_literal_one_byte() + is_literal_one_byte_string() is also 586 // requested for tokens that do not have a literal. Hence, we treat any 587 // token as a one-byte literal. E.g. Token::FUNCTION pretends to have a 588 // literal "function". literal_one_byte_string()589 Vector<const uint8_t> literal_one_byte_string() const { 590 DCHECK(current().CanAccessLiteral() || Token::IsKeyword(current().token) || 591 current().token == Token::ESCAPED_KEYWORD); 592 return current().literal_chars.one_byte_literal(); 593 } literal_two_byte_string()594 Vector<const uint16_t> literal_two_byte_string() const { 595 DCHECK(current().CanAccessLiteral() || Token::IsKeyword(current().token) || 596 current().token == Token::ESCAPED_KEYWORD); 597 return current().literal_chars.two_byte_literal(); 598 } is_literal_one_byte()599 bool is_literal_one_byte() const { 600 DCHECK(current().CanAccessLiteral() || Token::IsKeyword(current().token) || 601 current().token == Token::ESCAPED_KEYWORD); 602 return current().literal_chars.is_one_byte(); 603 } 604 // Returns the literal string for the next token (the token that 605 // would be returned if Next() were called). next_literal_one_byte_string()606 Vector<const uint8_t> next_literal_one_byte_string() const { 607 DCHECK(next().CanAccessLiteral()); 608 return next().literal_chars.one_byte_literal(); 609 } next_literal_two_byte_string()610 Vector<const uint16_t> next_literal_two_byte_string() const { 611 DCHECK(next().CanAccessLiteral()); 612 return next().literal_chars.two_byte_literal(); 613 } is_next_literal_one_byte()614 bool is_next_literal_one_byte() const { 615 DCHECK(next().CanAccessLiteral()); 616 return next().literal_chars.is_one_byte(); 617 } raw_literal_one_byte_string()618 Vector<const uint8_t> raw_literal_one_byte_string() const { 619 DCHECK(current().CanAccessRawLiteral()); 620 return current().raw_literal_chars.one_byte_literal(); 621 } raw_literal_two_byte_string()622 Vector<const uint16_t> raw_literal_two_byte_string() const { 623 DCHECK(current().CanAccessRawLiteral()); 624 return current().raw_literal_chars.two_byte_literal(); 625 } is_raw_literal_one_byte()626 bool is_raw_literal_one_byte() const { 627 DCHECK(current().CanAccessRawLiteral()); 628 return current().raw_literal_chars.is_one_byte(); 629 } 630 631 template <bool capture_raw, bool unicode = false> 632 uc32 ScanHexNumber(int expected_length); 633 // Scan a number of any length but not bigger than max_value. For example, the 634 // number can be 000000001, so it's very long in characters but its value is 635 // small. 636 template <bool capture_raw> 637 uc32 ScanUnlimitedLengthHexNumber(int max_value, int beg_pos); 638 639 // Scans a single JavaScript token. 640 V8_INLINE Token::Value ScanSingleToken(); 641 V8_INLINE void Scan(); 642 // Performance hack: pass through a pre-calculated "next()" value to avoid 643 // having to re-calculate it in Scan. You'd think the compiler would be able 644 // to hoist the next() calculation out of the inlined Scan method, but seems 645 // that pointer aliasing analysis fails show that this is safe. 646 V8_INLINE void Scan(TokenDesc* next_desc); 647 648 V8_INLINE Token::Value SkipWhiteSpace(); 649 Token::Value SkipSingleHTMLComment(); 650 Token::Value SkipSingleLineComment(); 651 Token::Value SkipSourceURLComment(); 652 void TryToParseSourceURLComment(); 653 Token::Value SkipMultiLineComment(); 654 // Scans a possible HTML comment -- begins with '<!'. 655 Token::Value ScanHtmlComment(); 656 657 bool ScanDigitsWithNumericSeparators(bool (*predicate)(uc32 ch), 658 bool is_check_first_digit); 659 bool ScanDecimalDigits(bool allow_numeric_separator); 660 // Optimized function to scan decimal number as Smi. 661 bool ScanDecimalAsSmi(uint64_t* value, bool allow_numeric_separator); 662 bool ScanDecimalAsSmiWithNumericSeparators(uint64_t* value); 663 bool ScanHexDigits(); 664 bool ScanBinaryDigits(); 665 bool ScanSignedInteger(); 666 bool ScanOctalDigits(); 667 bool ScanImplicitOctalDigits(int start_pos, NumberKind* kind); 668 669 Token::Value ScanNumber(bool seen_period); 670 V8_INLINE Token::Value ScanIdentifierOrKeyword(); 671 V8_INLINE Token::Value ScanIdentifierOrKeywordInner(); 672 Token::Value ScanIdentifierOrKeywordInnerSlow(bool escaped, 673 bool can_be_keyword); 674 675 Token::Value ScanString(); 676 Token::Value ScanPrivateName(); 677 678 // Scans an escape-sequence which is part of a string and adds the 679 // decoded character to the current literal. Returns true if a pattern 680 // is scanned. 681 template <bool capture_raw> 682 bool ScanEscape(); 683 684 // Decodes a Unicode escape-sequence which is part of an identifier. 685 // If the escape sequence cannot be decoded the result is kBadChar. 686 uc32 ScanIdentifierUnicodeEscape(); 687 // Helper for the above functions. 688 template <bool capture_raw> 689 uc32 ScanUnicodeEscape(); 690 691 Token::Value ScanTemplateSpan(); 692 693 // Return the current source position. source_pos()694 int source_pos() { 695 return static_cast<int>(source_->pos()) - kCharacterLookaheadBufferSize; 696 } 697 LiteralContainsEscapes(const TokenDesc & token)698 static bool LiteralContainsEscapes(const TokenDesc& token) { 699 Location location = token.location; 700 int source_length = (location.end_pos - location.beg_pos); 701 if (token.token == Token::STRING) { 702 // Subtract delimiters. 703 source_length -= 2; 704 } 705 return token.literal_chars.length() != source_length; 706 } 707 708 #ifdef DEBUG 709 void SanityCheckTokenDesc(const TokenDesc&) const; 710 #endif 711 next()712 TokenDesc& next() { return *next_; } 713 current()714 const TokenDesc& current() const { return *current_; } next()715 const TokenDesc& next() const { return *next_; } next_next()716 const TokenDesc& next_next() const { return *next_next_; } 717 718 TokenDesc* current_; // desc for current token (as returned by Next()) 719 TokenDesc* next_; // desc for next token (one token look-ahead) 720 TokenDesc* next_next_; // desc for the token after next (after PeakAhead()) 721 722 // Input stream. Must be initialized to an Utf16CharacterStream. 723 Utf16CharacterStream* const source_; 724 725 // One Unicode character look-ahead; c0_ < 0 at the end of the input. 726 uc32 c0_; 727 728 TokenDesc token_storage_[3]; 729 730 // Whether this scanner encountered an HTML comment. 731 bool found_html_comment_; 732 733 // Harmony flags to allow ESNext features. 734 bool allow_harmony_optional_chaining_; 735 bool allow_harmony_nullish_; 736 737 const bool is_module_; 738 739 // Values parsed from magic comments. 740 LiteralBuffer source_url_; 741 LiteralBuffer source_mapping_url_; 742 743 // Last-seen positions of potentially problematic tokens. 744 Location octal_pos_; 745 MessageTemplate octal_message_; 746 747 MessageTemplate scanner_error_; 748 Location scanner_error_location_; 749 }; 750 751 } // namespace internal 752 } // namespace v8 753 754 #endif // V8_PARSING_SCANNER_H_ 755