1 // Copyright 2011 the V8 project authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 // Features shared by parsing and pre-parsing scanners. 6 7 #ifndef V8_PARSING_SCANNER_H_ 8 #define V8_PARSING_SCANNER_H_ 9 10 #include <algorithm> 11 #include <memory> 12 13 #include "src/base/logging.h" 14 #include "src/base/strings.h" 15 #include "src/common/globals.h" 16 #include "src/common/message-template.h" 17 #include "src/parsing/literal-buffer.h" 18 #include "src/parsing/parse-info.h" 19 #include "src/parsing/token.h" 20 #include "src/regexp/regexp-flags.h" 21 #include "src/strings/char-predicates.h" 22 #include "src/strings/unicode.h" 23 #include "src/utils/allocation.h" 24 #include "src/utils/pointer-with-payload.h" 25 26 namespace v8 { 27 namespace internal { 28 29 class AstRawString; 30 class AstValueFactory; 31 class ExternalOneByteString; 32 class ExternalTwoByteString; 33 class ParserRecorder; 34 class RuntimeCallStats; 35 class Zone; 36 37 // --------------------------------------------------------------------- 38 // Buffered stream of UTF-16 code units, using an internal UTF-16 buffer. 39 // A code unit is a 16 bit value representing either a 16 bit code point 40 // or one part of a surrogate pair that make a single 21 bit code point. 41 class Utf16CharacterStream { 42 public: 43 static constexpr base::uc32 kEndOfInput = static_cast<base::uc32>(-1); 44 45 virtual ~Utf16CharacterStream() = default; 46 set_parser_error()47 V8_INLINE void set_parser_error() { 48 buffer_cursor_ = buffer_end_; 49 has_parser_error_ = true; 50 } reset_parser_error_flag()51 V8_INLINE void reset_parser_error_flag() { has_parser_error_ = false; } has_parser_error()52 V8_INLINE bool has_parser_error() const { return has_parser_error_; } 53 Peek()54 inline base::uc32 Peek() { 55 if (V8_LIKELY(buffer_cursor_ < buffer_end_)) { 56 return static_cast<base::uc32>(*buffer_cursor_); 57 } else if (ReadBlockChecked()) { 58 return static_cast<base::uc32>(*buffer_cursor_); 59 } else { 60 return kEndOfInput; 61 } 62 } 63 64 // Returns and advances past the next UTF-16 code unit in the input 65 // stream. If there are no more code units it returns kEndOfInput. Advance()66 inline base::uc32 Advance() { 67 base::uc32 result = Peek(); 68 buffer_cursor_++; 69 return result; 70 } 71 72 // Returns and advances past the next UTF-16 code unit in the input stream 73 // that meets the checks requirement. If there are no more code units it 74 // returns kEndOfInput. 75 template <typename FunctionType> AdvanceUntil(FunctionType check)76 V8_INLINE base::uc32 AdvanceUntil(FunctionType check) { 77 while (true) { 78 auto next_cursor_pos = 79 std::find_if(buffer_cursor_, buffer_end_, [&check](uint16_t raw_c0_) { 80 base::uc32 c0_ = static_cast<base::uc32>(raw_c0_); 81 return check(c0_); 82 }); 83 84 if (next_cursor_pos == buffer_end_) { 85 buffer_cursor_ = buffer_end_; 86 if (!ReadBlockChecked()) { 87 buffer_cursor_++; 88 return kEndOfInput; 89 } 90 } else { 91 buffer_cursor_ = next_cursor_pos + 1; 92 return static_cast<base::uc32>(*next_cursor_pos); 93 } 94 } 95 } 96 97 // Go back one by one character in the input stream. 98 // This undoes the most recent Advance(). Back()99 inline void Back() { 100 // The common case - if the previous character is within 101 // buffer_start_ .. buffer_end_ will be handles locally. 102 // Otherwise, a new block is requested. 103 if (V8_LIKELY(buffer_cursor_ > buffer_start_)) { 104 buffer_cursor_--; 105 } else { 106 ReadBlockAt(pos() - 1); 107 } 108 } 109 pos()110 inline size_t pos() const { 111 return buffer_pos_ + (buffer_cursor_ - buffer_start_); 112 } 113 Seek(size_t pos)114 inline void Seek(size_t pos) { 115 if (V8_LIKELY(pos >= buffer_pos_ && 116 pos < (buffer_pos_ + (buffer_end_ - buffer_start_)))) { 117 buffer_cursor_ = buffer_start_ + (pos - buffer_pos_); 118 } else { 119 ReadBlockAt(pos); 120 } 121 } 122 123 // Returns true if the stream could access the V8 heap after construction. can_be_cloned_for_parallel_access()124 bool can_be_cloned_for_parallel_access() const { 125 return can_be_cloned() && !can_access_heap(); 126 } 127 128 // Returns true if the stream can be cloned with Clone. 129 // TODO(rmcilroy): Remove this once ChunkedStreams can be cloned. 130 virtual bool can_be_cloned() const = 0; 131 132 // Clones the character stream to enable another independent scanner to access 133 // the same underlying stream. 134 virtual std::unique_ptr<Utf16CharacterStream> Clone() const = 0; 135 136 // Returns true if the stream could access the V8 heap after construction. 137 virtual bool can_access_heap() const = 0; 138 runtime_call_stats()139 RuntimeCallStats* runtime_call_stats() const { return runtime_call_stats_; } set_runtime_call_stats(RuntimeCallStats * runtime_call_stats)140 void set_runtime_call_stats(RuntimeCallStats* runtime_call_stats) { 141 runtime_call_stats_ = runtime_call_stats; 142 } 143 144 protected: Utf16CharacterStream(const uint16_t * buffer_start,const uint16_t * buffer_cursor,const uint16_t * buffer_end,size_t buffer_pos)145 Utf16CharacterStream(const uint16_t* buffer_start, 146 const uint16_t* buffer_cursor, 147 const uint16_t* buffer_end, size_t buffer_pos) 148 : buffer_start_(buffer_start), 149 buffer_cursor_(buffer_cursor), 150 buffer_end_(buffer_end), 151 buffer_pos_(buffer_pos) {} Utf16CharacterStream()152 Utf16CharacterStream() : Utf16CharacterStream(nullptr, nullptr, nullptr, 0) {} 153 ReadBlockChecked()154 bool ReadBlockChecked() { 155 size_t position = pos(); 156 USE(position); 157 bool success = !has_parser_error() && ReadBlock(); 158 159 // Post-conditions: 1, We should always be at the right position. 160 // 2, Cursor should be inside the buffer. 161 // 3, We should have more characters available iff success. 162 DCHECK_EQ(pos(), position); 163 DCHECK_LE(buffer_cursor_, buffer_end_); 164 DCHECK_LE(buffer_start_, buffer_cursor_); 165 DCHECK_EQ(success, buffer_cursor_ < buffer_end_); 166 return success; 167 } 168 ReadBlockAt(size_t new_pos)169 void ReadBlockAt(size_t new_pos) { 170 // The callers of this method (Back/Back2/Seek) should handle the easy 171 // case (seeking within the current buffer), and we should only get here 172 // if we actually require new data. 173 // (This is really an efficiency check, not a correctness invariant.) 174 DCHECK(new_pos < buffer_pos_ || 175 new_pos >= buffer_pos_ + (buffer_end_ - buffer_start_)); 176 177 // Change pos() to point to new_pos. 178 buffer_pos_ = new_pos; 179 buffer_cursor_ = buffer_start_; 180 DCHECK_EQ(pos(), new_pos); 181 ReadBlockChecked(); 182 } 183 184 // Read more data, and update buffer_*_ to point to it. 185 // Returns true if more data was available. 186 // 187 // ReadBlock() may modify any of the buffer_*_ members, but must sure that 188 // the result of pos() remains unaffected. 189 // 190 // Examples: 191 // - a stream could either fill a separate buffer. Then buffer_start_ and 192 // buffer_cursor_ would point to the beginning of the buffer, and 193 // buffer_pos would be the old pos(). 194 // - a stream with existing buffer chunks would set buffer_start_ and 195 // buffer_end_ to cover the full chunk, and then buffer_cursor_ would 196 // point into the middle of the buffer, while buffer_pos_ would describe 197 // the start of the buffer. 198 virtual bool ReadBlock() = 0; 199 200 const uint16_t* buffer_start_; 201 const uint16_t* buffer_cursor_; 202 const uint16_t* buffer_end_; 203 size_t buffer_pos_; 204 RuntimeCallStats* runtime_call_stats_; 205 bool has_parser_error_ = false; 206 }; 207 208 // ---------------------------------------------------------------------------- 209 // JavaScript Scanner. 210 211 class V8_EXPORT_PRIVATE Scanner { 212 public: 213 // Scoped helper for a re-settable bookmark. 214 class V8_EXPORT_PRIVATE V8_NODISCARD BookmarkScope { 215 public: BookmarkScope(Scanner * scanner)216 explicit BookmarkScope(Scanner* scanner) 217 : scanner_(scanner), 218 bookmark_(kNoBookmark), 219 had_parser_error_(scanner->has_parser_error()) { 220 DCHECK_NOT_NULL(scanner_); 221 } 222 ~BookmarkScope() = default; 223 BookmarkScope(const BookmarkScope&) = delete; 224 BookmarkScope& operator=(const BookmarkScope&) = delete; 225 226 void Set(size_t bookmark); 227 void Apply(); 228 bool HasBeenSet() const; 229 bool HasBeenApplied() const; 230 231 private: 232 static const size_t kNoBookmark; 233 static const size_t kBookmarkWasApplied; 234 235 Scanner* scanner_; 236 size_t bookmark_; 237 bool had_parser_error_; 238 }; 239 240 // Sets the Scanner into an error state to stop further scanning and terminate 241 // the parsing by only returning ILLEGAL tokens after that. set_parser_error()242 V8_INLINE void set_parser_error() { 243 if (!has_parser_error()) { 244 c0_ = kEndOfInput; 245 source_->set_parser_error(); 246 for (TokenDesc& desc : token_storage_) desc.token = Token::ILLEGAL; 247 } 248 } reset_parser_error_flag()249 V8_INLINE void reset_parser_error_flag() { 250 source_->reset_parser_error_flag(); 251 } has_parser_error()252 V8_INLINE bool has_parser_error() const { 253 return source_->has_parser_error(); 254 } 255 256 // Representation of an interval of source positions. 257 struct Location { LocationLocation258 Location(int b, int e) : beg_pos(b), end_pos(e) { } LocationLocation259 Location() : beg_pos(0), end_pos(0) { } 260 lengthLocation261 int length() const { return end_pos - beg_pos; } IsValidLocation262 bool IsValid() const { return base::IsInRange(beg_pos, 0, end_pos); } 263 invalidLocation264 static Location invalid() { return Location(-1, 0); } 265 266 int beg_pos; 267 int end_pos; 268 }; 269 270 // -1 is outside of the range of any real source code. 271 static constexpr base::uc32 kEndOfInput = Utf16CharacterStream::kEndOfInput; 272 static constexpr base::uc32 kInvalidSequence = static_cast<base::uc32>(-1); 273 Invalid()274 static constexpr base::uc32 Invalid() { return Scanner::kInvalidSequence; } 275 static bool IsInvalid(base::uc32 c); 276 277 explicit Scanner(Utf16CharacterStream* source, UnoptimizedCompileFlags flags); 278 279 void Initialize(); 280 281 // Returns the next token and advances input. 282 Token::Value Next(); 283 // Returns the token following peek() 284 Token::Value PeekAhead(); 285 // Returns the current token again. current_token()286 Token::Value current_token() const { return current().token; } 287 288 // Returns the location information for the current token 289 // (the token last returned by Next()). location()290 const Location& location() const { return current().location; } 291 292 // This error is specifically an invalid hex or unicode escape sequence. has_error()293 bool has_error() const { return scanner_error_ != MessageTemplate::kNone; } error()294 MessageTemplate error() const { return scanner_error_; } error_location()295 const Location& error_location() const { return scanner_error_location_; } 296 has_invalid_template_escape()297 bool has_invalid_template_escape() const { 298 return current().invalid_template_escape_message != MessageTemplate::kNone; 299 } invalid_template_escape_message()300 MessageTemplate invalid_template_escape_message() const { 301 DCHECK(has_invalid_template_escape()); 302 return current().invalid_template_escape_message; 303 } 304 clear_invalid_template_escape_message()305 void clear_invalid_template_escape_message() { 306 DCHECK(has_invalid_template_escape()); 307 current_->invalid_template_escape_message = MessageTemplate::kNone; 308 } 309 invalid_template_escape_location()310 Location invalid_template_escape_location() const { 311 DCHECK(has_invalid_template_escape()); 312 return current().invalid_template_escape_location; 313 } 314 315 // Similar functions for the upcoming token. 316 317 // One token look-ahead (past the token returned by Next()). peek()318 Token::Value peek() const { return next().token; } 319 peek_location()320 const Location& peek_location() const { return next().location; } 321 literal_contains_escapes()322 bool literal_contains_escapes() const { 323 return LiteralContainsEscapes(current()); 324 } 325 next_literal_contains_escapes()326 bool next_literal_contains_escapes() const { 327 return LiteralContainsEscapes(next()); 328 } 329 330 const AstRawString* CurrentSymbol(AstValueFactory* ast_value_factory) const; 331 332 const AstRawString* NextSymbol(AstValueFactory* ast_value_factory) const; 333 const AstRawString* CurrentRawSymbol( 334 AstValueFactory* ast_value_factory) const; 335 336 double DoubleValue(); 337 338 const char* CurrentLiteralAsCString(Zone* zone) const; 339 CurrentMatches(Token::Value token)340 inline bool CurrentMatches(Token::Value token) const { 341 DCHECK(Token::IsKeyword(token)); 342 return current().token == token; 343 } 344 345 template <size_t N> NextLiteralExactlyEquals(const char (& s)[N])346 bool NextLiteralExactlyEquals(const char (&s)[N]) { 347 DCHECK(next().CanAccessLiteral()); 348 // The length of the token is used to make sure the literal equals without 349 // taking escape sequences (e.g., "use \x73trict") or line continuations 350 // (e.g., "use \(newline) strict") into account. 351 if (!is_next_literal_one_byte()) return false; 352 if (peek_location().length() != N + 1) return false; 353 354 base::Vector<const uint8_t> next = next_literal_one_byte_string(); 355 const char* chars = reinterpret_cast<const char*>(next.begin()); 356 return next.length() == N - 1 && strncmp(s, chars, N - 1) == 0; 357 } 358 359 template <size_t N> CurrentLiteralEquals(const char (& s)[N])360 bool CurrentLiteralEquals(const char (&s)[N]) { 361 DCHECK(current().CanAccessLiteral()); 362 if (!is_literal_one_byte()) return false; 363 364 base::Vector<const uint8_t> current = literal_one_byte_string(); 365 const char* chars = reinterpret_cast<const char*>(current.begin()); 366 return current.length() == N - 1 && strncmp(s, chars, N - 1) == 0; 367 } 368 369 // Returns the location of the last seen octal literal. octal_position()370 Location octal_position() const { return octal_pos_; } clear_octal_position()371 void clear_octal_position() { 372 octal_pos_ = Location::invalid(); 373 octal_message_ = MessageTemplate::kNone; 374 } octal_message()375 MessageTemplate octal_message() const { return octal_message_; } 376 377 // Returns the value of the last smi that was scanned. smi_value()378 uint32_t smi_value() const { return current().smi_value_; } 379 380 // Seek forward to the given position. This operation does not 381 // work in general, for instance when there are pushed back 382 // characters, but works for seeking forward until simple delimiter 383 // tokens, which is what it is used for. 384 void SeekForward(int pos); 385 386 // Returns true if there was a line terminator before the peek'ed token, 387 // possibly inside a multi-line comment. HasLineTerminatorBeforeNext()388 bool HasLineTerminatorBeforeNext() const { 389 return next().after_line_terminator; 390 } 391 HasLineTerminatorAfterNext()392 bool HasLineTerminatorAfterNext() { 393 Token::Value ensure_next_next = PeekAhead(); 394 USE(ensure_next_next); 395 return next_next().after_line_terminator; 396 } 397 398 // Scans the input as a regular expression pattern, next token must be /(=). 399 // Returns true if a pattern is scanned. 400 bool ScanRegExpPattern(); 401 // Scans the input as regular expression flags. Returns the flags on success. 402 base::Optional<RegExpFlags> ScanRegExpFlags(); 403 404 // Scans the input as a template literal ScanTemplateContinuation()405 Token::Value ScanTemplateContinuation() { 406 DCHECK_EQ(next().token, Token::RBRACE); 407 DCHECK_EQ(source_pos() - 1, next().location.beg_pos); 408 return ScanTemplateSpan(); 409 } 410 411 template <typename IsolateT> 412 Handle<String> SourceUrl(IsolateT* isolate) const; 413 template <typename IsolateT> 414 Handle<String> SourceMappingUrl(IsolateT* isolate) const; 415 FoundHtmlComment()416 bool FoundHtmlComment() const { return found_html_comment_; } 417 stream()418 const Utf16CharacterStream* stream() const { return source_; } 419 420 private: 421 // Scoped helper for saving & restoring scanner error state. 422 // This is used for tagged template literals, in which normally forbidden 423 // escape sequences are allowed. 424 class ErrorState; 425 426 // The current and look-ahead token. 427 struct TokenDesc { 428 Location location = {0, 0}; 429 LiteralBuffer literal_chars; 430 LiteralBuffer raw_literal_chars; 431 Token::Value token = Token::UNINITIALIZED; 432 MessageTemplate invalid_template_escape_message = MessageTemplate::kNone; 433 Location invalid_template_escape_location; 434 uint32_t smi_value_ = 0; 435 bool after_line_terminator = false; 436 437 #ifdef DEBUG CanAccessLiteralTokenDesc438 bool CanAccessLiteral() const { 439 return token == Token::PRIVATE_NAME || token == Token::ILLEGAL || 440 token == Token::ESCAPED_KEYWORD || token == Token::UNINITIALIZED || 441 token == Token::REGEXP_LITERAL || 442 base::IsInRange(token, Token::NUMBER, Token::STRING) || 443 Token::IsAnyIdentifier(token) || Token::IsKeyword(token) || 444 base::IsInRange(token, Token::TEMPLATE_SPAN, Token::TEMPLATE_TAIL); 445 } CanAccessRawLiteralTokenDesc446 bool CanAccessRawLiteral() const { 447 return token == Token::ILLEGAL || token == Token::UNINITIALIZED || 448 base::IsInRange(token, Token::TEMPLATE_SPAN, Token::TEMPLATE_TAIL); 449 } 450 #endif // DEBUG 451 }; 452 453 enum NumberKind { 454 IMPLICIT_OCTAL, 455 BINARY, 456 OCTAL, 457 HEX, 458 DECIMAL, 459 DECIMAL_WITH_LEADING_ZERO 460 }; 461 IsValidBigIntKind(NumberKind kind)462 inline bool IsValidBigIntKind(NumberKind kind) { 463 return base::IsInRange(kind, BINARY, DECIMAL); 464 } 465 IsDecimalNumberKind(NumberKind kind)466 inline bool IsDecimalNumberKind(NumberKind kind) { 467 return base::IsInRange(kind, DECIMAL, DECIMAL_WITH_LEADING_ZERO); 468 } 469 470 static const int kCharacterLookaheadBufferSize = 1; 471 static const int kMaxAscii = 127; 472 473 // Scans octal escape sequence. Also accepts "\0" decimal escape sequence. 474 template <bool capture_raw> 475 base::uc32 ScanOctalEscape(base::uc32 c, int length); 476 477 // Call this after setting source_ to the input. Init()478 void Init() { 479 // Set c0_ (one character ahead) 480 STATIC_ASSERT(kCharacterLookaheadBufferSize == 1); 481 Advance(); 482 483 current_ = &token_storage_[0]; 484 next_ = &token_storage_[1]; 485 next_next_ = &token_storage_[2]; 486 487 found_html_comment_ = false; 488 scanner_error_ = MessageTemplate::kNone; 489 } 490 ReportScannerError(const Location & location,MessageTemplate error)491 void ReportScannerError(const Location& location, MessageTemplate error) { 492 if (has_error()) return; 493 scanner_error_ = error; 494 scanner_error_location_ = location; 495 } 496 ReportScannerError(int pos,MessageTemplate error)497 void ReportScannerError(int pos, MessageTemplate error) { 498 if (has_error()) return; 499 scanner_error_ = error; 500 scanner_error_location_ = Location(pos, pos + 1); 501 } 502 503 // Seek to the next_ token at the given position. 504 void SeekNext(size_t position); 505 AddLiteralChar(base::uc32 c)506 V8_INLINE void AddLiteralChar(base::uc32 c) { 507 next().literal_chars.AddChar(c); 508 } 509 AddLiteralChar(char c)510 V8_INLINE void AddLiteralChar(char c) { next().literal_chars.AddChar(c); } 511 AddRawLiteralChar(base::uc32 c)512 V8_INLINE void AddRawLiteralChar(base::uc32 c) { 513 next().raw_literal_chars.AddChar(c); 514 } 515 AddLiteralCharAdvance()516 V8_INLINE void AddLiteralCharAdvance() { 517 AddLiteralChar(c0_); 518 Advance(); 519 } 520 521 // Low-level scanning support. 522 template <bool capture_raw = false> Advance()523 void Advance() { 524 if (capture_raw) { 525 AddRawLiteralChar(c0_); 526 } 527 c0_ = source_->Advance(); 528 } 529 530 template <typename FunctionType> AdvanceUntil(FunctionType check)531 V8_INLINE void AdvanceUntil(FunctionType check) { 532 c0_ = source_->AdvanceUntil(check); 533 } 534 CombineSurrogatePair()535 bool CombineSurrogatePair() { 536 DCHECK(!unibrow::Utf16::IsLeadSurrogate(kEndOfInput)); 537 if (unibrow::Utf16::IsLeadSurrogate(c0_)) { 538 base::uc32 c1 = source_->Advance(); 539 DCHECK(!unibrow::Utf16::IsTrailSurrogate(kEndOfInput)); 540 if (unibrow::Utf16::IsTrailSurrogate(c1)) { 541 c0_ = unibrow::Utf16::CombineSurrogatePair(c0_, c1); 542 return true; 543 } 544 source_->Back(); 545 } 546 return false; 547 } 548 PushBack(base::uc32 ch)549 void PushBack(base::uc32 ch) { 550 DCHECK(IsInvalid(c0_) || 551 base::IsInRange(c0_, 0u, unibrow::Utf16::kMaxNonSurrogateCharCode)); 552 source_->Back(); 553 c0_ = ch; 554 } 555 Peek()556 base::uc32 Peek() const { return source_->Peek(); } 557 Select(Token::Value tok)558 inline Token::Value Select(Token::Value tok) { 559 Advance(); 560 return tok; 561 } 562 Select(base::uc32 next,Token::Value then,Token::Value else_)563 inline Token::Value Select(base::uc32 next, Token::Value then, 564 Token::Value else_) { 565 Advance(); 566 if (c0_ == next) { 567 Advance(); 568 return then; 569 } else { 570 return else_; 571 } 572 } 573 // Returns the literal string, if any, for the current token (the 574 // token last returned by Next()). The string is 0-terminated. 575 // Literal strings are collected for identifiers, strings, numbers as well 576 // as for template literals. For template literals we also collect the raw 577 // form. 578 // These functions only give the correct result if the literal was scanned 579 // when a LiteralScope object is alive. 580 // 581 // Current usage of these functions is unfortunately a little undisciplined, 582 // and is_literal_one_byte() + is_literal_one_byte_string() is also 583 // requested for tokens that do not have a literal. Hence, we treat any 584 // token as a one-byte literal. E.g. Token::FUNCTION pretends to have a 585 // literal "function". literal_one_byte_string()586 base::Vector<const uint8_t> literal_one_byte_string() const { 587 DCHECK(current().CanAccessLiteral() || Token::IsKeyword(current().token) || 588 current().token == Token::ESCAPED_KEYWORD); 589 return current().literal_chars.one_byte_literal(); 590 } literal_two_byte_string()591 base::Vector<const uint16_t> literal_two_byte_string() const { 592 DCHECK(current().CanAccessLiteral() || Token::IsKeyword(current().token) || 593 current().token == Token::ESCAPED_KEYWORD); 594 return current().literal_chars.two_byte_literal(); 595 } is_literal_one_byte()596 bool is_literal_one_byte() const { 597 DCHECK(current().CanAccessLiteral() || Token::IsKeyword(current().token) || 598 current().token == Token::ESCAPED_KEYWORD); 599 return current().literal_chars.is_one_byte(); 600 } 601 // Returns the literal string for the next token (the token that 602 // would be returned if Next() were called). next_literal_one_byte_string()603 base::Vector<const uint8_t> next_literal_one_byte_string() const { 604 DCHECK(next().CanAccessLiteral()); 605 return next().literal_chars.one_byte_literal(); 606 } next_literal_two_byte_string()607 base::Vector<const uint16_t> next_literal_two_byte_string() const { 608 DCHECK(next().CanAccessLiteral()); 609 return next().literal_chars.two_byte_literal(); 610 } is_next_literal_one_byte()611 bool is_next_literal_one_byte() const { 612 DCHECK(next().CanAccessLiteral()); 613 return next().literal_chars.is_one_byte(); 614 } raw_literal_one_byte_string()615 base::Vector<const uint8_t> raw_literal_one_byte_string() const { 616 DCHECK(current().CanAccessRawLiteral()); 617 return current().raw_literal_chars.one_byte_literal(); 618 } raw_literal_two_byte_string()619 base::Vector<const uint16_t> raw_literal_two_byte_string() const { 620 DCHECK(current().CanAccessRawLiteral()); 621 return current().raw_literal_chars.two_byte_literal(); 622 } is_raw_literal_one_byte()623 bool is_raw_literal_one_byte() const { 624 DCHECK(current().CanAccessRawLiteral()); 625 return current().raw_literal_chars.is_one_byte(); 626 } 627 628 template <bool capture_raw, bool unicode = false> 629 base::uc32 ScanHexNumber(int expected_length); 630 // Scan a number of any length but not bigger than max_value. For example, the 631 // number can be 000000001, so it's very long in characters but its value is 632 // small. 633 template <bool capture_raw> 634 base::uc32 ScanUnlimitedLengthHexNumber(base::uc32 max_value, int beg_pos); 635 636 // Scans a single JavaScript token. 637 V8_INLINE Token::Value ScanSingleToken(); 638 V8_INLINE void Scan(); 639 // Performance hack: pass through a pre-calculated "next()" value to avoid 640 // having to re-calculate it in Scan. You'd think the compiler would be able 641 // to hoist the next() calculation out of the inlined Scan method, but seems 642 // that pointer aliasing analysis fails show that this is safe. 643 V8_INLINE void Scan(TokenDesc* next_desc); 644 645 V8_INLINE Token::Value SkipWhiteSpace(); 646 Token::Value SkipSingleHTMLComment(); 647 Token::Value SkipSingleLineComment(); 648 Token::Value SkipSourceURLComment(); 649 void TryToParseSourceURLComment(); 650 Token::Value SkipMultiLineComment(); 651 // Scans a possible HTML comment -- begins with '<!'. 652 Token::Value ScanHtmlComment(); 653 654 bool ScanDigitsWithNumericSeparators(bool (*predicate)(base::uc32 ch), 655 bool is_check_first_digit); 656 bool ScanDecimalDigits(bool allow_numeric_separator); 657 // Optimized function to scan decimal number as Smi. 658 bool ScanDecimalAsSmi(uint64_t* value, bool allow_numeric_separator); 659 bool ScanDecimalAsSmiWithNumericSeparators(uint64_t* value); 660 bool ScanHexDigits(); 661 bool ScanBinaryDigits(); 662 bool ScanSignedInteger(); 663 bool ScanOctalDigits(); 664 bool ScanImplicitOctalDigits(int start_pos, NumberKind* kind); 665 666 Token::Value ScanNumber(bool seen_period); 667 V8_INLINE Token::Value ScanIdentifierOrKeyword(); 668 V8_INLINE Token::Value ScanIdentifierOrKeywordInner(); 669 Token::Value ScanIdentifierOrKeywordInnerSlow(bool escaped, 670 bool can_be_keyword); 671 672 Token::Value ScanString(); 673 Token::Value ScanPrivateName(); 674 675 // Scans an escape-sequence which is part of a string and adds the 676 // decoded character to the current literal. Returns true if a pattern 677 // is scanned. 678 template <bool capture_raw> 679 bool ScanEscape(); 680 681 // Decodes a Unicode escape-sequence which is part of an identifier. 682 // If the escape sequence cannot be decoded the result is kBadChar. 683 base::uc32 ScanIdentifierUnicodeEscape(); 684 // Helper for the above functions. 685 template <bool capture_raw> 686 base::uc32 ScanUnicodeEscape(); 687 688 Token::Value ScanTemplateSpan(); 689 690 // Return the current source position. source_pos()691 int source_pos() { 692 return static_cast<int>(source_->pos()) - kCharacterLookaheadBufferSize; 693 } 694 LiteralContainsEscapes(const TokenDesc & token)695 static bool LiteralContainsEscapes(const TokenDesc& token) { 696 Location location = token.location; 697 int source_length = (location.end_pos - location.beg_pos); 698 if (token.token == Token::STRING) { 699 // Subtract delimiters. 700 source_length -= 2; 701 } 702 return token.literal_chars.length() != source_length; 703 } 704 705 #ifdef DEBUG 706 void SanityCheckTokenDesc(const TokenDesc&) const; 707 #endif 708 next()709 TokenDesc& next() { return *next_; } 710 current()711 const TokenDesc& current() const { return *current_; } next()712 const TokenDesc& next() const { return *next_; } next_next()713 const TokenDesc& next_next() const { return *next_next_; } 714 715 UnoptimizedCompileFlags flags_; 716 717 TokenDesc* current_; // desc for current token (as returned by Next()) 718 TokenDesc* next_; // desc for next token (one token look-ahead) 719 TokenDesc* next_next_; // desc for the token after next (after PeakAhead()) 720 721 // Input stream. Must be initialized to an Utf16CharacterStream. 722 Utf16CharacterStream* const source_; 723 724 // One Unicode character look-ahead; c0_ < 0 at the end of the input. 725 base::uc32 c0_; 726 727 TokenDesc token_storage_[3]; 728 729 // Whether this scanner encountered an HTML comment. 730 bool found_html_comment_; 731 732 // Values parsed from magic comments. 733 LiteralBuffer source_url_; 734 LiteralBuffer source_mapping_url_; 735 736 // Last-seen positions of potentially problematic tokens. 737 Location octal_pos_; 738 MessageTemplate octal_message_; 739 740 MessageTemplate scanner_error_; 741 Location scanner_error_location_; 742 }; 743 744 } // namespace internal 745 } // namespace v8 746 747 #endif // V8_PARSING_SCANNER_H_ 748