LCOV - code coverage report
Current view: top level - src/parsing - scanner.h (source / functions) Hit Total Coverage
Test: app.info Lines: 157 158 99.4 %
Date: 2017-04-26 Functions: 21 23 91.3 %

          Line data    Source code
       1             : // Copyright 2011 the V8 project authors. All rights reserved.
       2             : // Use of this source code is governed by a BSD-style license that can be
       3             : // found in the LICENSE file.
       4             : 
       5             : // Features shared by parsing and pre-parsing scanners.
       6             : 
       7             : #ifndef V8_PARSING_SCANNER_H_
       8             : #define V8_PARSING_SCANNER_H_
       9             : 
      10             : #include "src/allocation.h"
      11             : #include "src/base/logging.h"
      12             : #include "src/char-predicates.h"
      13             : #include "src/globals.h"
      14             : #include "src/messages.h"
      15             : #include "src/parsing/token.h"
      16             : #include "src/unicode-decoder.h"
      17             : #include "src/unicode.h"
      18             : 
      19             : namespace v8 {
      20             : namespace internal {
      21             : 
      22             : 
      23             : class AstRawString;
      24             : class AstValueFactory;
      25             : class DuplicateFinder;
      26             : class ExternalOneByteString;
      27             : class ExternalTwoByteString;
      28             : class ParserRecorder;
      29             : class UnicodeCache;
      30             : 
      31             : // ---------------------------------------------------------------------
      32             : // Buffered stream of UTF-16 code units, using an internal UTF-16 buffer.
      33             : // A code unit is a 16 bit value representing either a 16 bit code point
      34             : // or one part of a surrogate pair that make a single 21 bit code point.
      35             : class Utf16CharacterStream {
      36             :  public:
      37             :   static const uc32 kEndOfInput = -1;
      38             : 
      39     4402280 :   virtual ~Utf16CharacterStream() { }
      40             : 
      41             :   // Returns and advances past the next UTF-16 code unit in the input
      42             :   // stream. If there are no more code units it returns kEndOfInput.
      43  2712325074 :   inline uc32 Advance() {
      44  2712325074 :     if (V8_LIKELY(buffer_cursor_ < buffer_end_)) {
      45  2699733532 :       return static_cast<uc32>(*(buffer_cursor_++));
      46    12591542 :     } else if (ReadBlock()) {
      47     8988618 :       return static_cast<uc32>(*(buffer_cursor_++));
      48             :     } else {
      49             :       // Note: currently the following increment is necessary to avoid a
      50             :       // parser problem! The scanner treats the final kEndOfInput as
      51             :       // a code unit with a position, and does math relative to that
      52             :       // position.
      53     3602929 :       buffer_cursor_++;
      54     3602929 :       return kEndOfInput;
      55             :     }
      56             :   }
      57             : 
      58             :   // Go back one by one character in the input stream.
      59             :   // This undoes the most recent Advance().
      60     7683605 :   inline void Back() {
      61             :     // The common case - if the previous character is within
      62             :     // buffer_start_ .. buffer_end_ will be handles locally.
      63             :     // Otherwise, a new block is requested.
      64     7683605 :     if (V8_LIKELY(buffer_cursor_ > buffer_start_)) {
      65     7683600 :       buffer_cursor_--;
      66             :     } else {
      67           5 :       ReadBlockAt(pos() - 1);
      68             :     }
      69     7683605 :   }
      70             : 
      71             :   // Go back one by two characters in the input stream. (This is the same as
      72             :   // calling Back() twice. But Back() may - in some instances - do substantial
      73             :   // work. Back2() guarantees this work will be done only once.)
      74          96 :   inline void Back2() {
      75          96 :     if (V8_LIKELY(buffer_cursor_ - 2 >= buffer_start_)) {
      76          67 :       buffer_cursor_ -= 2;
      77             :     } else {
      78          29 :       ReadBlockAt(pos() - 2);
      79             :     }
      80          96 :   }
      81             : 
      82             :   inline size_t pos() const {
      83  1412474358 :     return buffer_pos_ + (buffer_cursor_ - buffer_start_);
      84             :   }
      85             : 
      86         473 :   inline void Seek(size_t pos) {
      87         473 :     if (V8_LIKELY(pos >= buffer_pos_ &&
      88             :                   pos < (buffer_pos_ + (buffer_end_ - buffer_start_)))) {
      89         276 :       buffer_cursor_ = buffer_start_ + (pos - buffer_pos_);
      90             :     } else {
      91             :       ReadBlockAt(pos);
      92             :     }
      93         473 :   }
      94             : 
      95             :  protected:
      96             :   Utf16CharacterStream(const uint16_t* buffer_start,
      97             :                        const uint16_t* buffer_cursor,
      98             :                        const uint16_t* buffer_end, size_t buffer_pos)
      99             :       : buffer_start_(buffer_start),
     100             :         buffer_cursor_(buffer_cursor),
     101             :         buffer_end_(buffer_end),
     102     4402289 :         buffer_pos_(buffer_pos) {}
     103             :   Utf16CharacterStream() : Utf16CharacterStream(nullptr, nullptr, nullptr, 0) {}
     104             : 
     105             :   void ReadBlockAt(size_t new_pos) {
     106             :     // The callers of this method (Back/Back2/Seek) should handle the easy
     107             :     // case (seeking within the current buffer), and we should only get here
     108             :     // if we actually require new data.
     109             :     // (This is really an efficiency check, not a correctness invariant.)
     110             :     DCHECK(new_pos < buffer_pos_ ||
     111             :            new_pos >= buffer_pos_ + (buffer_end_ - buffer_start_));
     112             : 
     113             :     // Change pos() to point to new_pos.
     114         231 :     buffer_pos_ = new_pos;
     115         231 :     buffer_cursor_ = buffer_start_;
     116         231 :     bool success = ReadBlock();
     117             :     USE(success);
     118             : 
     119             :     // Post-conditions: 1, on success, we should be at the right position.
     120             :     //                  2, success == we should have more characters available.
     121             :     DCHECK_IMPLIES(success, pos() == new_pos);
     122             :     DCHECK_EQ(success, buffer_cursor_ < buffer_end_);
     123             :     DCHECK_EQ(success, buffer_start_ < buffer_end_);
     124             :   }
     125             : 
     126             :   // Read more data, and update buffer_*_ to point to it.
     127             :   // Returns true if more data was available.
     128             :   //
     129             :   // ReadBlock() may modify any of the buffer_*_ members, but must sure that
     130             :   // the result of pos() remains unaffected.
     131             :   //
     132             :   // Examples:
     133             :   // - a stream could either fill a separate buffer. Then buffer_start_ and
     134             :   //   buffer_cursor_ would point to the beginning of the buffer, and
     135             :   //   buffer_pos would be the old pos().
     136             :   // - a stream with existing buffer chunks would set buffer_start_ and
     137             :   //   buffer_end_ to cover the full chunk, and then buffer_cursor_ would
     138             :   //   point into the middle of the buffer, while buffer_pos_ would describe
     139             :   //   the start of the buffer.
     140             :   virtual bool ReadBlock() = 0;
     141             : 
     142             :   const uint16_t* buffer_start_;
     143             :   const uint16_t* buffer_cursor_;
     144             :   const uint16_t* buffer_end_;
     145             :   size_t buffer_pos_;
     146             : };
     147             : 
     148             : 
     149             : // ----------------------------------------------------------------------------
     150             : // JavaScript Scanner.
     151             : 
     152     7135462 : class Scanner {
     153             :  public:
     154             :   // Scoped helper for a re-settable bookmark.
     155             :   class BookmarkScope {
     156             :    public:
     157             :     explicit BookmarkScope(Scanner* scanner)
     158     1434202 :         : scanner_(scanner), bookmark_(kNoBookmark) {
     159             :       DCHECK_NOT_NULL(scanner_);
     160             :     }
     161             :     ~BookmarkScope() {}
     162             : 
     163             :     void Set();
     164             :     void Apply();
     165             :     bool HasBeenSet();
     166             :     bool HasBeenApplied();
     167             : 
     168             :    private:
     169             :     static const size_t kNoBookmark;
     170             :     static const size_t kBookmarkWasApplied;
     171             :     static const size_t kBookmarkAtFirstPos;
     172             : 
     173             :     Scanner* scanner_;
     174             :     size_t bookmark_;
     175             : 
     176             :     DISALLOW_COPY_AND_ASSIGN(BookmarkScope);
     177             :   };
     178             : 
     179             :   // Representation of an interval of source positions.
     180             :   struct Location {
     181    15815111 :     Location(int b, int e) : beg_pos(b), end_pos(e) { }
     182    39654997 :     Location() : beg_pos(0), end_pos(0) { }
     183             : 
     184             :     bool IsValid() const {
     185    37686283 :       return beg_pos >= 0 && end_pos >= beg_pos;
     186             :     }
     187             : 
     188             :     static Location invalid() { return Location(-1, -1); }
     189             : 
     190             :     int beg_pos;
     191             :     int end_pos;
     192             :   };
     193             : 
     194             :   // -1 is outside of the range of any real source code.
     195             :   static const int kNoOctalLocation = -1;
     196             :   static const uc32 kEndOfInput = Utf16CharacterStream::kEndOfInput;
     197             : 
     198             :   explicit Scanner(UnicodeCache* scanner_contants);
     199             : 
     200             :   void Initialize(Utf16CharacterStream* source);
     201             : 
     202             :   // Returns the next token and advances input.
     203             :   Token::Value Next();
     204             :   // Returns the token following peek()
     205             :   Token::Value PeekAhead();
     206             :   // Returns the current token again.
     207             :   Token::Value current_token() { return current_.token; }
     208             : 
     209             :   Token::Value current_contextual_token() { return current_.contextual_token; }
     210             :   Token::Value next_contextual_token() { return next_.contextual_token; }
     211             : 
     212             :   // Returns the location information for the current token
     213             :   // (the token last returned by Next()).
     214    10058037 :   Location location() const { return current_.location; }
     215             : 
     216             :   // This error is specifically an invalid hex or unicode escape sequence.
     217             :   bool has_error() const { return scanner_error_ != MessageTemplate::kNone; }
     218             :   MessageTemplate::Template error() const { return scanner_error_; }
     219             :   Location error_location() const { return scanner_error_location_; }
     220             : 
     221             :   bool has_invalid_template_escape() const {
     222             :     return current_.invalid_template_escape_message != MessageTemplate::kNone;
     223             :   }
     224             :   MessageTemplate::Template invalid_template_escape_message() const {
     225             :     DCHECK(has_invalid_template_escape());
     226             :     return current_.invalid_template_escape_message;
     227             :   }
     228             :   Location invalid_template_escape_location() const {
     229             :     DCHECK(has_invalid_template_escape());
     230             :     return current_.invalid_template_escape_location;
     231             :   }
     232             : 
     233             :   // Similar functions for the upcoming token.
     234             : 
     235             :   // One token look-ahead (past the token returned by Next()).
     236  3140315130 :   Token::Value peek() const { return next_.token; }
     237             : 
     238             :   Location peek_location() const { return next_.location; }
     239             : 
     240             :   bool literal_contains_escapes() const {
     241    32765409 :     return LiteralContainsEscapes(current_);
     242             :   }
     243             : 
     244             :   const AstRawString* CurrentSymbol(AstValueFactory* ast_value_factory) const;
     245             :   const AstRawString* NextSymbol(AstValueFactory* ast_value_factory) const;
     246             :   const AstRawString* CurrentRawSymbol(
     247             :       AstValueFactory* ast_value_factory) const;
     248             : 
     249             :   double DoubleValue();
     250             :   bool ContainsDot();
     251             : 
     252             :   inline bool CurrentMatches(Token::Value token) const {
     253             :     DCHECK(Token::IsKeyword(token));
     254             :     return current_.token == token;
     255             :   }
     256             : 
     257             :   inline bool CurrentMatchesContextual(Token::Value token) const {
     258             :     DCHECK(Token::IsContextualKeyword(token));
     259      124140 :     return current_.contextual_token == token;
     260             :   }
     261             : 
     262             :   // Match the token against the contextual keyword or literal buffer.
     263     4961349 :   inline bool CurrentMatchesContextualEscaped(Token::Value token) const {
     264             :     DCHECK(Token::IsContextualKeyword(token) || token == Token::LET);
     265             :     // Escaped keywords are not matched as tokens. So if we require escape
     266             :     // and/or string processing we need to look at the literal content
     267             :     // (which was escape-processed already).
     268             :     // Conveniently, current_.literal_chars == nullptr for all proper keywords,
     269             :     // so this second condition should exit early in common cases.
     270     4961349 :     return (current_.contextual_token == token) ||
     271     4790049 :            (current_.literal_chars &&
     272             :             current_.literal_chars->Equals(Vector<const char>(
     273    14541447 :                 Token::String(token), Token::StringLength(token))));
     274             :   }
     275             : 
     276     3323053 :   bool IsUseStrict() const {
     277     6646106 :     return current_.token == Token::STRING &&
     278             :            current_.literal_chars->Equals(
     279     6646106 :                Vector<const char>("use strict", strlen("use strict")));
     280             :   }
     281      124140 :   bool IsGetOrSet(bool* is_get, bool* is_set) const {
     282       62070 :     *is_get = CurrentMatchesContextual(Token::GET);
     283       62070 :     *is_set = CurrentMatchesContextual(Token::SET);
     284       62070 :     return *is_get || *is_set;
     285             :   }
     286       94373 :   bool IsLet() const {
     287       94373 :     return CurrentMatches(Token::LET) ||
     288       94373 :            CurrentMatchesContextualEscaped(Token::LET);
     289             :   }
     290             : 
     291             :   // Check whether the CurrentSymbol() has already been seen.
     292             :   // The DuplicateFinder holds the data, so different instances can be used
     293             :   // for different sets of duplicates to check for.
     294             :   bool IsDuplicateSymbol(DuplicateFinder* duplicate_finder,
     295             :                          AstValueFactory* ast_value_factory) const;
     296             : 
     297             :   UnicodeCache* unicode_cache() { return unicode_cache_; }
     298             : 
     299             :   // Returns the location of the last seen octal literal.
     300             :   Location octal_position() const { return octal_pos_; }
     301             :   void clear_octal_position() {
     302         555 :     octal_pos_ = Location::invalid();
     303         555 :     octal_message_ = MessageTemplate::kNone;
     304             :   }
     305             :   MessageTemplate::Template octal_message() const { return octal_message_; }
     306             : 
     307             :   // Returns the value of the last smi that was scanned.
     308             :   uint32_t smi_value() const { return current_.smi_value_; }
     309             : 
     310             :   // Seek forward to the given position.  This operation does not
     311             :   // work in general, for instance when there are pushed back
     312             :   // characters, but works for seeking forward until simple delimiter
     313             :   // tokens, which is what it is used for.
     314             :   void SeekForward(int pos);
     315             : 
     316             :   // Returns true if there was a line terminator before the peek'ed token,
     317             :   // possibly inside a multi-line comment.
     318             :   bool HasAnyLineTerminatorBeforeNext() const {
     319    96898602 :     return has_line_terminator_before_next_ ||
     320             :            has_multiline_comment_before_next_;
     321             :   }
     322             : 
     323             :   bool HasAnyLineTerminatorAfterNext() {
     324      272895 :     Token::Value ensure_next_next = PeekAhead();
     325             :     USE(ensure_next_next);
     326      272895 :     return has_line_terminator_after_next_;
     327             :   }
     328             : 
     329             :   // Scans the input as a regular expression pattern, next token must be /(=).
     330             :   // Returns true if a pattern is scanned.
     331             :   bool ScanRegExpPattern();
     332             :   // Scans the input as regular expression flags. Returns the flags on success.
     333             :   Maybe<RegExp::Flags> ScanRegExpFlags();
     334             : 
     335             :   // Scans the input as a template literal
     336             :   Token::Value ScanTemplateStart();
     337             :   Token::Value ScanTemplateContinuation();
     338             : 
     339             :   Handle<String> SourceUrl(Isolate* isolate) const;
     340             :   Handle<String> SourceMappingUrl(Isolate* isolate) const;
     341             : 
     342             :   bool FoundHtmlComment() const { return found_html_comment_; }
     343             : 
     344             :  private:
     345             :   // Scoped helper for saving & restoring scanner error state.
     346             :   // This is used for tagged template literals, in which normally forbidden
     347             :   // escape sequences are allowed.
     348             :   class ErrorState;
     349             : 
     350             :   // Scoped helper for literal recording. Automatically drops the literal
     351             :   // if aborting the scanning before it's complete.
     352             :   class LiteralScope {
     353             :    public:
     354   200166533 :     explicit LiteralScope(Scanner* self) : scanner_(self), complete_(false) {
     355             :       scanner_->StartLiteral();
     356             :     }
     357             :     ~LiteralScope() {
     358   254814312 :       if (!complete_) scanner_->DropLiteral();
     359             :     }
     360   142974246 :     void Complete() { complete_ = true; }
     361             : 
     362             :    private:
     363             :     Scanner* scanner_;
     364             :     bool complete_;
     365             :   };
     366             : 
     367             :   // LiteralBuffer -  Collector of chars of literals.
     368             :   class LiteralBuffer {
     369             :    public:
     370    32439974 :     LiteralBuffer() : is_one_byte_(true), position_(0), backing_store_() {}
     371             : 
     372             :     ~LiteralBuffer() { backing_store_.Dispose(); }
     373             : 
     374             :     INLINE(void AddChar(char code_unit)) {
     375             :       DCHECK(IsValidAscii(code_unit));
     376  1453197255 :       AddOneByteChar(static_cast<byte>(code_unit));
     377             :     }
     378             : 
     379             :     INLINE(void AddChar(uc32 code_unit)) {
     380   110859475 :       if (is_one_byte_ &&
     381             :           code_unit <= static_cast<uc32>(unibrow::Latin1::kMaxChar)) {
     382   109402304 :         AddOneByteChar(static_cast<byte>(code_unit));
     383             :       } else {
     384     1457069 :         AddCharSlow(code_unit);
     385             :       }
     386             :     }
     387             : 
     388             :     bool is_one_byte() const { return is_one_byte_; }
     389             : 
     390     8113102 :     bool Equals(Vector<const char> keyword) const {
     391     8720259 :       return is_one_byte() && keyword.length() == position_ &&
     392     8720259 :              (memcmp(keyword.start(), backing_store_.start(), position_) == 0);
     393             :     }
     394             : 
     395             :     Vector<const uint16_t> two_byte_literal() const {
     396             :       DCHECK(!is_one_byte_);
     397             :       DCHECK((position_ & 0x1) == 0);
     398             :       return Vector<const uint16_t>(
     399       66191 :           reinterpret_cast<const uint16_t*>(backing_store_.start()),
     400       66191 :           position_ >> 1);
     401             :     }
     402             : 
     403             :     Vector<const uint8_t> one_byte_literal() const {
     404             :       DCHECK(is_one_byte_);
     405             :       return Vector<const uint8_t>(
     406   332498551 :           reinterpret_cast<const uint8_t*>(backing_store_.start()), position_);
     407             :     }
     408             : 
     409    37273867 :     int length() const { return is_one_byte_ ? position_ : (position_ >> 1); }
     410             : 
     411      167621 :     void ReduceLength(int delta) {
     412      167621 :       position_ -= delta * (is_one_byte_ ? kOneByteSize : kUC16Size);
     413      167621 :     }
     414             : 
     415             :     void Reset() {
     416   255153721 :       position_ = 0;
     417   255153721 :       is_one_byte_ = true;
     418             :     }
     419             : 
     420             :     Handle<String> Internalize(Isolate* isolate) const;
     421             : 
     422             :    private:
     423             :     static const int kInitialCapacity = 16;
     424             :     static const int kGrowthFactory = 4;
     425             :     static const int kMinConversionSlack = 256;
     426             :     static const int kMaxGrowth = 1 * MB;
     427             : 
     428             :     inline bool IsValidAscii(char code_unit) {
     429             :       // Control characters and printable characters span the range of
     430             :       // valid ASCII characters (0-127). Chars are unsigned on some
     431             :       // platforms which causes compiler warnings if the validity check
     432             :       // tests the lower bound >= 0 as it's always true.
     433             :       return iscntrl(code_unit) || isprint(code_unit);
     434             :     }
     435             : 
     436             :     INLINE(void AddOneByteChar(byte one_byte_char)) {
     437             :       DCHECK(is_one_byte_);
     438  1562604825 :       if (position_ >= backing_store_.length()) ExpandBuffer();
     439  1562604876 :       backing_store_[position_] = one_byte_char;
     440  1562604900 :       position_ += kOneByteSize;
     441             :     }
     442             : 
     443             :     void AddCharSlow(uc32 code_unit);
     444             :     int NewCapacity(int min_capacity);
     445             :     void ExpandBuffer();
     446             :     void ConvertToTwoByte();
     447             : 
     448             :     bool is_one_byte_;
     449             :     int position_;
     450             :     Vector<byte> backing_store_;
     451             : 
     452             :     DISALLOW_COPY_AND_ASSIGN(LiteralBuffer);
     453             :   };
     454             : 
     455             :   // The current and look-ahead token.
     456             :   struct TokenDesc {
     457             :     Location location;
     458             :     LiteralBuffer* literal_chars;
     459             :     LiteralBuffer* raw_literal_chars;
     460             :     uint32_t smi_value_;
     461             :     Token::Value token;
     462             :     MessageTemplate::Template invalid_template_escape_message;
     463             :     Location invalid_template_escape_location;
     464             :     Token::Value contextual_token;
     465             :   };
     466             : 
     467             :   static const int kCharacterLookaheadBufferSize = 1;
     468             :   const int kMaxAscii = 127;
     469             : 
     470             :   // Scans octal escape sequence. Also accepts "\0" decimal escape sequence.
     471             :   template <bool capture_raw>
     472             :   uc32 ScanOctalEscape(uc32 c, int length);
     473             : 
     474             :   // Call this after setting source_ to the input.
     475     4050465 :   void Init() {
     476             :     // Set c0_ (one character ahead)
     477             :     STATIC_ASSERT(kCharacterLookaheadBufferSize == 1);
     478     4050465 :     Advance();
     479             :     // Initialize current_ to not refer to a literal.
     480     4050467 :     current_.token = Token::UNINITIALIZED;
     481     4050467 :     current_.contextual_token = Token::UNINITIALIZED;
     482     4050467 :     current_.literal_chars = NULL;
     483     4050467 :     current_.raw_literal_chars = NULL;
     484     4050467 :     current_.invalid_template_escape_message = MessageTemplate::kNone;
     485     4050467 :     next_.token = Token::UNINITIALIZED;
     486     4050467 :     next_.contextual_token = Token::UNINITIALIZED;
     487     4050467 :     next_.literal_chars = NULL;
     488     4050467 :     next_.raw_literal_chars = NULL;
     489     4050467 :     next_.invalid_template_escape_message = MessageTemplate::kNone;
     490     4050467 :     next_next_.token = Token::UNINITIALIZED;
     491     4050467 :     next_next_.contextual_token = Token::UNINITIALIZED;
     492     4050467 :     next_next_.literal_chars = NULL;
     493     4050467 :     next_next_.raw_literal_chars = NULL;
     494     4050467 :     next_next_.invalid_template_escape_message = MessageTemplate::kNone;
     495     4050467 :     found_html_comment_ = false;
     496     4050467 :     scanner_error_ = MessageTemplate::kNone;
     497     4050467 :   }
     498             : 
     499             :   void ReportScannerError(const Location& location,
     500       16665 :                           MessageTemplate::Template error) {
     501       16665 :     if (has_error()) return;
     502       16635 :     scanner_error_ = error;
     503       16635 :     scanner_error_location_ = location;
     504             :   }
     505             : 
     506       12384 :   void ReportScannerError(int pos, MessageTemplate::Template error) {
     507       12384 :     if (has_error()) return;
     508        9632 :     scanner_error_ = error;
     509        9632 :     scanner_error_location_ = Location(pos, pos + 1);
     510             :   }
     511             : 
     512             :   // Seek to the next_ token at the given position.
     513             :   void SeekNext(size_t position);
     514             : 
     515             :   // Literal buffer support
     516             :   inline void StartLiteral() {
     517             :     LiteralBuffer* free_buffer =
     518   254982523 :         (current_.literal_chars == &literal_buffer0_)
     519             :             ? &literal_buffer1_
     520   254212821 :             : (current_.literal_chars == &literal_buffer1_) ? &literal_buffer2_
     521   509195344 :                                                             : &literal_buffer0_;
     522             :     free_buffer->Reset();
     523   254982523 :     next_.literal_chars = free_buffer;
     524             :   }
     525             : 
     526             :   inline void StartRawLiteral() {
     527             :     LiteralBuffer* free_buffer =
     528      168651 :         (current_.raw_literal_chars == &raw_literal_buffer0_)
     529             :             ? &raw_literal_buffer1_
     530      166559 :             : (current_.raw_literal_chars == &raw_literal_buffer1_)
     531             :                   ? &raw_literal_buffer2_
     532      335210 :                   : &raw_literal_buffer0_;
     533             :     free_buffer->Reset();
     534      168651 :     next_.raw_literal_chars = free_buffer;
     535             :   }
     536             : 
     537             :   INLINE(void AddLiteralChar(uc32 c)) {
     538             :     DCHECK_NOT_NULL(next_.literal_chars);
     539             :     next_.literal_chars->AddChar(c);
     540             :   }
     541             : 
     542             :   INLINE(void AddLiteralChar(char c)) {
     543             :     DCHECK_NOT_NULL(next_.literal_chars);
     544             :     next_.literal_chars->AddChar(c);
     545             :   }
     546             : 
     547             :   INLINE(void AddRawLiteralChar(uc32 c)) {
     548             :     DCHECK_NOT_NULL(next_.raw_literal_chars);
     549             :     next_.raw_literal_chars->AddChar(c);
     550             :   }
     551             : 
     552             :   INLINE(void ReduceRawLiteralLength(int delta)) {
     553             :     DCHECK_NOT_NULL(next_.raw_literal_chars);
     554      167621 :     next_.raw_literal_chars->ReduceLength(delta);
     555             :   }
     556             : 
     557             :   // Stops scanning of a literal and drop the collected characters,
     558             :   // e.g., due to an encountered error.
     559             :   inline void DropLiteral() {
     560    57195633 :     next_.literal_chars = NULL;
     561    57195633 :     next_.raw_literal_chars = NULL;
     562             :   }
     563             : 
     564    46576926 :   inline void AddLiteralCharAdvance() {
     565    23288463 :     AddLiteralChar(c0_);
     566    23288465 :     Advance();
     567    23288465 :   }
     568             : 
     569             :   // Low-level scanning support.
     570             :   template <bool capture_raw = false, bool check_surrogate = true>
     571  1146077054 :   void Advance() {
     572             :     if (capture_raw) {
     573     1585275 :       AddRawLiteralChar(c0_);
     574             :     }
     575  2693402868 :     c0_ = source_->Advance();
     576  1144491620 :     if (check_surrogate) HandleLeadSurrogate();
     577  1144491689 :   }
     578             : 
     579  1184331696 :   void HandleLeadSurrogate() {
     580  2368663392 :     if (unibrow::Utf16::IsLeadSurrogate(c0_)) {
     581        1059 :       uc32 c1 = source_->Advance();
     582        1059 :       if (!unibrow::Utf16::IsTrailSurrogate(c1)) {
     583          48 :         source_->Back();
     584             :       } else {
     585        2022 :         c0_ = unibrow::Utf16::CombineSurrogatePair(c0_, c1);
     586             :       }
     587             :     }
     588  1184331696 :   }
     589             : 
     590             :   void PushBack(uc32 ch) {
     591     6787039 :     if (c0_ > static_cast<uc32>(unibrow::Utf16::kMaxNonSurrogateCharCode)) {
     592           0 :       source_->Back2();
     593             :     } else {
     594     6787039 :       source_->Back();
     595             :     }
     596     6787039 :     c0_ = ch;
     597             :   }
     598             : 
     599             :   // Same as PushBack(ch1); PushBack(ch2).
     600             :   // - Potentially more efficient as it uses Back2() on the stream.
     601             :   // - Uses char as parameters, since we're only calling it with ASCII chars in
     602             :   //   practice. This way, we can avoid a few edge cases.
     603             :   void PushBack2(char ch1, char ch2) {
     604          96 :     source_->Back2();
     605          96 :     c0_ = ch2;
     606             :   }
     607             : 
     608             :   inline Token::Value Select(Token::Value tok) {
     609    42817716 :     Advance();
     610             :     return tok;
     611             :   }
     612             : 
     613             :   inline Token::Value Select(uc32 next, Token::Value then, Token::Value else_) {
     614     7832551 :     Advance();
     615     7832550 :     if (c0_ == next) {
     616     3953602 :       Advance();
     617             :       return then;
     618             :     } else {
     619             :       return else_;
     620             :     }
     621             :   }
     622             :   // Returns the literal string, if any, for the current token (the
     623             :   // token last returned by Next()). The string is 0-terminated.
     624             :   // Literal strings are collected for identifiers, strings, numbers as well
     625             :   // as for template literals. For template literals we also collect the raw
     626             :   // form.
     627             :   // These functions only give the correct result if the literal was scanned
     628             :   // when a LiteralScope object is alive.
     629             :   //
     630             :   // Current usage of these functions is unfortunately a little undisciplined,
     631             :   // and is_literal_one_byte() + is_literal_one_byte_string() is also
     632             :   // requested for tokens that do not have a literal. Hence, we treat any
     633             :   // token as a one-byte literal. E.g. Token::FUNCTION pretends to have a
     634             :   // literal "function".
     635             :   Vector<const uint8_t> literal_one_byte_string() const {
     636   146842781 :     if (current_.literal_chars)
     637             :       return current_.literal_chars->one_byte_literal();
     638       88783 :     const char* str = Token::String(current_.token);
     639             :     const uint8_t* str_as_uint8 = reinterpret_cast<const uint8_t*>(str);
     640             :     return Vector<const uint8_t>(str_as_uint8,
     641       88783 :                                  Token::StringLength(current_.token));
     642             :   }
     643             :   Vector<const uint16_t> literal_two_byte_string() const {
     644             :     DCHECK_NOT_NULL(current_.literal_chars);
     645             :     return current_.literal_chars->two_byte_literal();
     646             :   }
     647             :   bool is_literal_one_byte() const {
     648   142597687 :     return !current_.literal_chars || current_.literal_chars->is_one_byte();
     649             :   }
     650             :   int literal_length() const {
     651             :     if (current_.literal_chars) return current_.literal_chars->length();
     652             :     return Token::StringLength(current_.token);
     653             :   }
     654             :   // Returns the literal string for the next token (the token that
     655             :   // would be returned if Next() were called).
     656             :   Vector<const uint8_t> next_literal_one_byte_string() const {
     657             :     DCHECK_NOT_NULL(next_.literal_chars);
     658             :     return next_.literal_chars->one_byte_literal();
     659             :   }
     660             :   Vector<const uint16_t> next_literal_two_byte_string() const {
     661             :     DCHECK_NOT_NULL(next_.literal_chars);
     662             :     return next_.literal_chars->two_byte_literal();
     663             :   }
     664             :   bool is_next_literal_one_byte() const {
     665             :     DCHECK_NOT_NULL(next_.literal_chars);
     666       98348 :     return next_.literal_chars->is_one_byte();
     667             :   }
     668             :   Vector<const uint8_t> raw_literal_one_byte_string() const {
     669             :     DCHECK_NOT_NULL(current_.raw_literal_chars);
     670             :     return current_.raw_literal_chars->one_byte_literal();
     671             :   }
     672             :   Vector<const uint16_t> raw_literal_two_byte_string() const {
     673             :     DCHECK_NOT_NULL(current_.raw_literal_chars);
     674             :     return current_.raw_literal_chars->two_byte_literal();
     675             :   }
     676             :   bool is_raw_literal_one_byte() const {
     677             :     DCHECK_NOT_NULL(current_.raw_literal_chars);
     678       70792 :     return current_.raw_literal_chars->is_one_byte();
     679             :   }
     680             : 
     681             :   template <bool capture_raw, bool unicode = false>
     682             :   uc32 ScanHexNumber(int expected_length);
     683             :   // Scan a number of any length but not bigger than max_value. For example, the
     684             :   // number can be 000000001, so it's very long in characters but its value is
     685             :   // small.
     686             :   template <bool capture_raw>
     687             :   uc32 ScanUnlimitedLengthHexNumber(int max_value, int beg_pos);
     688             : 
     689             :   // Scans a single JavaScript token.
     690             :   void Scan();
     691             : 
     692             :   bool SkipWhiteSpace();
     693             :   Token::Value SkipSingleLineComment();
     694             :   Token::Value SkipSourceURLComment();
     695             :   void TryToParseSourceURLComment();
     696             :   Token::Value SkipMultiLineComment();
     697             :   // Scans a possible HTML comment -- begins with '<!'.
     698             :   Token::Value ScanHtmlComment();
     699             : 
     700             :   void ScanDecimalDigits();
     701             :   Token::Value ScanNumber(bool seen_period);
     702             :   Token::Value ScanIdentifierOrKeyword();
     703             :   Token::Value ScanIdentifierSuffix(LiteralScope* literal, bool escaped);
     704             : 
     705             :   Token::Value ScanString();
     706             : 
     707             :   // Scans an escape-sequence which is part of a string and adds the
     708             :   // decoded character to the current literal. Returns true if a pattern
     709             :   // is scanned.
     710             :   template <bool capture_raw, bool in_template_literal>
     711             :   bool ScanEscape();
     712             : 
     713             :   // Decodes a Unicode escape-sequence which is part of an identifier.
     714             :   // If the escape sequence cannot be decoded the result is kBadChar.
     715             :   uc32 ScanIdentifierUnicodeEscape();
     716             :   // Helper for the above functions.
     717             :   template <bool capture_raw>
     718       24981 :   uc32 ScanUnicodeEscape();
     719             : 
     720             :   Token::Value ScanTemplateSpan();
     721             : 
     722             :   // Return the current source position.
     723             :   int source_pos() {
     724  1395685892 :     return static_cast<int>(source_->pos()) - kCharacterLookaheadBufferSize;
     725             :   }
     726             : 
     727    32765409 :   static bool LiteralContainsEscapes(const TokenDesc& token) {
     728    32765409 :     Location location = token.location;
     729    32765409 :     int source_length = (location.end_pos - location.beg_pos);
     730    32765409 :     if (token.token == Token::STRING) {
     731             :       // Subtract delimiters.
     732      172688 :       source_length -= 2;
     733             :     }
     734    65512281 :     return token.literal_chars &&
     735    32765409 :            (token.literal_chars->length() != source_length);
     736             :   }
     737             : 
     738             : #ifdef DEBUG
     739             :   void SanityCheckTokenDesc(const TokenDesc&) const;
     740             : #endif
     741             : 
     742             :   UnicodeCache* unicode_cache_;
     743             : 
     744             :   // Buffers collecting literal strings, numbers, etc.
     745             :   LiteralBuffer literal_buffer0_;
     746             :   LiteralBuffer literal_buffer1_;
     747             :   LiteralBuffer literal_buffer2_;
     748             : 
     749             :   // Values parsed from magic comments.
     750             :   LiteralBuffer source_url_;
     751             :   LiteralBuffer source_mapping_url_;
     752             : 
     753             :   // Buffer to store raw string values
     754             :   LiteralBuffer raw_literal_buffer0_;
     755             :   LiteralBuffer raw_literal_buffer1_;
     756             :   LiteralBuffer raw_literal_buffer2_;
     757             : 
     758             :   TokenDesc current_;    // desc for current token (as returned by Next())
     759             :   TokenDesc next_;       // desc for next token (one token look-ahead)
     760             :   TokenDesc next_next_;  // desc for the token after next (after PeakAhead())
     761             : 
     762             :   // Input stream. Must be initialized to an Utf16CharacterStream.
     763             :   Utf16CharacterStream* source_;
     764             : 
     765             :   // Last-seen positions of potentially problematic tokens.
     766             :   Location octal_pos_;
     767             :   MessageTemplate::Template octal_message_;
     768             : 
     769             :   // One Unicode character look-ahead; c0_ < 0 at the end of the input.
     770             :   uc32 c0_;
     771             : 
     772             :   // Whether there is a line terminator whitespace character after
     773             :   // the current token, and  before the next. Does not count newlines
     774             :   // inside multiline comments.
     775             :   bool has_line_terminator_before_next_;
     776             :   // Whether there is a multi-line comment that contains a
     777             :   // line-terminator after the current token, and before the next.
     778             :   bool has_multiline_comment_before_next_;
     779             :   bool has_line_terminator_after_next_;
     780             : 
     781             :   // Whether this scanner encountered an HTML comment.
     782             :   bool found_html_comment_;
     783             : 
     784             :   MessageTemplate::Template scanner_error_;
     785             :   Location scanner_error_location_;
     786             : };
     787             : 
     788             : }  // namespace internal
     789             : }  // namespace v8
     790             : 
     791             : #endif  // V8_PARSING_SCANNER_H_

Generated by: LCOV version 1.10