Line data Source code
1 : // Copyright 2011 the V8 project authors. All rights reserved.
2 : // Use of this source code is governed by a BSD-style license that can be
3 : // found in the LICENSE file.
4 :
5 : // Features shared by parsing and pre-parsing scanners.
6 :
7 : #ifndef V8_PARSING_SCANNER_H_
8 : #define V8_PARSING_SCANNER_H_
9 :
10 : #include <algorithm>
11 :
12 : #include "src/allocation.h"
13 : #include "src/base/logging.h"
14 : #include "src/char-predicates.h"
15 : #include "src/globals.h"
16 : #include "src/message-template.h"
17 : #include "src/parsing/token.h"
18 : #include "src/pointer-with-payload.h"
19 : #include "src/unicode-decoder.h"
20 : #include "src/unicode.h"
21 :
22 : namespace v8 {
23 : namespace internal {
24 :
25 : class AstRawString;
26 : class AstValueFactory;
27 : class ExternalOneByteString;
28 : class ExternalTwoByteString;
29 : class ParserRecorder;
30 : class RuntimeCallStats;
31 : class Zone;
32 :
33 : // ---------------------------------------------------------------------
34 : // Buffered stream of UTF-16 code units, using an internal UTF-16 buffer.
35 : // A code unit is a 16 bit value representing either a 16 bit code point
36 : // or one part of a surrogate pair that make a single 21 bit code point.
37 : class Utf16CharacterStream {
38 : public:
39 : static const uc32 kEndOfInput = -1;
40 :
41 2952825 : virtual ~Utf16CharacterStream() = default;
42 :
43 : V8_INLINE void set_parser_error() {
44 677390 : buffer_cursor_ = buffer_end_;
45 677390 : has_parser_error_ = true;
46 : }
47 43007 : V8_INLINE void reset_parser_error_flag() { has_parser_error_ = false; }
48 : V8_INLINE bool has_parser_error() const { return has_parser_error_; }
49 :
50 833228489 : inline uc32 Peek() {
51 833228489 : if (V8_LIKELY(buffer_cursor_ < buffer_end_)) {
52 826478761 : return static_cast<uc32>(*buffer_cursor_);
53 6750014 : } else if (ReadBlockChecked()) {
54 4430316 : return static_cast<uc32>(*buffer_cursor_);
55 : } else {
56 : return kEndOfInput;
57 : }
58 : }
59 :
60 : // Returns and advances past the next UTF-16 code unit in the input
61 : // stream. If there are no more code units it returns kEndOfInput.
62 : inline uc32 Advance() {
63 827183920 : uc32 result = Peek();
64 827171535 : buffer_cursor_++;
65 : return result;
66 : }
67 :
68 : // Returns and advances past the next UTF-16 code unit in the input stream
69 : // that meets the checks requirement. If there are no more code units it
70 : // returns kEndOfInput.
71 : template <typename FunctionType>
72 : V8_INLINE uc32 AdvanceUntil(FunctionType check) {
73 : while (true) {
74 1121186777 : for (; buffer_cursor_ < buffer_end_; ++buffer_cursor_) {
75 1267049501 : uc32 c0_ = static_cast<uc32>(*buffer_cursor_);
76 1267049501 : if (check(c0_)) {
77 146017414 : buffer_cursor_++;
78 : return c0_;
79 : }
80 : }
81 :
82 : DCHECK_EQ(buffer_cursor_, buffer_end_);
83 2207004 : if (!ReadBlockChecked()) {
84 114965 : buffer_cursor_++;
85 : return kEndOfInput;
86 : }
87 : }
88 : }
89 :
90 : // Go back one by one character in the input stream.
91 : // This undoes the most recent Advance().
92 6199984 : inline void Back() {
93 : // The common case - if the previous character is within
94 : // buffer_start_ .. buffer_end_ will be handles locally.
95 : // Otherwise, a new block is requested.
96 6199984 : if (V8_LIKELY(buffer_cursor_ > buffer_start_)) {
97 5996059 : buffer_cursor_--;
98 : } else {
99 203925 : ReadBlockAt(pos() - 1);
100 : }
101 6199984 : }
102 :
103 : inline size_t pos() const {
104 1373827197 : return buffer_pos_ + (buffer_cursor_ - buffer_start_);
105 : }
106 :
107 111938 : inline void Seek(size_t pos) {
108 111938 : if (V8_LIKELY(pos >= buffer_pos_ &&
109 : pos < (buffer_pos_ + (buffer_end_ - buffer_start_)))) {
110 95529 : buffer_cursor_ = buffer_start_ + (pos - buffer_pos_);
111 : } else {
112 : ReadBlockAt(pos);
113 : }
114 111938 : }
115 :
116 : // Returns true if the stream could access the V8 heap after construction.
117 66 : bool can_be_cloned_for_parallel_access() const {
118 66 : return can_be_cloned() && !can_access_heap();
119 : }
120 :
121 : // Returns true if the stream can be cloned with Clone.
122 : // TODO(rmcilroy): Remove this once ChunkedStreams can be cloned.
123 : virtual bool can_be_cloned() const = 0;
124 :
125 : // Clones the character stream to enable another independent scanner to access
126 : // the same underlying stream.
127 : virtual std::unique_ptr<Utf16CharacterStream> Clone() const = 0;
128 :
129 : // Returns true if the stream could access the V8 heap after construction.
130 : virtual bool can_access_heap() const = 0;
131 :
132 : RuntimeCallStats* runtime_call_stats() const { return runtime_call_stats_; }
133 : void set_runtime_call_stats(RuntimeCallStats* runtime_call_stats) {
134 13122 : runtime_call_stats_ = runtime_call_stats;
135 : }
136 :
137 : protected:
138 : Utf16CharacterStream(const uint16_t* buffer_start,
139 : const uint16_t* buffer_cursor,
140 : const uint16_t* buffer_end, size_t buffer_pos)
141 : : buffer_start_(buffer_start),
142 : buffer_cursor_(buffer_cursor),
143 : buffer_end_(buffer_end),
144 2952851 : buffer_pos_(buffer_pos) {}
145 : Utf16CharacterStream() : Utf16CharacterStream(nullptr, nullptr, nullptr, 0) {}
146 :
147 9177023 : bool ReadBlockChecked() {
148 : size_t position = pos();
149 : USE(position);
150 9177023 : bool success = !has_parser_error() && ReadBlock();
151 :
152 : // Post-conditions: 1, We should always be at the right position.
153 : // 2, Cursor should be inside the buffer.
154 : // 3, We should have more characters available iff success.
155 : DCHECK_EQ(pos(), position);
156 : DCHECK_LE(buffer_cursor_, buffer_end_);
157 : DCHECK_LE(buffer_start_, buffer_cursor_);
158 : DCHECK_EQ(success, buffer_cursor_ < buffer_end_);
159 2206978 : return success;
160 : }
161 :
162 : void ReadBlockAt(size_t new_pos) {
163 : // The callers of this method (Back/Back2/Seek) should handle the easy
164 : // case (seeking within the current buffer), and we should only get here
165 : // if we actually require new data.
166 : // (This is really an efficiency check, not a correctness invariant.)
167 : DCHECK(new_pos < buffer_pos_ ||
168 : new_pos >= buffer_pos_ + (buffer_end_ - buffer_start_));
169 :
170 : // Change pos() to point to new_pos.
171 220334 : buffer_pos_ = new_pos;
172 220334 : buffer_cursor_ = buffer_start_;
173 : DCHECK_EQ(pos(), new_pos);
174 : ReadBlockChecked();
175 : }
176 :
177 : // Read more data, and update buffer_*_ to point to it.
178 : // Returns true if more data was available.
179 : //
180 : // ReadBlock() may modify any of the buffer_*_ members, but must sure that
181 : // the result of pos() remains unaffected.
182 : //
183 : // Examples:
184 : // - a stream could either fill a separate buffer. Then buffer_start_ and
185 : // buffer_cursor_ would point to the beginning of the buffer, and
186 : // buffer_pos would be the old pos().
187 : // - a stream with existing buffer chunks would set buffer_start_ and
188 : // buffer_end_ to cover the full chunk, and then buffer_cursor_ would
189 : // point into the middle of the buffer, while buffer_pos_ would describe
190 : // the start of the buffer.
191 : virtual bool ReadBlock() = 0;
192 :
193 : const uint16_t* buffer_start_;
194 : const uint16_t* buffer_cursor_;
195 : const uint16_t* buffer_end_;
196 : size_t buffer_pos_;
197 : RuntimeCallStats* runtime_call_stats_;
198 : bool has_parser_error_ = false;
199 : };
200 :
201 : // ----------------------------------------------------------------------------
202 : // JavaScript Scanner.
203 :
204 5904262 : class Scanner {
205 : public:
206 : // Scoped helper for a re-settable bookmark.
207 : class BookmarkScope {
208 : public:
209 2446283 : explicit BookmarkScope(Scanner* scanner)
210 : : scanner_(scanner),
211 : bookmark_(kNoBookmark),
212 4892566 : had_parser_error_(scanner->has_parser_error()) {
213 : DCHECK_NOT_NULL(scanner_);
214 : }
215 : ~BookmarkScope() = default;
216 :
217 : void Set(size_t bookmark);
218 : void Apply();
219 : bool HasBeenSet() const;
220 : bool HasBeenApplied() const;
221 :
222 : private:
223 : static const size_t kNoBookmark;
224 : static const size_t kBookmarkWasApplied;
225 :
226 : Scanner* scanner_;
227 : size_t bookmark_;
228 : bool had_parser_error_;
229 :
230 : DISALLOW_COPY_AND_ASSIGN(BookmarkScope);
231 : };
232 :
233 : // Sets the Scanner into an error state to stop further scanning and terminate
234 : // the parsing by only returning ILLEGAL tokens after that.
235 3739739 : V8_INLINE void set_parser_error() {
236 3739739 : if (!has_parser_error()) {
237 677390 : c0_ = kEndOfInput;
238 : source_->set_parser_error();
239 677390 : for (TokenDesc& desc : token_storage_) desc.token = Token::ILLEGAL;
240 : }
241 : }
242 : V8_INLINE void reset_parser_error_flag() {
243 : source_->reset_parser_error_flag();
244 : }
245 : V8_INLINE bool has_parser_error() const {
246 46907363 : return source_->has_parser_error();
247 : }
248 :
249 : // Representation of an interval of source positions.
250 : struct Location {
251 51659297 : Location(int b, int e) : beg_pos(b), end_pos(e) { }
252 233629034 : Location() : beg_pos(0), end_pos(0) { }
253 :
254 2618059 : int length() const { return end_pos - beg_pos; }
255 499623409 : bool IsValid() const { return IsInRange(beg_pos, 0, end_pos); }
256 :
257 : static Location invalid() { return Location(-1, 0); }
258 :
259 : int beg_pos;
260 : int end_pos;
261 : };
262 :
263 : // -1 is outside of the range of any real source code.
264 : static const int kNoOctalLocation = -1;
265 : static const uc32 kEndOfInput = Utf16CharacterStream::kEndOfInput;
266 :
267 : explicit Scanner(Utf16CharacterStream* source, bool is_module);
268 :
269 : void Initialize();
270 :
271 : // Returns the next token and advances input.
272 : Token::Value Next();
273 : // Returns the token following peek()
274 : Token::Value PeekAhead();
275 : // Returns the current token again.
276 66350461 : Token::Value current_token() const { return current().token; }
277 :
278 : // Returns the location information for the current token
279 : // (the token last returned by Next()).
280 206329877 : const Location& location() const { return current().location; }
281 :
282 : // This error is specifically an invalid hex or unicode escape sequence.
283 : bool has_error() const { return scanner_error_ != MessageTemplate::kNone; }
284 : MessageTemplate error() const { return scanner_error_; }
285 : const Location& error_location() const { return scanner_error_location_; }
286 :
287 153405 : bool has_invalid_template_escape() const {
288 153405 : return current().invalid_template_escape_message != MessageTemplate::kNone;
289 : }
290 : MessageTemplate invalid_template_escape_message() const {
291 : DCHECK(has_invalid_template_escape());
292 : return current().invalid_template_escape_message;
293 : }
294 :
295 : void clear_invalid_template_escape_message() {
296 : DCHECK(has_invalid_template_escape());
297 13009 : current_->invalid_template_escape_message = MessageTemplate::kNone;
298 : }
299 :
300 : Location invalid_template_escape_location() const {
301 : DCHECK(has_invalid_template_escape());
302 6358 : return current().invalid_template_escape_location;
303 : }
304 :
305 : // Similar functions for the upcoming token.
306 :
307 : // One token look-ahead (past the token returned by Next()).
308 2161364234 : Token::Value peek() const { return next().token; }
309 :
310 636064154 : const Location& peek_location() const { return next().location; }
311 :
312 49021777 : bool literal_contains_escapes() const {
313 49021777 : return LiteralContainsEscapes(current());
314 : }
315 :
316 : const AstRawString* CurrentSymbol(AstValueFactory* ast_value_factory) const;
317 :
318 : const AstRawString* NextSymbol(AstValueFactory* ast_value_factory) const;
319 : const AstRawString* CurrentRawSymbol(
320 : AstValueFactory* ast_value_factory) const;
321 :
322 : double DoubleValue();
323 :
324 : const char* CurrentLiteralAsCString(Zone* zone) const;
325 :
326 : inline bool CurrentMatches(Token::Value token) const {
327 : DCHECK(Token::IsKeyword(token));
328 : return current().token == token;
329 : }
330 :
331 : template <size_t N>
332 2621457 : bool NextLiteralEquals(const char (&s)[N]) {
333 : DCHECK_EQ(Token::STRING, peek());
334 : // The length of the token is used to make sure the literal equals without
335 : // taking escape sequences (e.g., "use \x73trict") or line continuations
336 : // (e.g., "use \(newline) strict") into account.
337 2621457 : if (!is_next_literal_one_byte()) return false;
338 2618059 : if (peek_location().length() != N + 1) return false;
339 :
340 : Vector<const uint8_t> next = next_literal_one_byte_string();
341 : const char* chars = reinterpret_cast<const char*>(next.start());
342 429927 : return next.length() == N - 1 && strncmp(s, chars, N - 1) == 0;
343 : }
344 :
345 : // Returns the location of the last seen octal literal.
346 : Location octal_position() const { return octal_pos_; }
347 : void clear_octal_position() {
348 1322 : octal_pos_ = Location::invalid();
349 1322 : octal_message_ = MessageTemplate::kNone;
350 : }
351 : MessageTemplate octal_message() const { return octal_message_; }
352 :
353 : // Returns the value of the last smi that was scanned.
354 21195718 : uint32_t smi_value() const { return current().smi_value_; }
355 :
356 : // Seek forward to the given position. This operation does not
357 : // work in general, for instance when there are pushed back
358 : // characters, but works for seeking forward until simple delimiter
359 : // tokens, which is what it is used for.
360 : void SeekForward(int pos);
361 :
362 : // Returns true if there was a line terminator before the peek'ed token,
363 : // possibly inside a multi-line comment.
364 121986640 : bool HasLineTerminatorBeforeNext() const {
365 121986640 : return next().after_line_terminator;
366 : }
367 :
368 157865 : bool HasLineTerminatorAfterNext() {
369 157866 : Token::Value ensure_next_next = PeekAhead();
370 : USE(ensure_next_next);
371 157865 : return next_next().after_line_terminator;
372 : }
373 :
374 : // Scans the input as a regular expression pattern, next token must be /(=).
375 : // Returns true if a pattern is scanned.
376 : bool ScanRegExpPattern();
377 : // Scans the input as regular expression flags. Returns the flags on success.
378 : Maybe<RegExp::Flags> ScanRegExpFlags();
379 :
380 : // Scans the input as a template literal
381 : Token::Value ScanTemplateContinuation() {
382 : DCHECK_EQ(next().token, Token::RBRACE);
383 : DCHECK_EQ(source_pos() - 1, next().location.beg_pos);
384 79911 : return ScanTemplateSpan();
385 : }
386 :
387 : Handle<String> SourceUrl(Isolate* isolate) const;
388 : Handle<String> SourceMappingUrl(Isolate* isolate) const;
389 :
390 : bool FoundHtmlComment() const { return found_html_comment_; }
391 :
392 : bool allow_harmony_private_fields() const {
393 : return allow_harmony_private_fields_;
394 : }
395 : void set_allow_harmony_private_fields(bool allow) {
396 3351358 : allow_harmony_private_fields_ = allow;
397 : }
398 : bool allow_harmony_numeric_separator() const {
399 : return allow_harmony_numeric_separator_;
400 : }
401 : void set_allow_harmony_numeric_separator(bool allow) {
402 2950414 : allow_harmony_numeric_separator_ = allow;
403 : }
404 :
405 : const Utf16CharacterStream* stream() const { return source_; }
406 :
407 : // If the next characters in the stream are "#!", the line is skipped.
408 : void SkipHashBang();
409 :
410 : private:
411 : // Scoped helper for saving & restoring scanner error state.
412 : // This is used for tagged template literals, in which normally forbidden
413 : // escape sequences are allowed.
414 : class ErrorState;
415 :
416 : // LiteralBuffer - Collector of chars of literals.
417 : class LiteralBuffer {
418 : public:
419 23620661 : LiteralBuffer() : backing_store_(), position_(0), is_one_byte_(true) {}
420 :
421 : ~LiteralBuffer() { backing_store_.Dispose(); }
422 :
423 : V8_INLINE void AddChar(char code_unit) {
424 : DCHECK(IsValidAscii(code_unit));
425 767579252 : AddOneByteChar(static_cast<byte>(code_unit));
426 : }
427 :
428 249428616 : V8_INLINE void AddChar(uc32 code_unit) {
429 249428616 : if (is_one_byte()) {
430 248162059 : if (code_unit <= static_cast<uc32>(unibrow::Latin1::kMaxChar)) {
431 248083881 : AddOneByteChar(static_cast<byte>(code_unit));
432 : return;
433 : }
434 78178 : ConvertToTwoByte();
435 : }
436 1333872 : AddTwoByteChar(code_unit);
437 : }
438 :
439 249385055 : bool is_one_byte() const { return is_one_byte_; }
440 :
441 : bool Equals(Vector<const char> keyword) const {
442 : return is_one_byte() && keyword.length() == position_ &&
443 : (memcmp(keyword.start(), backing_store_.start(), position_) == 0);
444 : }
445 :
446 : Vector<const uint16_t> two_byte_literal() const {
447 : DCHECK(!is_one_byte());
448 : DCHECK_EQ(position_ & 0x1, 0);
449 : return Vector<const uint16_t>(
450 77462 : reinterpret_cast<const uint16_t*>(backing_store_.start()),
451 77462 : position_ >> 1);
452 : }
453 :
454 76429656 : Vector<const uint8_t> one_byte_literal() const {
455 : DCHECK(is_one_byte());
456 : return Vector<const uint8_t>(
457 209995245 : reinterpret_cast<const uint8_t*>(backing_store_.start()), position_);
458 : }
459 :
460 52443180 : int length() const { return is_one_byte() ? position_ : (position_ >> 1); }
461 :
462 129982253 : void Start() {
463 174094578 : position_ = 0;
464 174094578 : is_one_byte_ = true;
465 129982253 : }
466 :
467 : Handle<String> Internalize(Isolate* isolate) const;
468 :
469 : private:
470 : static const int kInitialCapacity = 16;
471 : static const int kGrowthFactor = 4;
472 : static const int kMaxGrowth = 1 * MB;
473 :
474 : inline bool IsValidAscii(char code_unit) {
475 : // Control characters and printable characters span the range of
476 : // valid ASCII characters (0-127). Chars are unsigned on some
477 : // platforms which causes compiler warnings if the validity check
478 : // tests the lower bound >= 0 as it's always true.
479 : return iscntrl(code_unit) || isprint(code_unit);
480 : }
481 :
482 : V8_INLINE void AddOneByteChar(byte one_byte_char) {
483 : DCHECK(is_one_byte());
484 1015681773 : if (position_ >= backing_store_.length()) ExpandBuffer();
485 1015655080 : backing_store_[position_] = one_byte_char;
486 1015513143 : position_ += kOneByteSize;
487 : }
488 :
489 : void AddTwoByteChar(uc32 code_unit);
490 : int NewCapacity(int min_capacity);
491 : void ExpandBuffer();
492 : void ConvertToTwoByte();
493 :
494 : Vector<byte> backing_store_;
495 : int position_;
496 :
497 : bool is_one_byte_;
498 :
499 : DISALLOW_COPY_AND_ASSIGN(LiteralBuffer);
500 : };
501 :
502 : // The current and look-ahead token.
503 35425277 : struct TokenDesc {
504 : Location location = {0, 0};
505 : LiteralBuffer literal_chars;
506 : LiteralBuffer raw_literal_chars;
507 : Token::Value token = Token::UNINITIALIZED;
508 : MessageTemplate invalid_template_escape_message = MessageTemplate::kNone;
509 : Location invalid_template_escape_location;
510 : uint32_t smi_value_ = 0;
511 : bool after_line_terminator = false;
512 :
513 : #ifdef DEBUG
514 : bool CanAccessLiteral() const {
515 : return token == Token::PRIVATE_NAME || token == Token::ILLEGAL ||
516 : token == Token::UNINITIALIZED || token == Token::REGEXP_LITERAL ||
517 : token == Token::ESCAPED_KEYWORD ||
518 : IsInRange(token, Token::NUMBER, Token::STRING) ||
519 : (Token::IsAnyIdentifier(token) && !Token::IsKeyword(token)) ||
520 : IsInRange(token, Token::TEMPLATE_SPAN, Token::TEMPLATE_TAIL);
521 : }
522 : bool CanAccessRawLiteral() const {
523 : return token == Token::ILLEGAL || token == Token::UNINITIALIZED ||
524 : IsInRange(token, Token::TEMPLATE_SPAN, Token::TEMPLATE_TAIL);
525 : }
526 : #endif // DEBUG
527 : };
528 :
529 : enum NumberKind {
530 : BINARY,
531 : OCTAL,
532 : IMPLICIT_OCTAL,
533 : HEX,
534 : DECIMAL,
535 : DECIMAL_WITH_LEADING_ZERO
536 : };
537 :
538 : static const int kCharacterLookaheadBufferSize = 1;
539 : static const int kMaxAscii = 127;
540 :
541 : // Scans octal escape sequence. Also accepts "\0" decimal escape sequence.
542 : template <bool capture_raw>
543 : uc32 ScanOctalEscape(uc32 c, int length);
544 :
545 : // Call this after setting source_ to the input.
546 2951947 : void Init() {
547 : // Set c0_ (one character ahead)
548 : STATIC_ASSERT(kCharacterLookaheadBufferSize == 1);
549 : Advance();
550 :
551 2952138 : current_ = &token_storage_[0];
552 2952138 : next_ = &token_storage_[1];
553 2952138 : next_next_ = &token_storage_[2];
554 :
555 2952138 : found_html_comment_ = false;
556 2952138 : scanner_error_ = MessageTemplate::kNone;
557 2952138 : }
558 :
559 4357 : void ReportScannerError(const Location& location, MessageTemplate error) {
560 4357 : if (has_error()) return;
561 10171 : scanner_error_ = error;
562 10171 : scanner_error_location_ = location;
563 : }
564 :
565 208805 : void ReportScannerError(int pos, MessageTemplate error) {
566 208805 : if (has_error()) return;
567 103779 : scanner_error_ = error;
568 103779 : scanner_error_location_ = Location(pos, pos + 1);
569 : }
570 :
571 : // Seek to the next_ token at the given position.
572 : void SeekNext(size_t position);
573 :
574 247367546 : V8_INLINE void AddLiteralChar(uc32 c) { next().literal_chars.AddChar(c); }
575 :
576 767631041 : V8_INLINE void AddLiteralChar(char c) { next().literal_chars.AddChar(c); }
577 :
578 : V8_INLINE void AddRawLiteralChar(uc32 c) {
579 1974831 : next().raw_literal_chars.AddChar(c);
580 : }
581 :
582 : V8_INLINE void AddLiteralCharAdvance() {
583 99392 : AddLiteralChar(c0_);
584 18159985 : Advance();
585 : }
586 :
587 : // Low-level scanning support.
588 : template <bool capture_raw = false>
589 427690742 : void Advance() {
590 : if (capture_raw) {
591 35813 : AddRawLiteralChar(c0_);
592 : }
593 1223406073 : c0_ = source_->Advance();
594 427697444 : }
595 :
596 : template <typename FunctionType>
597 : V8_INLINE void AdvanceUntil(FunctionType check) {
598 146132299 : c0_ = source_->AdvanceUntil(check);
599 : }
600 :
601 4743694 : bool CombineSurrogatePair() {
602 : DCHECK(!unibrow::Utf16::IsLeadSurrogate(kEndOfInput));
603 9487388 : if (unibrow::Utf16::IsLeadSurrogate(c0_)) {
604 174 : uc32 c1 = source_->Advance();
605 : DCHECK(!unibrow::Utf16::IsTrailSurrogate(kEndOfInput));
606 174 : if (unibrow::Utf16::IsTrailSurrogate(c1)) {
607 294 : c0_ = unibrow::Utf16::CombineSurrogatePair(c0_, c1);
608 : return true;
609 : }
610 27 : source_->Back();
611 : }
612 : return false;
613 : }
614 :
615 : void PushBack(uc32 ch) {
616 : DCHECK_LE(c0_, static_cast<uc32>(unibrow::Utf16::kMaxNonSurrogateCharCode));
617 34 : source_->Back();
618 34 : c0_ = ch;
619 : }
620 :
621 7309695 : uc32 Peek() const { return source_->Peek(); }
622 :
623 160176598 : inline Token::Value Select(Token::Value tok) {
624 : Advance();
625 160173549 : return tok;
626 : }
627 :
628 6124204 : inline Token::Value Select(uc32 next, Token::Value then, Token::Value else_) {
629 : Advance();
630 3653074 : if (c0_ == next) {
631 : Advance();
632 2471140 : return then;
633 : } else {
634 : return else_;
635 : }
636 : }
637 : // Returns the literal string, if any, for the current token (the
638 : // token last returned by Next()). The string is 0-terminated.
639 : // Literal strings are collected for identifiers, strings, numbers as well
640 : // as for template literals. For template literals we also collect the raw
641 : // form.
642 : // These functions only give the correct result if the literal was scanned
643 : // when a LiteralScope object is alive.
644 : //
645 : // Current usage of these functions is unfortunately a little undisciplined,
646 : // and is_literal_one_byte() + is_literal_one_byte_string() is also
647 : // requested for tokens that do not have a literal. Hence, we treat any
648 : // token as a one-byte literal. E.g. Token::FUNCTION pretends to have a
649 : // literal "function".
650 1327006 : Vector<const uint8_t> literal_one_byte_string() const {
651 : DCHECK(current().CanAccessLiteral() || Token::IsKeyword(current().token));
652 : return current().literal_chars.one_byte_literal();
653 : }
654 : Vector<const uint16_t> literal_two_byte_string() const {
655 : DCHECK(current().CanAccessLiteral() || Token::IsKeyword(current().token));
656 : return current().literal_chars.two_byte_literal();
657 : }
658 98518009 : bool is_literal_one_byte() const {
659 : DCHECK(current().CanAccessLiteral() || Token::IsKeyword(current().token));
660 98518009 : return current().literal_chars.is_one_byte();
661 : }
662 : // Returns the literal string for the next token (the token that
663 : // would be returned if Next() were called).
664 : Vector<const uint8_t> next_literal_one_byte_string() const {
665 : DCHECK(next().CanAccessLiteral());
666 : return next().literal_chars.one_byte_literal();
667 : }
668 : Vector<const uint16_t> next_literal_two_byte_string() const {
669 : DCHECK(next().CanAccessLiteral());
670 : return next().literal_chars.two_byte_literal();
671 : }
672 3063697 : bool is_next_literal_one_byte() const {
673 : DCHECK(next().CanAccessLiteral());
674 3063697 : return next().literal_chars.is_one_byte();
675 : }
676 : Vector<const uint8_t> raw_literal_one_byte_string() const {
677 : DCHECK(current().CanAccessRawLiteral());
678 : return current().raw_literal_chars.one_byte_literal();
679 : }
680 : Vector<const uint16_t> raw_literal_two_byte_string() const {
681 : DCHECK(current().CanAccessRawLiteral());
682 : return current().raw_literal_chars.two_byte_literal();
683 : }
684 83835 : bool is_raw_literal_one_byte() const {
685 : DCHECK(current().CanAccessRawLiteral());
686 83835 : return current().raw_literal_chars.is_one_byte();
687 : }
688 :
689 : template <bool capture_raw, bool unicode = false>
690 : uc32 ScanHexNumber(int expected_length);
691 : // Scan a number of any length but not bigger than max_value. For example, the
692 : // number can be 000000001, so it's very long in characters but its value is
693 : // small.
694 : template <bool capture_raw>
695 : uc32 ScanUnlimitedLengthHexNumber(int max_value, int beg_pos);
696 :
697 : // Scans a single JavaScript token.
698 : V8_INLINE Token::Value ScanSingleToken();
699 : V8_INLINE void Scan();
700 : // Performance hack: pass through a pre-calculated "next()" value to avoid
701 : // having to re-calculate it in Scan. You'd think the compiler would be able
702 : // to hoist the next() calculation out of the inlined Scan method, but seems
703 : // that pointer aliasing analysis fails show that this is safe.
704 : V8_INLINE void Scan(TokenDesc* next_desc);
705 :
706 : V8_INLINE Token::Value SkipWhiteSpace();
707 : Token::Value SkipSingleHTMLComment();
708 : Token::Value SkipSingleLineComment();
709 : Token::Value SkipSourceURLComment();
710 : void TryToParseSourceURLComment();
711 : Token::Value SkipMultiLineComment();
712 : // Scans a possible HTML comment -- begins with '<!'.
713 : Token::Value ScanHtmlComment();
714 :
715 : bool ScanDigitsWithNumericSeparators(bool (*predicate)(uc32 ch),
716 : bool is_check_first_digit);
717 : bool ScanDecimalDigits();
718 : // Optimized function to scan decimal number as Smi.
719 : bool ScanDecimalAsSmi(uint64_t* value);
720 : bool ScanDecimalAsSmiWithNumericSeparators(uint64_t* value);
721 : bool ScanHexDigits();
722 : bool ScanBinaryDigits();
723 : bool ScanSignedInteger();
724 : bool ScanOctalDigits();
725 : bool ScanImplicitOctalDigits(int start_pos, NumberKind* kind);
726 :
727 : Token::Value ScanNumber(bool seen_period);
728 : V8_INLINE Token::Value ScanIdentifierOrKeyword();
729 : V8_INLINE Token::Value ScanIdentifierOrKeywordInner();
730 : Token::Value ScanIdentifierOrKeywordInnerSlow(bool escaped,
731 : bool can_be_keyword);
732 :
733 : Token::Value ScanString();
734 : Token::Value ScanPrivateName();
735 :
736 : // Scans an escape-sequence which is part of a string and adds the
737 : // decoded character to the current literal. Returns true if a pattern
738 : // is scanned.
739 : template <bool capture_raw>
740 : bool ScanEscape();
741 :
742 : // Decodes a Unicode escape-sequence which is part of an identifier.
743 : // If the escape sequence cannot be decoded the result is kBadChar.
744 : uc32 ScanIdentifierUnicodeEscape();
745 : // Helper for the above functions.
746 : template <bool capture_raw>
747 73980 : uc32 ScanUnicodeEscape();
748 :
749 : Token::Value ScanTemplateSpan();
750 :
751 : // Return the current source position.
752 1284266337 : int source_pos() {
753 2631454366 : return static_cast<int>(source_->pos()) - kCharacterLookaheadBufferSize;
754 : }
755 :
756 : static bool LiteralContainsEscapes(const TokenDesc& token) {
757 49021777 : Location location = token.location;
758 49021777 : int source_length = (location.end_pos - location.beg_pos);
759 49021777 : if (token.token == Token::STRING) {
760 : // Subtract delimiters.
761 39110 : source_length -= 2;
762 : }
763 49021777 : return token.literal_chars.length() != source_length;
764 : }
765 :
766 : #ifdef DEBUG
767 : void SanityCheckTokenDesc(const TokenDesc&) const;
768 : #endif
769 :
770 2166101305 : TokenDesc& next() { return *next_; }
771 :
772 : const TokenDesc& current() const { return *current_; }
773 : const TokenDesc& next() const { return *next_; }
774 : const TokenDesc& next_next() const { return *next_next_; }
775 :
776 : TokenDesc* current_; // desc for current token (as returned by Next())
777 : TokenDesc* next_; // desc for next token (one token look-ahead)
778 : TokenDesc* next_next_; // desc for the token after next (after PeakAhead())
779 :
780 : // Input stream. Must be initialized to an Utf16CharacterStream.
781 : Utf16CharacterStream* const source_;
782 :
783 : // One Unicode character look-ahead; c0_ < 0 at the end of the input.
784 : uc32 c0_;
785 :
786 : TokenDesc token_storage_[3];
787 :
788 : // Whether this scanner encountered an HTML comment.
789 : bool found_html_comment_;
790 :
791 : // Harmony flags to allow ESNext features.
792 : bool allow_harmony_private_fields_;
793 : bool allow_harmony_numeric_separator_;
794 :
795 : const bool is_module_;
796 :
797 : // Values parsed from magic comments.
798 : LiteralBuffer source_url_;
799 : LiteralBuffer source_mapping_url_;
800 :
801 : // Last-seen positions of potentially problematic tokens.
802 : Location octal_pos_;
803 : MessageTemplate octal_message_;
804 :
805 : MessageTemplate scanner_error_;
806 : Location scanner_error_location_;
807 : };
808 :
809 : } // namespace internal
810 : } // namespace v8
811 :
812 : #endif // V8_PARSING_SCANNER_H_
|
