Line data Source code
1 : // Copyright 2011 the V8 project authors. All rights reserved.
2 : // Use of this source code is governed by a BSD-style license that can be
3 : // found in the LICENSE file.
4 :
5 : // Features shared by parsing and pre-parsing scanners.
6 :
7 : #ifndef V8_PARSING_SCANNER_H_
8 : #define V8_PARSING_SCANNER_H_
9 :
10 : #include "src/allocation.h"
11 : #include "src/base/logging.h"
12 : #include "src/char-predicates.h"
13 : #include "src/globals.h"
14 : #include "src/messages.h"
15 : #include "src/parsing/token.h"
16 : #include "src/unicode-decoder.h"
17 : #include "src/unicode.h"
18 :
19 : namespace v8 {
20 : namespace internal {
21 :
22 :
23 : class AstRawString;
24 : class AstValueFactory;
25 : class DuplicateFinder;
26 : class ExternalOneByteString;
27 : class ExternalTwoByteString;
28 : class ParserRecorder;
29 : class UnicodeCache;
30 :
31 : // ---------------------------------------------------------------------
32 : // Buffered stream of UTF-16 code units, using an internal UTF-16 buffer.
33 : // A code unit is a 16 bit value representing either a 16 bit code point
34 : // or one part of a surrogate pair that make a single 21 bit code point.
35 : class Utf16CharacterStream {
36 : public:
37 : static const uc32 kEndOfInput = -1;
38 :
39 4402280 : virtual ~Utf16CharacterStream() { }
40 :
41 : // Returns and advances past the next UTF-16 code unit in the input
42 : // stream. If there are no more code units it returns kEndOfInput.
43 2712325074 : inline uc32 Advance() {
44 2712325074 : if (V8_LIKELY(buffer_cursor_ < buffer_end_)) {
45 2699733532 : return static_cast<uc32>(*(buffer_cursor_++));
46 12591542 : } else if (ReadBlock()) {
47 8988618 : return static_cast<uc32>(*(buffer_cursor_++));
48 : } else {
49 : // Note: currently the following increment is necessary to avoid a
50 : // parser problem! The scanner treats the final kEndOfInput as
51 : // a code unit with a position, and does math relative to that
52 : // position.
53 3602929 : buffer_cursor_++;
54 3602929 : return kEndOfInput;
55 : }
56 : }
57 :
58 : // Go back one by one character in the input stream.
59 : // This undoes the most recent Advance().
60 7683605 : inline void Back() {
61 : // The common case - if the previous character is within
62 : // buffer_start_ .. buffer_end_ will be handles locally.
63 : // Otherwise, a new block is requested.
64 7683605 : if (V8_LIKELY(buffer_cursor_ > buffer_start_)) {
65 7683600 : buffer_cursor_--;
66 : } else {
67 5 : ReadBlockAt(pos() - 1);
68 : }
69 7683605 : }
70 :
71 : // Go back one by two characters in the input stream. (This is the same as
72 : // calling Back() twice. But Back() may - in some instances - do substantial
73 : // work. Back2() guarantees this work will be done only once.)
74 96 : inline void Back2() {
75 96 : if (V8_LIKELY(buffer_cursor_ - 2 >= buffer_start_)) {
76 67 : buffer_cursor_ -= 2;
77 : } else {
78 29 : ReadBlockAt(pos() - 2);
79 : }
80 96 : }
81 :
82 : inline size_t pos() const {
83 1412474358 : return buffer_pos_ + (buffer_cursor_ - buffer_start_);
84 : }
85 :
86 473 : inline void Seek(size_t pos) {
87 473 : if (V8_LIKELY(pos >= buffer_pos_ &&
88 : pos < (buffer_pos_ + (buffer_end_ - buffer_start_)))) {
89 276 : buffer_cursor_ = buffer_start_ + (pos - buffer_pos_);
90 : } else {
91 : ReadBlockAt(pos);
92 : }
93 473 : }
94 :
95 : protected:
96 : Utf16CharacterStream(const uint16_t* buffer_start,
97 : const uint16_t* buffer_cursor,
98 : const uint16_t* buffer_end, size_t buffer_pos)
99 : : buffer_start_(buffer_start),
100 : buffer_cursor_(buffer_cursor),
101 : buffer_end_(buffer_end),
102 4402289 : buffer_pos_(buffer_pos) {}
103 : Utf16CharacterStream() : Utf16CharacterStream(nullptr, nullptr, nullptr, 0) {}
104 :
105 : void ReadBlockAt(size_t new_pos) {
106 : // The callers of this method (Back/Back2/Seek) should handle the easy
107 : // case (seeking within the current buffer), and we should only get here
108 : // if we actually require new data.
109 : // (This is really an efficiency check, not a correctness invariant.)
110 : DCHECK(new_pos < buffer_pos_ ||
111 : new_pos >= buffer_pos_ + (buffer_end_ - buffer_start_));
112 :
113 : // Change pos() to point to new_pos.
114 231 : buffer_pos_ = new_pos;
115 231 : buffer_cursor_ = buffer_start_;
116 231 : bool success = ReadBlock();
117 : USE(success);
118 :
119 : // Post-conditions: 1, on success, we should be at the right position.
120 : // 2, success == we should have more characters available.
121 : DCHECK_IMPLIES(success, pos() == new_pos);
122 : DCHECK_EQ(success, buffer_cursor_ < buffer_end_);
123 : DCHECK_EQ(success, buffer_start_ < buffer_end_);
124 : }
125 :
126 : // Read more data, and update buffer_*_ to point to it.
127 : // Returns true if more data was available.
128 : //
129 : // ReadBlock() may modify any of the buffer_*_ members, but must sure that
130 : // the result of pos() remains unaffected.
131 : //
132 : // Examples:
133 : // - a stream could either fill a separate buffer. Then buffer_start_ and
134 : // buffer_cursor_ would point to the beginning of the buffer, and
135 : // buffer_pos would be the old pos().
136 : // - a stream with existing buffer chunks would set buffer_start_ and
137 : // buffer_end_ to cover the full chunk, and then buffer_cursor_ would
138 : // point into the middle of the buffer, while buffer_pos_ would describe
139 : // the start of the buffer.
140 : virtual bool ReadBlock() = 0;
141 :
142 : const uint16_t* buffer_start_;
143 : const uint16_t* buffer_cursor_;
144 : const uint16_t* buffer_end_;
145 : size_t buffer_pos_;
146 : };
147 :
148 :
149 : // ----------------------------------------------------------------------------
150 : // JavaScript Scanner.
151 :
152 7135462 : class Scanner {
153 : public:
154 : // Scoped helper for a re-settable bookmark.
155 : class BookmarkScope {
156 : public:
157 : explicit BookmarkScope(Scanner* scanner)
158 1434202 : : scanner_(scanner), bookmark_(kNoBookmark) {
159 : DCHECK_NOT_NULL(scanner_);
160 : }
161 : ~BookmarkScope() {}
162 :
163 : void Set();
164 : void Apply();
165 : bool HasBeenSet();
166 : bool HasBeenApplied();
167 :
168 : private:
169 : static const size_t kNoBookmark;
170 : static const size_t kBookmarkWasApplied;
171 : static const size_t kBookmarkAtFirstPos;
172 :
173 : Scanner* scanner_;
174 : size_t bookmark_;
175 :
176 : DISALLOW_COPY_AND_ASSIGN(BookmarkScope);
177 : };
178 :
179 : // Representation of an interval of source positions.
180 : struct Location {
181 15815111 : Location(int b, int e) : beg_pos(b), end_pos(e) { }
182 39654997 : Location() : beg_pos(0), end_pos(0) { }
183 :
184 : bool IsValid() const {
185 37686283 : return beg_pos >= 0 && end_pos >= beg_pos;
186 : }
187 :
188 : static Location invalid() { return Location(-1, -1); }
189 :
190 : int beg_pos;
191 : int end_pos;
192 : };
193 :
194 : // -1 is outside of the range of any real source code.
195 : static const int kNoOctalLocation = -1;
196 : static const uc32 kEndOfInput = Utf16CharacterStream::kEndOfInput;
197 :
198 : explicit Scanner(UnicodeCache* scanner_contants);
199 :
200 : void Initialize(Utf16CharacterStream* source);
201 :
202 : // Returns the next token and advances input.
203 : Token::Value Next();
204 : // Returns the token following peek()
205 : Token::Value PeekAhead();
206 : // Returns the current token again.
207 : Token::Value current_token() { return current_.token; }
208 :
209 : Token::Value current_contextual_token() { return current_.contextual_token; }
210 : Token::Value next_contextual_token() { return next_.contextual_token; }
211 :
212 : // Returns the location information for the current token
213 : // (the token last returned by Next()).
214 10058037 : Location location() const { return current_.location; }
215 :
216 : // This error is specifically an invalid hex or unicode escape sequence.
217 : bool has_error() const { return scanner_error_ != MessageTemplate::kNone; }
218 : MessageTemplate::Template error() const { return scanner_error_; }
219 : Location error_location() const { return scanner_error_location_; }
220 :
221 : bool has_invalid_template_escape() const {
222 : return current_.invalid_template_escape_message != MessageTemplate::kNone;
223 : }
224 : MessageTemplate::Template invalid_template_escape_message() const {
225 : DCHECK(has_invalid_template_escape());
226 : return current_.invalid_template_escape_message;
227 : }
228 : Location invalid_template_escape_location() const {
229 : DCHECK(has_invalid_template_escape());
230 : return current_.invalid_template_escape_location;
231 : }
232 :
233 : // Similar functions for the upcoming token.
234 :
235 : // One token look-ahead (past the token returned by Next()).
236 3140315130 : Token::Value peek() const { return next_.token; }
237 :
238 : Location peek_location() const { return next_.location; }
239 :
240 : bool literal_contains_escapes() const {
241 32765409 : return LiteralContainsEscapes(current_);
242 : }
243 :
244 : const AstRawString* CurrentSymbol(AstValueFactory* ast_value_factory) const;
245 : const AstRawString* NextSymbol(AstValueFactory* ast_value_factory) const;
246 : const AstRawString* CurrentRawSymbol(
247 : AstValueFactory* ast_value_factory) const;
248 :
249 : double DoubleValue();
250 : bool ContainsDot();
251 :
252 : inline bool CurrentMatches(Token::Value token) const {
253 : DCHECK(Token::IsKeyword(token));
254 : return current_.token == token;
255 : }
256 :
257 : inline bool CurrentMatchesContextual(Token::Value token) const {
258 : DCHECK(Token::IsContextualKeyword(token));
259 124140 : return current_.contextual_token == token;
260 : }
261 :
262 : // Match the token against the contextual keyword or literal buffer.
263 4961349 : inline bool CurrentMatchesContextualEscaped(Token::Value token) const {
264 : DCHECK(Token::IsContextualKeyword(token) || token == Token::LET);
265 : // Escaped keywords are not matched as tokens. So if we require escape
266 : // and/or string processing we need to look at the literal content
267 : // (which was escape-processed already).
268 : // Conveniently, current_.literal_chars == nullptr for all proper keywords,
269 : // so this second condition should exit early in common cases.
270 4961349 : return (current_.contextual_token == token) ||
271 4790049 : (current_.literal_chars &&
272 : current_.literal_chars->Equals(Vector<const char>(
273 14541447 : Token::String(token), Token::StringLength(token))));
274 : }
275 :
276 3323053 : bool IsUseStrict() const {
277 6646106 : return current_.token == Token::STRING &&
278 : current_.literal_chars->Equals(
279 6646106 : Vector<const char>("use strict", strlen("use strict")));
280 : }
281 124140 : bool IsGetOrSet(bool* is_get, bool* is_set) const {
282 62070 : *is_get = CurrentMatchesContextual(Token::GET);
283 62070 : *is_set = CurrentMatchesContextual(Token::SET);
284 62070 : return *is_get || *is_set;
285 : }
286 94373 : bool IsLet() const {
287 94373 : return CurrentMatches(Token::LET) ||
288 94373 : CurrentMatchesContextualEscaped(Token::LET);
289 : }
290 :
291 : // Check whether the CurrentSymbol() has already been seen.
292 : // The DuplicateFinder holds the data, so different instances can be used
293 : // for different sets of duplicates to check for.
294 : bool IsDuplicateSymbol(DuplicateFinder* duplicate_finder,
295 : AstValueFactory* ast_value_factory) const;
296 :
297 : UnicodeCache* unicode_cache() { return unicode_cache_; }
298 :
299 : // Returns the location of the last seen octal literal.
300 : Location octal_position() const { return octal_pos_; }
301 : void clear_octal_position() {
302 555 : octal_pos_ = Location::invalid();
303 555 : octal_message_ = MessageTemplate::kNone;
304 : }
305 : MessageTemplate::Template octal_message() const { return octal_message_; }
306 :
307 : // Returns the value of the last smi that was scanned.
308 : uint32_t smi_value() const { return current_.smi_value_; }
309 :
310 : // Seek forward to the given position. This operation does not
311 : // work in general, for instance when there are pushed back
312 : // characters, but works for seeking forward until simple delimiter
313 : // tokens, which is what it is used for.
314 : void SeekForward(int pos);
315 :
316 : // Returns true if there was a line terminator before the peek'ed token,
317 : // possibly inside a multi-line comment.
318 : bool HasAnyLineTerminatorBeforeNext() const {
319 96898602 : return has_line_terminator_before_next_ ||
320 : has_multiline_comment_before_next_;
321 : }
322 :
323 : bool HasAnyLineTerminatorAfterNext() {
324 272895 : Token::Value ensure_next_next = PeekAhead();
325 : USE(ensure_next_next);
326 272895 : return has_line_terminator_after_next_;
327 : }
328 :
329 : // Scans the input as a regular expression pattern, next token must be /(=).
330 : // Returns true if a pattern is scanned.
331 : bool ScanRegExpPattern();
332 : // Scans the input as regular expression flags. Returns the flags on success.
333 : Maybe<RegExp::Flags> ScanRegExpFlags();
334 :
335 : // Scans the input as a template literal
336 : Token::Value ScanTemplateStart();
337 : Token::Value ScanTemplateContinuation();
338 :
339 : Handle<String> SourceUrl(Isolate* isolate) const;
340 : Handle<String> SourceMappingUrl(Isolate* isolate) const;
341 :
342 : bool FoundHtmlComment() const { return found_html_comment_; }
343 :
344 : private:
345 : // Scoped helper for saving & restoring scanner error state.
346 : // This is used for tagged template literals, in which normally forbidden
347 : // escape sequences are allowed.
348 : class ErrorState;
349 :
350 : // Scoped helper for literal recording. Automatically drops the literal
351 : // if aborting the scanning before it's complete.
352 : class LiteralScope {
353 : public:
354 200166533 : explicit LiteralScope(Scanner* self) : scanner_(self), complete_(false) {
355 : scanner_->StartLiteral();
356 : }
357 : ~LiteralScope() {
358 254814312 : if (!complete_) scanner_->DropLiteral();
359 : }
360 142974246 : void Complete() { complete_ = true; }
361 :
362 : private:
363 : Scanner* scanner_;
364 : bool complete_;
365 : };
366 :
367 : // LiteralBuffer - Collector of chars of literals.
368 : class LiteralBuffer {
369 : public:
370 32439974 : LiteralBuffer() : is_one_byte_(true), position_(0), backing_store_() {}
371 :
372 : ~LiteralBuffer() { backing_store_.Dispose(); }
373 :
374 : INLINE(void AddChar(char code_unit)) {
375 : DCHECK(IsValidAscii(code_unit));
376 1453197255 : AddOneByteChar(static_cast<byte>(code_unit));
377 : }
378 :
379 : INLINE(void AddChar(uc32 code_unit)) {
380 110859475 : if (is_one_byte_ &&
381 : code_unit <= static_cast<uc32>(unibrow::Latin1::kMaxChar)) {
382 109402304 : AddOneByteChar(static_cast<byte>(code_unit));
383 : } else {
384 1457069 : AddCharSlow(code_unit);
385 : }
386 : }
387 :
388 : bool is_one_byte() const { return is_one_byte_; }
389 :
390 8113102 : bool Equals(Vector<const char> keyword) const {
391 8720259 : return is_one_byte() && keyword.length() == position_ &&
392 8720259 : (memcmp(keyword.start(), backing_store_.start(), position_) == 0);
393 : }
394 :
395 : Vector<const uint16_t> two_byte_literal() const {
396 : DCHECK(!is_one_byte_);
397 : DCHECK((position_ & 0x1) == 0);
398 : return Vector<const uint16_t>(
399 66191 : reinterpret_cast<const uint16_t*>(backing_store_.start()),
400 66191 : position_ >> 1);
401 : }
402 :
403 : Vector<const uint8_t> one_byte_literal() const {
404 : DCHECK(is_one_byte_);
405 : return Vector<const uint8_t>(
406 332498551 : reinterpret_cast<const uint8_t*>(backing_store_.start()), position_);
407 : }
408 :
409 37273867 : int length() const { return is_one_byte_ ? position_ : (position_ >> 1); }
410 :
411 167621 : void ReduceLength(int delta) {
412 167621 : position_ -= delta * (is_one_byte_ ? kOneByteSize : kUC16Size);
413 167621 : }
414 :
415 : void Reset() {
416 255153721 : position_ = 0;
417 255153721 : is_one_byte_ = true;
418 : }
419 :
420 : Handle<String> Internalize(Isolate* isolate) const;
421 :
422 : private:
423 : static const int kInitialCapacity = 16;
424 : static const int kGrowthFactory = 4;
425 : static const int kMinConversionSlack = 256;
426 : static const int kMaxGrowth = 1 * MB;
427 :
428 : inline bool IsValidAscii(char code_unit) {
429 : // Control characters and printable characters span the range of
430 : // valid ASCII characters (0-127). Chars are unsigned on some
431 : // platforms which causes compiler warnings if the validity check
432 : // tests the lower bound >= 0 as it's always true.
433 : return iscntrl(code_unit) || isprint(code_unit);
434 : }
435 :
436 : INLINE(void AddOneByteChar(byte one_byte_char)) {
437 : DCHECK(is_one_byte_);
438 1562604825 : if (position_ >= backing_store_.length()) ExpandBuffer();
439 1562604876 : backing_store_[position_] = one_byte_char;
440 1562604900 : position_ += kOneByteSize;
441 : }
442 :
443 : void AddCharSlow(uc32 code_unit);
444 : int NewCapacity(int min_capacity);
445 : void ExpandBuffer();
446 : void ConvertToTwoByte();
447 :
448 : bool is_one_byte_;
449 : int position_;
450 : Vector<byte> backing_store_;
451 :
452 : DISALLOW_COPY_AND_ASSIGN(LiteralBuffer);
453 : };
454 :
455 : // The current and look-ahead token.
456 : struct TokenDesc {
457 : Location location;
458 : LiteralBuffer* literal_chars;
459 : LiteralBuffer* raw_literal_chars;
460 : uint32_t smi_value_;
461 : Token::Value token;
462 : MessageTemplate::Template invalid_template_escape_message;
463 : Location invalid_template_escape_location;
464 : Token::Value contextual_token;
465 : };
466 :
467 : static const int kCharacterLookaheadBufferSize = 1;
468 : const int kMaxAscii = 127;
469 :
470 : // Scans octal escape sequence. Also accepts "\0" decimal escape sequence.
471 : template <bool capture_raw>
472 : uc32 ScanOctalEscape(uc32 c, int length);
473 :
474 : // Call this after setting source_ to the input.
475 4050465 : void Init() {
476 : // Set c0_ (one character ahead)
477 : STATIC_ASSERT(kCharacterLookaheadBufferSize == 1);
478 4050465 : Advance();
479 : // Initialize current_ to not refer to a literal.
480 4050467 : current_.token = Token::UNINITIALIZED;
481 4050467 : current_.contextual_token = Token::UNINITIALIZED;
482 4050467 : current_.literal_chars = NULL;
483 4050467 : current_.raw_literal_chars = NULL;
484 4050467 : current_.invalid_template_escape_message = MessageTemplate::kNone;
485 4050467 : next_.token = Token::UNINITIALIZED;
486 4050467 : next_.contextual_token = Token::UNINITIALIZED;
487 4050467 : next_.literal_chars = NULL;
488 4050467 : next_.raw_literal_chars = NULL;
489 4050467 : next_.invalid_template_escape_message = MessageTemplate::kNone;
490 4050467 : next_next_.token = Token::UNINITIALIZED;
491 4050467 : next_next_.contextual_token = Token::UNINITIALIZED;
492 4050467 : next_next_.literal_chars = NULL;
493 4050467 : next_next_.raw_literal_chars = NULL;
494 4050467 : next_next_.invalid_template_escape_message = MessageTemplate::kNone;
495 4050467 : found_html_comment_ = false;
496 4050467 : scanner_error_ = MessageTemplate::kNone;
497 4050467 : }
498 :
499 : void ReportScannerError(const Location& location,
500 16665 : MessageTemplate::Template error) {
501 16665 : if (has_error()) return;
502 16635 : scanner_error_ = error;
503 16635 : scanner_error_location_ = location;
504 : }
505 :
506 12384 : void ReportScannerError(int pos, MessageTemplate::Template error) {
507 12384 : if (has_error()) return;
508 9632 : scanner_error_ = error;
509 9632 : scanner_error_location_ = Location(pos, pos + 1);
510 : }
511 :
512 : // Seek to the next_ token at the given position.
513 : void SeekNext(size_t position);
514 :
515 : // Literal buffer support
516 : inline void StartLiteral() {
517 : LiteralBuffer* free_buffer =
518 254982523 : (current_.literal_chars == &literal_buffer0_)
519 : ? &literal_buffer1_
520 254212821 : : (current_.literal_chars == &literal_buffer1_) ? &literal_buffer2_
521 509195344 : : &literal_buffer0_;
522 : free_buffer->Reset();
523 254982523 : next_.literal_chars = free_buffer;
524 : }
525 :
526 : inline void StartRawLiteral() {
527 : LiteralBuffer* free_buffer =
528 168651 : (current_.raw_literal_chars == &raw_literal_buffer0_)
529 : ? &raw_literal_buffer1_
530 166559 : : (current_.raw_literal_chars == &raw_literal_buffer1_)
531 : ? &raw_literal_buffer2_
532 335210 : : &raw_literal_buffer0_;
533 : free_buffer->Reset();
534 168651 : next_.raw_literal_chars = free_buffer;
535 : }
536 :
537 : INLINE(void AddLiteralChar(uc32 c)) {
538 : DCHECK_NOT_NULL(next_.literal_chars);
539 : next_.literal_chars->AddChar(c);
540 : }
541 :
542 : INLINE(void AddLiteralChar(char c)) {
543 : DCHECK_NOT_NULL(next_.literal_chars);
544 : next_.literal_chars->AddChar(c);
545 : }
546 :
547 : INLINE(void AddRawLiteralChar(uc32 c)) {
548 : DCHECK_NOT_NULL(next_.raw_literal_chars);
549 : next_.raw_literal_chars->AddChar(c);
550 : }
551 :
552 : INLINE(void ReduceRawLiteralLength(int delta)) {
553 : DCHECK_NOT_NULL(next_.raw_literal_chars);
554 167621 : next_.raw_literal_chars->ReduceLength(delta);
555 : }
556 :
557 : // Stops scanning of a literal and drop the collected characters,
558 : // e.g., due to an encountered error.
559 : inline void DropLiteral() {
560 57195633 : next_.literal_chars = NULL;
561 57195633 : next_.raw_literal_chars = NULL;
562 : }
563 :
564 46576926 : inline void AddLiteralCharAdvance() {
565 23288463 : AddLiteralChar(c0_);
566 23288465 : Advance();
567 23288465 : }
568 :
569 : // Low-level scanning support.
570 : template <bool capture_raw = false, bool check_surrogate = true>
571 1146077054 : void Advance() {
572 : if (capture_raw) {
573 1585275 : AddRawLiteralChar(c0_);
574 : }
575 2693402868 : c0_ = source_->Advance();
576 1144491620 : if (check_surrogate) HandleLeadSurrogate();
577 1144491689 : }
578 :
579 1184331696 : void HandleLeadSurrogate() {
580 2368663392 : if (unibrow::Utf16::IsLeadSurrogate(c0_)) {
581 1059 : uc32 c1 = source_->Advance();
582 1059 : if (!unibrow::Utf16::IsTrailSurrogate(c1)) {
583 48 : source_->Back();
584 : } else {
585 2022 : c0_ = unibrow::Utf16::CombineSurrogatePair(c0_, c1);
586 : }
587 : }
588 1184331696 : }
589 :
590 : void PushBack(uc32 ch) {
591 6787039 : if (c0_ > static_cast<uc32>(unibrow::Utf16::kMaxNonSurrogateCharCode)) {
592 0 : source_->Back2();
593 : } else {
594 6787039 : source_->Back();
595 : }
596 6787039 : c0_ = ch;
597 : }
598 :
599 : // Same as PushBack(ch1); PushBack(ch2).
600 : // - Potentially more efficient as it uses Back2() on the stream.
601 : // - Uses char as parameters, since we're only calling it with ASCII chars in
602 : // practice. This way, we can avoid a few edge cases.
603 : void PushBack2(char ch1, char ch2) {
604 96 : source_->Back2();
605 96 : c0_ = ch2;
606 : }
607 :
608 : inline Token::Value Select(Token::Value tok) {
609 42817716 : Advance();
610 : return tok;
611 : }
612 :
613 : inline Token::Value Select(uc32 next, Token::Value then, Token::Value else_) {
614 7832551 : Advance();
615 7832550 : if (c0_ == next) {
616 3953602 : Advance();
617 : return then;
618 : } else {
619 : return else_;
620 : }
621 : }
622 : // Returns the literal string, if any, for the current token (the
623 : // token last returned by Next()). The string is 0-terminated.
624 : // Literal strings are collected for identifiers, strings, numbers as well
625 : // as for template literals. For template literals we also collect the raw
626 : // form.
627 : // These functions only give the correct result if the literal was scanned
628 : // when a LiteralScope object is alive.
629 : //
630 : // Current usage of these functions is unfortunately a little undisciplined,
631 : // and is_literal_one_byte() + is_literal_one_byte_string() is also
632 : // requested for tokens that do not have a literal. Hence, we treat any
633 : // token as a one-byte literal. E.g. Token::FUNCTION pretends to have a
634 : // literal "function".
635 : Vector<const uint8_t> literal_one_byte_string() const {
636 146842781 : if (current_.literal_chars)
637 : return current_.literal_chars->one_byte_literal();
638 88783 : const char* str = Token::String(current_.token);
639 : const uint8_t* str_as_uint8 = reinterpret_cast<const uint8_t*>(str);
640 : return Vector<const uint8_t>(str_as_uint8,
641 88783 : Token::StringLength(current_.token));
642 : }
643 : Vector<const uint16_t> literal_two_byte_string() const {
644 : DCHECK_NOT_NULL(current_.literal_chars);
645 : return current_.literal_chars->two_byte_literal();
646 : }
647 : bool is_literal_one_byte() const {
648 142597687 : return !current_.literal_chars || current_.literal_chars->is_one_byte();
649 : }
650 : int literal_length() const {
651 : if (current_.literal_chars) return current_.literal_chars->length();
652 : return Token::StringLength(current_.token);
653 : }
654 : // Returns the literal string for the next token (the token that
655 : // would be returned if Next() were called).
656 : Vector<const uint8_t> next_literal_one_byte_string() const {
657 : DCHECK_NOT_NULL(next_.literal_chars);
658 : return next_.literal_chars->one_byte_literal();
659 : }
660 : Vector<const uint16_t> next_literal_two_byte_string() const {
661 : DCHECK_NOT_NULL(next_.literal_chars);
662 : return next_.literal_chars->two_byte_literal();
663 : }
664 : bool is_next_literal_one_byte() const {
665 : DCHECK_NOT_NULL(next_.literal_chars);
666 98348 : return next_.literal_chars->is_one_byte();
667 : }
668 : Vector<const uint8_t> raw_literal_one_byte_string() const {
669 : DCHECK_NOT_NULL(current_.raw_literal_chars);
670 : return current_.raw_literal_chars->one_byte_literal();
671 : }
672 : Vector<const uint16_t> raw_literal_two_byte_string() const {
673 : DCHECK_NOT_NULL(current_.raw_literal_chars);
674 : return current_.raw_literal_chars->two_byte_literal();
675 : }
676 : bool is_raw_literal_one_byte() const {
677 : DCHECK_NOT_NULL(current_.raw_literal_chars);
678 70792 : return current_.raw_literal_chars->is_one_byte();
679 : }
680 :
681 : template <bool capture_raw, bool unicode = false>
682 : uc32 ScanHexNumber(int expected_length);
683 : // Scan a number of any length but not bigger than max_value. For example, the
684 : // number can be 000000001, so it's very long in characters but its value is
685 : // small.
686 : template <bool capture_raw>
687 : uc32 ScanUnlimitedLengthHexNumber(int max_value, int beg_pos);
688 :
689 : // Scans a single JavaScript token.
690 : void Scan();
691 :
692 : bool SkipWhiteSpace();
693 : Token::Value SkipSingleLineComment();
694 : Token::Value SkipSourceURLComment();
695 : void TryToParseSourceURLComment();
696 : Token::Value SkipMultiLineComment();
697 : // Scans a possible HTML comment -- begins with '<!'.
698 : Token::Value ScanHtmlComment();
699 :
700 : void ScanDecimalDigits();
701 : Token::Value ScanNumber(bool seen_period);
702 : Token::Value ScanIdentifierOrKeyword();
703 : Token::Value ScanIdentifierSuffix(LiteralScope* literal, bool escaped);
704 :
705 : Token::Value ScanString();
706 :
707 : // Scans an escape-sequence which is part of a string and adds the
708 : // decoded character to the current literal. Returns true if a pattern
709 : // is scanned.
710 : template <bool capture_raw, bool in_template_literal>
711 : bool ScanEscape();
712 :
713 : // Decodes a Unicode escape-sequence which is part of an identifier.
714 : // If the escape sequence cannot be decoded the result is kBadChar.
715 : uc32 ScanIdentifierUnicodeEscape();
716 : // Helper for the above functions.
717 : template <bool capture_raw>
718 24981 : uc32 ScanUnicodeEscape();
719 :
720 : Token::Value ScanTemplateSpan();
721 :
722 : // Return the current source position.
723 : int source_pos() {
724 1395685892 : return static_cast<int>(source_->pos()) - kCharacterLookaheadBufferSize;
725 : }
726 :
727 32765409 : static bool LiteralContainsEscapes(const TokenDesc& token) {
728 32765409 : Location location = token.location;
729 32765409 : int source_length = (location.end_pos - location.beg_pos);
730 32765409 : if (token.token == Token::STRING) {
731 : // Subtract delimiters.
732 172688 : source_length -= 2;
733 : }
734 65512281 : return token.literal_chars &&
735 32765409 : (token.literal_chars->length() != source_length);
736 : }
737 :
738 : #ifdef DEBUG
739 : void SanityCheckTokenDesc(const TokenDesc&) const;
740 : #endif
741 :
742 : UnicodeCache* unicode_cache_;
743 :
744 : // Buffers collecting literal strings, numbers, etc.
745 : LiteralBuffer literal_buffer0_;
746 : LiteralBuffer literal_buffer1_;
747 : LiteralBuffer literal_buffer2_;
748 :
749 : // Values parsed from magic comments.
750 : LiteralBuffer source_url_;
751 : LiteralBuffer source_mapping_url_;
752 :
753 : // Buffer to store raw string values
754 : LiteralBuffer raw_literal_buffer0_;
755 : LiteralBuffer raw_literal_buffer1_;
756 : LiteralBuffer raw_literal_buffer2_;
757 :
758 : TokenDesc current_; // desc for current token (as returned by Next())
759 : TokenDesc next_; // desc for next token (one token look-ahead)
760 : TokenDesc next_next_; // desc for the token after next (after PeakAhead())
761 :
762 : // Input stream. Must be initialized to an Utf16CharacterStream.
763 : Utf16CharacterStream* source_;
764 :
765 : // Last-seen positions of potentially problematic tokens.
766 : Location octal_pos_;
767 : MessageTemplate::Template octal_message_;
768 :
769 : // One Unicode character look-ahead; c0_ < 0 at the end of the input.
770 : uc32 c0_;
771 :
772 : // Whether there is a line terminator whitespace character after
773 : // the current token, and before the next. Does not count newlines
774 : // inside multiline comments.
775 : bool has_line_terminator_before_next_;
776 : // Whether there is a multi-line comment that contains a
777 : // line-terminator after the current token, and before the next.
778 : bool has_multiline_comment_before_next_;
779 : bool has_line_terminator_after_next_;
780 :
781 : // Whether this scanner encountered an HTML comment.
782 : bool found_html_comment_;
783 :
784 : MessageTemplate::Template scanner_error_;
785 : Location scanner_error_location_;
786 : };
787 :
788 : } // namespace internal
789 : } // namespace v8
790 :
791 : #endif // V8_PARSING_SCANNER_H_
|
