Line data    Source code

       1             : // Copyright 2016 the V8 project authors. All rights reserved.
       2             : // Use of this source code is governed by a BSD-style license that can be
       3             : // found in the LICENSE file.
       4             : 
       5             : // Tests v8::internal::Scanner. Note that presently most unit tests for the
       6             : // Scanner are in cctest/test-parsing.cc, rather than here.
       7             : 
       8             : #include "src/handles-inl.h"
       9             : #include "src/objects-inl.h"
      10             : #include "src/parsing/scanner-character-streams.h"
      11             : #include "src/parsing/scanner.h"
      12             : #include "test/cctest/cctest.h"
      13             : 
      14             : namespace v8 {
      15             : namespace internal {
      16             : 
      17             : namespace {
      18             : 
      19             : const char src_simple[] = "function foo() { var x = 2 * a() + b; }";
      20             : 
      21         230 : struct ScannerTestHelper {
      22             :   ScannerTestHelper() = default;
      23             :   ScannerTestHelper(ScannerTestHelper&& other) V8_NOEXCEPT
      24             :       : stream(std::move(other.stream)),
      25             :         scanner(std::move(other.scanner)) {}
      26             : 
      27             :   std::unique_ptr<Utf16CharacterStream> stream;
      28             :   std::unique_ptr<Scanner> scanner;
      29             : 
      30             :   Scanner* operator->() const { return scanner.get(); }
      31             :   Scanner* get() const { return scanner.get(); }
      32             : };
      33             : 
      34         115 : ScannerTestHelper make_scanner(const char* src) {
      35         115 :   ScannerTestHelper helper;
      36         230 :   helper.stream = ScannerStream::ForTesting(src);
      37         345 :   helper.scanner =
      38         115 :       std::unique_ptr<Scanner>(new Scanner(helper.stream.get(), false));
      39         115 :   helper.scanner->Initialize();
      40         115 :   return helper;
      41             : }
      42             : 
      43             : }  // anonymous namespace
      44             : 
      45             : // CHECK_TOK checks token equality, but by checking for equality of the token
      46             : // names. That should have the same result, but has much nicer error messaages.
      47             : #define CHECK_TOK(a, b) CHECK_EQ(Token::Name(a), Token::Name(b))
      48             : 
      49       26644 : TEST(Bookmarks) {
      50             :   // Scan through the given source and record the tokens for use as reference
      51             :   // below.
      52             :   std::vector<Token::Value> tokens;
      53             :   {
      54          10 :     auto scanner = make_scanner(src_simple);
      55          90 :     do {
      56         180 :       tokens.push_back(scanner->Next());
      57             :     } while (scanner->current_token() != Token::EOS);
      58             :   }
      59             : 
      60             :   // For each position:
      61             :   // - Scan through file,
      62             :   // - set a bookmark once the position is reached,
      63             :   // - scan a bit more,
      64             :   // - reset to the bookmark, and
      65             :   // - scan until the end.
      66             :   // At each step, compare to the reference token sequence generated above.
      67         185 :   for (size_t bookmark_pos = 0; bookmark_pos < tokens.size(); bookmark_pos++) {
      68         180 :     auto scanner = make_scanner(src_simple);
      69             :     Scanner::BookmarkScope bookmark(scanner.get());
      70             : 
      71        4500 :     for (size_t i = 0; i < std::min(bookmark_pos + 10, tokens.size()); i++) {
      72        1440 :       if (i == bookmark_pos) {
      73          90 :         bookmark.Set(scanner->peek_location().beg_pos);
      74             :       }
      75        4320 :       CHECK_TOK(tokens[i], scanner->Next());
      76             :     }
      77             : 
      78          90 :     bookmark.Apply();
      79        1800 :     for (size_t i = bookmark_pos; i < tokens.size(); i++) {
      80        2565 :       CHECK_TOK(tokens[i], scanner->Next());
      81             :     }
      82             :   }
      83           5 : }
      84             : 
      85       26644 : TEST(AllThePushbacks) {
      86             :   const struct {
      87             :     const char* src;
      88             :     const Token::Value tokens[5];  // Large enough for any of the test cases.
      89             :   } test_cases[] = {
      90             :       {"<-x", {Token::LT, Token::SUB, Token::IDENTIFIER, Token::EOS}},
      91             :       {"<!x", {Token::LT, Token::NOT, Token::IDENTIFIER, Token::EOS}},
      92             :       {"<!-x",
      93             :        {Token::LT, Token::NOT, Token::SUB, Token::IDENTIFIER, Token::EOS}},
      94             :       {"<!-- xx -->\nx", {Token::IDENTIFIER, Token::EOS}},
      95           5 :   };
      96             : 
      97          45 :   for (const auto& test_case : test_cases) {
      98          40 :     auto scanner = make_scanner(test_case.src);
      99         130 :     for (size_t i = 0; test_case.tokens[i] != Token::EOS; i++) {
     100         110 :       CHECK_TOK(test_case.tokens[i], scanner->Next());
     101             :     }
     102          40 :     CHECK_TOK(Token::EOS, scanner->Next());
     103             :   }
     104           5 : }
     105             : 
     106             : }  // namespace internal
     107       79917 : }  // namespace v8