Line data    Source code

       1             : // Copyright 2016 the V8 project authors. All rights reserved.
       2             : // Use of this source code is governed by a BSD-style license that can be
       3             : // found in the LICENSE file.
       4             : 
       5             : // Tests v8::internal::Scanner. Note that presently most unit tests for the
       6             : // Scanner are in cctest/test-parsing.cc, rather than here.
       7             : 
       8             : #include "src/handles-inl.h"
       9             : #include "src/objects-inl.h"
      10             : #include "src/parsing/scanner-character-streams.h"
      11             : #include "src/parsing/scanner.h"
      12             : #include "src/unicode-cache.h"
      13             : #include "test/cctest/cctest.h"
      14             : 
      15             : namespace v8 {
      16             : namespace internal {
      17             : 
      18             : namespace {
      19             : 
      20             : const char src_simple[] = "function foo() { var x = 2 * a() + b; }";
      21             : 
      22         288 : struct ScannerTestHelper {
      23         144 :   ScannerTestHelper() = default;
      24             :   ScannerTestHelper(ScannerTestHelper&& other)
      25             :       : unicode_cache(std::move(other.unicode_cache)),
      26             :         stream(std::move(other.stream)),
      27             :         scanner(std::move(other.scanner)) {}
      28             : 
      29             :   std::unique_ptr<UnicodeCache> unicode_cache;
      30             :   std::unique_ptr<Utf16CharacterStream> stream;
      31             :   std::unique_ptr<Scanner> scanner;
      32             :   int use_counts[v8::Isolate::kUseCounterFeatureCount];
      33             : 
      34             :   Scanner* operator->() const { return scanner.get(); }
      35             :   Scanner* get() const { return scanner.get(); }
      36             : };
      37             : 
      38         144 : ScannerTestHelper make_scanner(const char* src) {
      39             :   ScannerTestHelper helper;
      40         144 :   helper.unicode_cache = std::unique_ptr<UnicodeCache>(new UnicodeCache);
      41         288 :   helper.stream = ScannerStream::ForTesting(src);
      42        6192 :   for (int i = 0; i < v8::Isolate::kUseCounterFeatureCount; i++) {
      43        6048 :     helper.use_counts[i] = 0;
      44             :   }
      45         144 :   helper.scanner = std::unique_ptr<Scanner>(
      46         144 :       new Scanner(helper.unicode_cache.get(), helper.use_counts));
      47         144 :   helper.scanner->Initialize(helper.stream.get(), false);
      48         144 :   return helper;
      49             : }
      50             : 
      51             : }  // anonymous namespace
      52             : 
      53             : // CHECK_TOK checks token equality, but by checking for equality of the token
      54             : // names. That should have the same result, but has much nicer error messaages.
      55             : #define CHECK_TOK(a, b) CHECK_EQ(Token::Name(a), Token::Name(b))
      56             : 
      57       23724 : TEST(Bookmarks) {
      58             :   // Scan through the given source and record the tokens for use as reference
      59             :   // below.
      60             :   std::vector<Token::Value> tokens;
      61             :   {
      62           6 :     auto scanner = make_scanner(src_simple);
      63         108 :     do {
      64         216 :       tokens.push_back(scanner->Next());
      65         114 :     } while (scanner->current_token() != Token::EOS);
      66             :   }
      67             : 
      68             :   // For each position:
      69             :   // - Scan through file,
      70             :   // - set a bookmark once the position is reached,
      71             :   // - scan a bit more,
      72             :   // - reset to the bookmark, and
      73             :   // - scan until the end.
      74             :   // At each step, compare to the reference token sequence generated above.
      75         228 :   for (size_t bookmark_pos = 0; bookmark_pos < tokens.size(); bookmark_pos++) {
      76         108 :     auto scanner = make_scanner(src_simple);
      77             :     Scanner::BookmarkScope bookmark(scanner.get());
      78             : 
      79        5508 :     for (size_t i = 0; i < std::min(bookmark_pos + 10, tokens.size()); i++) {
      80        1728 :       if (i == bookmark_pos) {
      81         108 :         bookmark.Set();
      82             :       }
      83        6912 :       CHECK_TOK(tokens[i], scanner->Next());
      84             :     }
      85             : 
      86         108 :     bookmark.Apply();
      87        2268 :     for (size_t i = bookmark_pos; i < tokens.size(); i++) {
      88        4104 :       CHECK_TOK(tokens[i], scanner->Next());
      89             :     }
      90         108 :   }
      91           6 : }
      92             : 
      93       23724 : TEST(AllThePushbacks) {
      94             :   const struct {
      95             :     const char* src;
      96             :     const Token::Value tokens[5];  // Large enough for any of the test cases.
      97             :   } test_cases[] = {
      98             :       {"<-x", {Token::LT, Token::SUB, Token::IDENTIFIER, Token::EOS}},
      99             :       {"<!x", {Token::LT, Token::NOT, Token::IDENTIFIER, Token::EOS}},
     100             :       {"<!-x",
     101             :        {Token::LT, Token::NOT, Token::SUB, Token::IDENTIFIER, Token::EOS}},
     102             :       {"<!-- xx -->\nx", {Token::IDENTIFIER, Token::EOS}},
     103           6 :   };
     104             : 
     105          30 :   for (const auto& test_case : test_cases) {
     106          24 :     auto scanner = make_scanner(test_case.src);
     107          90 :     for (size_t i = 0; test_case.tokens[i] != Token::EOS; i++) {
     108         132 :       CHECK_TOK(test_case.tokens[i], scanner->Next());
     109             :     }
     110          48 :     CHECK_TOK(Token::EOS, scanner->Next());
     111          24 :   }
     112           6 : }
     113             : 
     114       23724 : TEST(ContextualKeywordTokens) {
     115           6 :   auto scanner = make_scanner("function of get bla");
     116             : 
     117             :   // function (regular keyword)
     118           6 :   scanner->Next();
     119          12 :   CHECK_TOK(Token::FUNCTION, scanner->current_token());
     120          12 :   CHECK_TOK(Token::UNINITIALIZED, scanner->current_contextual_token());
     121             : 
     122             :   // of (contextual keyword)
     123           6 :   scanner->Next();
     124          12 :   CHECK_TOK(Token::IDENTIFIER, scanner->current_token());
     125          12 :   CHECK_TOK(Token::OF, scanner->current_contextual_token());
     126             : 
     127             :   // get (contextual keyword)
     128           6 :   scanner->Next();
     129          12 :   CHECK_TOK(Token::IDENTIFIER, scanner->current_token());
     130          12 :   CHECK_TOK(Token::GET, scanner->current_contextual_token());
     131             : 
     132             :   // bla (identfier, not any sort of keyword)
     133           6 :   scanner->Next();
     134          12 :   CHECK_TOK(Token::IDENTIFIER, scanner->current_token());
     135          12 :   CHECK_TOK(Token::UNINITIALIZED, scanner->current_contextual_token());
     136           6 : }
     137             : 
     138             : }  // namespace internal
     139       71154 : }  // namespace v8