Coverage Report

Created: 2026-10-28 07:05

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/src/serenity/Userland/Libraries/LibJS/Token.h
Line
Count
Source
1
/*
2
 * Copyright (c) 2020, Stephan Unverwerth <s.unverwerth@serenityos.org>
3
 *
4
 * SPDX-License-Identifier: BSD-2-Clause
5
 */
6
7
#pragma once
8
9
#include <AK/DeprecatedFlyString.h>
10
#include <AK/String.h>
11
#include <AK/StringView.h>
12
#include <AK/Variant.h>
13
14
namespace JS {
15
16
// U+2028 LINE SEPARATOR
17
constexpr char const line_separator_chars[] { (char)0xe2, (char)0x80, (char)0xa8, 0 };
18
constexpr StringView const LINE_SEPARATOR_STRING { line_separator_chars, sizeof(line_separator_chars) - 1 };
19
constexpr u32 const LINE_SEPARATOR { 0x2028 };
20
21
// U+2029 PARAGRAPH SEPARATOR
22
constexpr char const paragraph_separator_chars[] { (char)0xe2, (char)0x80, (char)0xa9, 0 };
23
constexpr StringView const PARAGRAPH_SEPARATOR_STRING { paragraph_separator_chars, sizeof(paragraph_separator_chars) - 1 };
24
constexpr u32 const PARAGRAPH_SEPARATOR { 0x2029 };
25
26
// U+00A0 NO BREAK SPACE
27
constexpr u32 const NO_BREAK_SPACE { 0x00A0 };
28
29
// U+200C ZERO WIDTH NON-JOINER
30
constexpr u32 const ZERO_WIDTH_NON_JOINER { 0x200C };
31
32
// U+FEFF ZERO WIDTH NO-BREAK SPACE
33
constexpr u32 const ZERO_WIDTH_NO_BREAK_SPACE { 0xFEFF };
34
35
// U+200D ZERO WIDTH JOINER
36
constexpr u32 const ZERO_WIDTH_JOINER { 0x200D };
37
38
#define ENUMERATE_JS_TOKENS                                     \
39
0
    __ENUMERATE_JS_TOKEN(Ampersand, Operator)                   \
40
280
    __ENUMERATE_JS_TOKEN(AmpersandEquals, Operator)             \
41
280
    __ENUMERATE_JS_TOKEN(Arrow, Operator)                       \
42
226
    __ENUMERATE_JS_TOKEN(Asterisk, Operator)                    \
43
226
    __ENUMERATE_JS_TOKEN(AsteriskEquals, Operator)              \
44
187
    __ENUMERATE_JS_TOKEN(Async, Keyword)                        \
45
187
    __ENUMERATE_JS_TOKEN(Await, Keyword)                        \
46
0
    __ENUMERATE_JS_TOKEN(BigIntLiteral, Number)                 \
47
0
    __ENUMERATE_JS_TOKEN(BoolLiteral, Keyword)                  \
48
1.31k
    __ENUMERATE_JS_TOKEN(BracketClose, Punctuation)             \
49
1.31k
    __ENUMERATE_JS_TOKEN(BracketOpen, Punctuation)              \
50
52
    __ENUMERATE_JS_TOKEN(Break, Keyword)                        \
51
0
    __ENUMERATE_JS_TOKEN(Caret, Operator)                       \
52
0
    __ENUMERATE_JS_TOKEN(CaretEquals, Operator)                 \
53
0
    __ENUMERATE_JS_TOKEN(Case, ControlKeyword)                  \
54
0
    __ENUMERATE_JS_TOKEN(Catch, ControlKeyword)                 \
55
9
    __ENUMERATE_JS_TOKEN(Class, Keyword)                        \
56
441
    __ENUMERATE_JS_TOKEN(Colon, Punctuation)                    \
57
1.40k
    __ENUMERATE_JS_TOKEN(Comma, Punctuation)                    \
58
1.40k
    __ENUMERATE_JS_TOKEN(Const, Keyword)                        \
59
57
    __ENUMERATE_JS_TOKEN(Continue, ControlKeyword)              \
60
4.83k
    __ENUMERATE_JS_TOKEN(CurlyClose, Punctuation)               \
61
150k
    __ENUMERATE_JS_TOKEN(CurlyOpen, Punctuation)                \
62
150k
    __ENUMERATE_JS_TOKEN(Debugger, Keyword)                     \
63
0
    __ENUMERATE_JS_TOKEN(Default, ControlKeyword)               \
64
0
    __ENUMERATE_JS_TOKEN(Delete, Keyword)                       \
65
0
    __ENUMERATE_JS_TOKEN(Do, ControlKeyword)                    \
66
0
    __ENUMERATE_JS_TOKEN(DoubleAmpersand, Operator)             \
67
0
    __ENUMERATE_JS_TOKEN(DoubleAmpersandEquals, Operator)       \
68
0
    __ENUMERATE_JS_TOKEN(DoubleAsterisk, Operator)              \
69
0
    __ENUMERATE_JS_TOKEN(DoubleAsteriskEquals, Operator)        \
70
0
    __ENUMERATE_JS_TOKEN(DoublePipe, Operator)                  \
71
0
    __ENUMERATE_JS_TOKEN(DoublePipeEquals, Operator)            \
72
0
    __ENUMERATE_JS_TOKEN(DoubleQuestionMark, Operator)          \
73
0
    __ENUMERATE_JS_TOKEN(DoubleQuestionMarkEquals, Operator)    \
74
68
    __ENUMERATE_JS_TOKEN(Else, ControlKeyword)                  \
75
68
    __ENUMERATE_JS_TOKEN(Enum, Keyword)                         \
76
4.93k
    __ENUMERATE_JS_TOKEN(Eof, Invalid)                          \
77
4.93k
    __ENUMERATE_JS_TOKEN(Equals, Operator)                      \
78
2.11k
    __ENUMERATE_JS_TOKEN(EqualsEquals, Operator)                \
79
76
    __ENUMERATE_JS_TOKEN(EqualsEqualsEquals, Operator)          \
80
0
    __ENUMERATE_JS_TOKEN(EscapedKeyword, Identifier)            \
81
712
    __ENUMERATE_JS_TOKEN(ExclamationMark, Operator)             \
82
712
    __ENUMERATE_JS_TOKEN(ExclamationMarkEquals, Operator)       \
83
136
    __ENUMERATE_JS_TOKEN(ExclamationMarkEqualsEquals, Operator) \
84
136
    __ENUMERATE_JS_TOKEN(Export, Keyword)                       \
85
0
    __ENUMERATE_JS_TOKEN(Extends, Keyword)                      \
86
0
    __ENUMERATE_JS_TOKEN(Finally, ControlKeyword)               \
87
68
    __ENUMERATE_JS_TOKEN(For, ControlKeyword)                   \
88
68
    __ENUMERATE_JS_TOKEN(Function, Keyword)                     \
89
36
    __ENUMERATE_JS_TOKEN(GreaterThan, Operator)                 \
90
36
    __ENUMERATE_JS_TOKEN(GreaterThanEquals, Operator)           \
91
4.07k
    __ENUMERATE_JS_TOKEN(Identifier, Identifier)                \
92
4.07k
    __ENUMERATE_JS_TOKEN(If, ControlKeyword)                    \
93
68
    __ENUMERATE_JS_TOKEN(Implements, Keyword)                   \
94
0
    __ENUMERATE_JS_TOKEN(Import, Keyword)                       \
95
0
    __ENUMERATE_JS_TOKEN(In, Keyword)                           \
96
0
    __ENUMERATE_JS_TOKEN(Instanceof, Keyword)                   \
97
0
    __ENUMERATE_JS_TOKEN(Interface, Keyword)                    \
98
11.2k
    __ENUMERATE_JS_TOKEN(Invalid, Invalid)                      \
99
11.2k
    __ENUMERATE_JS_TOKEN(LessThan, Operator)                    \
100
68
    __ENUMERATE_JS_TOKEN(LessThanEquals, Operator)              \
101
68
    __ENUMERATE_JS_TOKEN(Let, Keyword)                          \
102
1
    __ENUMERATE_JS_TOKEN(Minus, Operator)                       \
103
68
    __ENUMERATE_JS_TOKEN(MinusEquals, Operator)                 \
104
68
    __ENUMERATE_JS_TOKEN(MinusMinus, Operator)                  \
105
0
    __ENUMERATE_JS_TOKEN(New, Keyword)                          \
106
0
    __ENUMERATE_JS_TOKEN(NullLiteral, Keyword)                  \
107
394
    __ENUMERATE_JS_TOKEN(NumericLiteral, Number)                \
108
394
    __ENUMERATE_JS_TOKEN(Package, Keyword)                      \
109
11.8k
    __ENUMERATE_JS_TOKEN(ParenClose, Punctuation)               \
110
11.8k
    __ENUMERATE_JS_TOKEN(ParenOpen, Punctuation)                \
111
1.69k
    __ENUMERATE_JS_TOKEN(Percent, Operator)                     \
112
0
    __ENUMERATE_JS_TOKEN(PercentEquals, Operator)               \
113
0
    __ENUMERATE_JS_TOKEN(Period, Operator)                      \
114
0
    __ENUMERATE_JS_TOKEN(Pipe, Operator)                        \
115
0
    __ENUMERATE_JS_TOKEN(PipeEquals, Operator)                  \
116
1
    __ENUMERATE_JS_TOKEN(Plus, Operator)                        \
117
1
    __ENUMERATE_JS_TOKEN(PlusEquals, Operator)                  \
118
0
    __ENUMERATE_JS_TOKEN(PlusPlus, Operator)                    \
119
0
    __ENUMERATE_JS_TOKEN(Private, Keyword)                      \
120
200
    __ENUMERATE_JS_TOKEN(PrivateIdentifier, Identifier)         \
121
200
    __ENUMERATE_JS_TOKEN(Protected, Keyword)                    \
122
0
    __ENUMERATE_JS_TOKEN(Public, Keyword)                       \
123
377
    __ENUMERATE_JS_TOKEN(QuestionMark, Operator)                \
124
377
    __ENUMERATE_JS_TOKEN(QuestionMarkPeriod, Operator)          \
125
0
    __ENUMERATE_JS_TOKEN(RegexFlags, String)                    \
126
0
    __ENUMERATE_JS_TOKEN(RegexLiteral, String)                  \
127
0
    __ENUMERATE_JS_TOKEN(Return, ControlKeyword)                \
128
264
    __ENUMERATE_JS_TOKEN(Semicolon, Punctuation)                \
129
264
    __ENUMERATE_JS_TOKEN(ShiftLeft, Operator)                   \
130
3
    __ENUMERATE_JS_TOKEN(ShiftLeftEquals, Operator)             \
131
4
    __ENUMERATE_JS_TOKEN(ShiftRight, Operator)                  \
132
4
    __ENUMERATE_JS_TOKEN(ShiftRightEquals, Operator)            \
133
80
    __ENUMERATE_JS_TOKEN(Slash, Operator)                       \
134
80
    __ENUMERATE_JS_TOKEN(SlashEquals, Operator)                 \
135
0
    __ENUMERATE_JS_TOKEN(Static, Keyword)                       \
136
262k
    __ENUMERATE_JS_TOKEN(StringLiteral, String)                 \
137
262k
    __ENUMERATE_JS_TOKEN(Super, Keyword)                        \
138
0
    __ENUMERATE_JS_TOKEN(Switch, ControlKeyword)                \
139
0
    __ENUMERATE_JS_TOKEN(TemplateLiteralEnd, String)            \
140
0
    __ENUMERATE_JS_TOKEN(TemplateLiteralExprEnd, Punctuation)   \
141
0
    __ENUMERATE_JS_TOKEN(TemplateLiteralExprStart, Punctuation) \
142
0
    __ENUMERATE_JS_TOKEN(TemplateLiteralStart, String)          \
143
0
    __ENUMERATE_JS_TOKEN(TemplateLiteralString, String)         \
144
0
    __ENUMERATE_JS_TOKEN(This, Keyword)                         \
145
0
    __ENUMERATE_JS_TOKEN(Throw, ControlKeyword)                 \
146
348
    __ENUMERATE_JS_TOKEN(Tilde, Operator)                       \
147
348
    __ENUMERATE_JS_TOKEN(TripleDot, Operator)                   \
148
0
    __ENUMERATE_JS_TOKEN(Trivia, Trivia)                        \
149
0
    __ENUMERATE_JS_TOKEN(Try, ControlKeyword)                   \
150
0
    __ENUMERATE_JS_TOKEN(Typeof, Keyword)                       \
151
0
    __ENUMERATE_JS_TOKEN(UnsignedShiftRight, Operator)          \
152
0
    __ENUMERATE_JS_TOKEN(UnsignedShiftRightEquals, Operator)    \
153
80
    __ENUMERATE_JS_TOKEN(UnterminatedRegexLiteral, String)      \
154
706
    __ENUMERATE_JS_TOKEN(UnterminatedStringLiteral, String)     \
155
706
    __ENUMERATE_JS_TOKEN(UnterminatedTemplateLiteral, String)   \
156
51
    __ENUMERATE_JS_TOKEN(Var, Keyword)                          \
157
51
    __ENUMERATE_JS_TOKEN(Void, Keyword)                         \
158
0
    __ENUMERATE_JS_TOKEN(While, ControlKeyword)                 \
159
320
    __ENUMERATE_JS_TOKEN(With, ControlKeyword)                  \
160
320
    __ENUMERATE_JS_TOKEN(Yield, ControlKeyword)
161
162
enum class TokenType {
163
#define __ENUMERATE_JS_TOKEN(type, category) type,
164
    ENUMERATE_JS_TOKENS
165
#undef __ENUMERATE_JS_TOKEN
166
        _COUNT_OF_TOKENS
167
};
168
constexpr size_t cs_num_of_js_tokens = static_cast<size_t>(TokenType::_COUNT_OF_TOKENS);
169
170
enum class TokenCategory {
171
    Invalid,
172
    Trivia,
173
    Number,
174
    String,
175
    Punctuation,
176
    Operator,
177
    Keyword,
178
    ControlKeyword,
179
    Identifier
180
};
181
182
class Token {
183
public:
184
1.08M
    Token() = default;
185
186
    Token(TokenType type, StringView message, StringView trivia, StringView value, size_t line_number, size_t line_column, size_t offset)
187
6.76M
        : m_type(type)
188
6.76M
        , m_message(message)
189
6.76M
        , m_trivia(trivia)
190
6.76M
        , m_original_value(value)
191
6.76M
        , m_value(value)
192
6.76M
        , m_line_number(line_number)
193
6.76M
        , m_line_column(line_column)
194
6.76M
        , m_offset(offset)
195
6.76M
    {
196
6.76M
    }
197
198
60.3M
    TokenType type() const { return m_type; }
199
    TokenCategory category() const;
200
    static TokenCategory category(TokenType);
201
    char const* name() const;
202
    static char const* name(TokenType);
203
204
444k
    StringView message() const { return m_message; }
205
2.93k
    StringView trivia() const { return m_trivia; }
206
177k
    StringView original_value() const { return m_original_value; }
207
    StringView value() const
208
4.00M
    {
209
4.00M
        return m_value.visit(
210
4.00M
            [](StringView view) { return view; },
211
4.00M
            [](DeprecatedFlyString const& identifier) { return identifier.view(); },
212
4.00M
            [](Empty) -> StringView { VERIFY_NOT_REACHED(); });
213
4.00M
    }
214
215
    DeprecatedFlyString DeprecatedFlyString_value() const
216
817k
    {
217
817k
        return m_value.visit(
218
817k
            [](StringView view) -> DeprecatedFlyString { return view; },
219
817k
            [](DeprecatedFlyString const& identifier) -> DeprecatedFlyString { return identifier; },
220
817k
            [](Empty) -> DeprecatedFlyString { VERIFY_NOT_REACHED(); });
221
817k
    }
222
223
15.4M
    size_t line_number() const { return m_line_number; }
224
15.4M
    size_t line_column() const { return m_line_column; }
225
15.4M
    size_t offset() const { return m_offset; }
226
    double double_value() const;
227
    bool bool_value() const;
228
229
    enum class StringValueStatus {
230
        Ok,
231
        MalformedHexEscape,
232
        MalformedUnicodeEscape,
233
        UnicodeEscapeOverflow,
234
        LegacyOctalEscapeSequence,
235
    };
236
    ByteString string_value(StringValueStatus& status) const;
237
    ByteString raw_template_value() const;
238
239
    void set_identifier_value(DeprecatedFlyString value)
240
825k
    {
241
825k
        m_value = move(value);
242
825k
    }
243
244
    bool is_identifier_name() const;
245
    bool trivia_contains_line_terminator() const;
246
247
private:
248
    TokenType m_type { TokenType::Invalid };
249
    StringView m_message;
250
    StringView m_trivia;
251
    StringView m_original_value;
252
    Variant<Empty, StringView, DeprecatedFlyString> m_value {};
253
    size_t m_line_number { 0 };
254
    size_t m_line_column { 0 };
255
    size_t m_offset { 0 };
256
};
257
258
}