/src/serenity/Userland/Libraries/LibJS/Token.h
Line | Count | Source |
1 | | /* |
2 | | * Copyright (c) 2020, Stephan Unverwerth <s.unverwerth@serenityos.org> |
3 | | * |
4 | | * SPDX-License-Identifier: BSD-2-Clause |
5 | | */ |
6 | | |
7 | | #pragma once |
8 | | |
9 | | #include <AK/DeprecatedFlyString.h> |
10 | | #include <AK/String.h> |
11 | | #include <AK/StringView.h> |
12 | | #include <AK/Variant.h> |
13 | | |
14 | | namespace JS { |
15 | | |
16 | | // U+2028 LINE SEPARATOR |
17 | | constexpr char const line_separator_chars[] { (char)0xe2, (char)0x80, (char)0xa8, 0 }; |
18 | | constexpr StringView const LINE_SEPARATOR_STRING { line_separator_chars, sizeof(line_separator_chars) - 1 }; |
19 | | constexpr u32 const LINE_SEPARATOR { 0x2028 }; |
20 | | |
21 | | // U+2029 PARAGRAPH SEPARATOR |
22 | | constexpr char const paragraph_separator_chars[] { (char)0xe2, (char)0x80, (char)0xa9, 0 }; |
23 | | constexpr StringView const PARAGRAPH_SEPARATOR_STRING { paragraph_separator_chars, sizeof(paragraph_separator_chars) - 1 }; |
24 | | constexpr u32 const PARAGRAPH_SEPARATOR { 0x2029 }; |
25 | | |
26 | | // U+00A0 NO BREAK SPACE |
27 | | constexpr u32 const NO_BREAK_SPACE { 0x00A0 }; |
28 | | |
29 | | // U+200C ZERO WIDTH NON-JOINER |
30 | | constexpr u32 const ZERO_WIDTH_NON_JOINER { 0x200C }; |
31 | | |
32 | | // U+FEFF ZERO WIDTH NO-BREAK SPACE |
33 | | constexpr u32 const ZERO_WIDTH_NO_BREAK_SPACE { 0xFEFF }; |
34 | | |
35 | | // U+200D ZERO WIDTH JOINER |
36 | | constexpr u32 const ZERO_WIDTH_JOINER { 0x200D }; |
37 | | |
38 | | #define ENUMERATE_JS_TOKENS \ |
39 | 0 | __ENUMERATE_JS_TOKEN(Ampersand, Operator) \ |
40 | 280 | __ENUMERATE_JS_TOKEN(AmpersandEquals, Operator) \ |
41 | 280 | __ENUMERATE_JS_TOKEN(Arrow, Operator) \ |
42 | 226 | __ENUMERATE_JS_TOKEN(Asterisk, Operator) \ |
43 | 226 | __ENUMERATE_JS_TOKEN(AsteriskEquals, Operator) \ |
44 | 187 | __ENUMERATE_JS_TOKEN(Async, Keyword) \ |
45 | 187 | __ENUMERATE_JS_TOKEN(Await, Keyword) \ |
46 | 0 | __ENUMERATE_JS_TOKEN(BigIntLiteral, Number) \ |
47 | 0 | __ENUMERATE_JS_TOKEN(BoolLiteral, Keyword) \ |
48 | 1.31k | __ENUMERATE_JS_TOKEN(BracketClose, Punctuation) \ |
49 | 1.31k | __ENUMERATE_JS_TOKEN(BracketOpen, Punctuation) \ |
50 | 52 | __ENUMERATE_JS_TOKEN(Break, Keyword) \ |
51 | 0 | __ENUMERATE_JS_TOKEN(Caret, Operator) \ |
52 | 0 | __ENUMERATE_JS_TOKEN(CaretEquals, Operator) \ |
53 | 0 | __ENUMERATE_JS_TOKEN(Case, ControlKeyword) \ |
54 | 0 | __ENUMERATE_JS_TOKEN(Catch, ControlKeyword) \ |
55 | 9 | __ENUMERATE_JS_TOKEN(Class, Keyword) \ |
56 | 441 | __ENUMERATE_JS_TOKEN(Colon, Punctuation) \ |
57 | 1.40k | __ENUMERATE_JS_TOKEN(Comma, Punctuation) \ |
58 | 1.40k | __ENUMERATE_JS_TOKEN(Const, Keyword) \ |
59 | 57 | __ENUMERATE_JS_TOKEN(Continue, ControlKeyword) \ |
60 | 4.83k | __ENUMERATE_JS_TOKEN(CurlyClose, Punctuation) \ |
61 | 150k | __ENUMERATE_JS_TOKEN(CurlyOpen, Punctuation) \ |
62 | 150k | __ENUMERATE_JS_TOKEN(Debugger, Keyword) \ |
63 | 0 | __ENUMERATE_JS_TOKEN(Default, ControlKeyword) \ |
64 | 0 | __ENUMERATE_JS_TOKEN(Delete, Keyword) \ |
65 | 0 | __ENUMERATE_JS_TOKEN(Do, ControlKeyword) \ |
66 | 0 | __ENUMERATE_JS_TOKEN(DoubleAmpersand, Operator) \ |
67 | 0 | __ENUMERATE_JS_TOKEN(DoubleAmpersandEquals, Operator) \ |
68 | 0 | __ENUMERATE_JS_TOKEN(DoubleAsterisk, Operator) \ |
69 | 0 | __ENUMERATE_JS_TOKEN(DoubleAsteriskEquals, Operator) \ |
70 | 0 | __ENUMERATE_JS_TOKEN(DoublePipe, Operator) \ |
71 | 0 | __ENUMERATE_JS_TOKEN(DoublePipeEquals, Operator) \ |
72 | 0 | __ENUMERATE_JS_TOKEN(DoubleQuestionMark, Operator) \ |
73 | 0 | __ENUMERATE_JS_TOKEN(DoubleQuestionMarkEquals, Operator) \ |
74 | 68 | __ENUMERATE_JS_TOKEN(Else, ControlKeyword) \ |
75 | 68 | __ENUMERATE_JS_TOKEN(Enum, Keyword) \ |
76 | 4.93k | __ENUMERATE_JS_TOKEN(Eof, Invalid) \ |
77 | 4.93k | __ENUMERATE_JS_TOKEN(Equals, Operator) \ |
78 | 2.11k | __ENUMERATE_JS_TOKEN(EqualsEquals, Operator) \ |
79 | 76 | __ENUMERATE_JS_TOKEN(EqualsEqualsEquals, Operator) \ |
80 | 0 | __ENUMERATE_JS_TOKEN(EscapedKeyword, Identifier) \ |
81 | 712 | __ENUMERATE_JS_TOKEN(ExclamationMark, Operator) \ |
82 | 712 | __ENUMERATE_JS_TOKEN(ExclamationMarkEquals, Operator) \ |
83 | 136 | __ENUMERATE_JS_TOKEN(ExclamationMarkEqualsEquals, Operator) \ |
84 | 136 | __ENUMERATE_JS_TOKEN(Export, Keyword) \ |
85 | 0 | __ENUMERATE_JS_TOKEN(Extends, Keyword) \ |
86 | 0 | __ENUMERATE_JS_TOKEN(Finally, ControlKeyword) \ |
87 | 68 | __ENUMERATE_JS_TOKEN(For, ControlKeyword) \ |
88 | 68 | __ENUMERATE_JS_TOKEN(Function, Keyword) \ |
89 | 36 | __ENUMERATE_JS_TOKEN(GreaterThan, Operator) \ |
90 | 36 | __ENUMERATE_JS_TOKEN(GreaterThanEquals, Operator) \ |
91 | 4.07k | __ENUMERATE_JS_TOKEN(Identifier, Identifier) \ |
92 | 4.07k | __ENUMERATE_JS_TOKEN(If, ControlKeyword) \ |
93 | 68 | __ENUMERATE_JS_TOKEN(Implements, Keyword) \ |
94 | 0 | __ENUMERATE_JS_TOKEN(Import, Keyword) \ |
95 | 0 | __ENUMERATE_JS_TOKEN(In, Keyword) \ |
96 | 0 | __ENUMERATE_JS_TOKEN(Instanceof, Keyword) \ |
97 | 0 | __ENUMERATE_JS_TOKEN(Interface, Keyword) \ |
98 | 11.2k | __ENUMERATE_JS_TOKEN(Invalid, Invalid) \ |
99 | 11.2k | __ENUMERATE_JS_TOKEN(LessThan, Operator) \ |
100 | 68 | __ENUMERATE_JS_TOKEN(LessThanEquals, Operator) \ |
101 | 68 | __ENUMERATE_JS_TOKEN(Let, Keyword) \ |
102 | 1 | __ENUMERATE_JS_TOKEN(Minus, Operator) \ |
103 | 68 | __ENUMERATE_JS_TOKEN(MinusEquals, Operator) \ |
104 | 68 | __ENUMERATE_JS_TOKEN(MinusMinus, Operator) \ |
105 | 0 | __ENUMERATE_JS_TOKEN(New, Keyword) \ |
106 | 0 | __ENUMERATE_JS_TOKEN(NullLiteral, Keyword) \ |
107 | 394 | __ENUMERATE_JS_TOKEN(NumericLiteral, Number) \ |
108 | 394 | __ENUMERATE_JS_TOKEN(Package, Keyword) \ |
109 | 11.8k | __ENUMERATE_JS_TOKEN(ParenClose, Punctuation) \ |
110 | 11.8k | __ENUMERATE_JS_TOKEN(ParenOpen, Punctuation) \ |
111 | 1.69k | __ENUMERATE_JS_TOKEN(Percent, Operator) \ |
112 | 0 | __ENUMERATE_JS_TOKEN(PercentEquals, Operator) \ |
113 | 0 | __ENUMERATE_JS_TOKEN(Period, Operator) \ |
114 | 0 | __ENUMERATE_JS_TOKEN(Pipe, Operator) \ |
115 | 0 | __ENUMERATE_JS_TOKEN(PipeEquals, Operator) \ |
116 | 1 | __ENUMERATE_JS_TOKEN(Plus, Operator) \ |
117 | 1 | __ENUMERATE_JS_TOKEN(PlusEquals, Operator) \ |
118 | 0 | __ENUMERATE_JS_TOKEN(PlusPlus, Operator) \ |
119 | 0 | __ENUMERATE_JS_TOKEN(Private, Keyword) \ |
120 | 200 | __ENUMERATE_JS_TOKEN(PrivateIdentifier, Identifier) \ |
121 | 200 | __ENUMERATE_JS_TOKEN(Protected, Keyword) \ |
122 | 0 | __ENUMERATE_JS_TOKEN(Public, Keyword) \ |
123 | 377 | __ENUMERATE_JS_TOKEN(QuestionMark, Operator) \ |
124 | 377 | __ENUMERATE_JS_TOKEN(QuestionMarkPeriod, Operator) \ |
125 | 0 | __ENUMERATE_JS_TOKEN(RegexFlags, String) \ |
126 | 0 | __ENUMERATE_JS_TOKEN(RegexLiteral, String) \ |
127 | 0 | __ENUMERATE_JS_TOKEN(Return, ControlKeyword) \ |
128 | 264 | __ENUMERATE_JS_TOKEN(Semicolon, Punctuation) \ |
129 | 264 | __ENUMERATE_JS_TOKEN(ShiftLeft, Operator) \ |
130 | 3 | __ENUMERATE_JS_TOKEN(ShiftLeftEquals, Operator) \ |
131 | 4 | __ENUMERATE_JS_TOKEN(ShiftRight, Operator) \ |
132 | 4 | __ENUMERATE_JS_TOKEN(ShiftRightEquals, Operator) \ |
133 | 80 | __ENUMERATE_JS_TOKEN(Slash, Operator) \ |
134 | 80 | __ENUMERATE_JS_TOKEN(SlashEquals, Operator) \ |
135 | 0 | __ENUMERATE_JS_TOKEN(Static, Keyword) \ |
136 | 262k | __ENUMERATE_JS_TOKEN(StringLiteral, String) \ |
137 | 262k | __ENUMERATE_JS_TOKEN(Super, Keyword) \ |
138 | 0 | __ENUMERATE_JS_TOKEN(Switch, ControlKeyword) \ |
139 | 0 | __ENUMERATE_JS_TOKEN(TemplateLiteralEnd, String) \ |
140 | 0 | __ENUMERATE_JS_TOKEN(TemplateLiteralExprEnd, Punctuation) \ |
141 | 0 | __ENUMERATE_JS_TOKEN(TemplateLiteralExprStart, Punctuation) \ |
142 | 0 | __ENUMERATE_JS_TOKEN(TemplateLiteralStart, String) \ |
143 | 0 | __ENUMERATE_JS_TOKEN(TemplateLiteralString, String) \ |
144 | 0 | __ENUMERATE_JS_TOKEN(This, Keyword) \ |
145 | 0 | __ENUMERATE_JS_TOKEN(Throw, ControlKeyword) \ |
146 | 348 | __ENUMERATE_JS_TOKEN(Tilde, Operator) \ |
147 | 348 | __ENUMERATE_JS_TOKEN(TripleDot, Operator) \ |
148 | 0 | __ENUMERATE_JS_TOKEN(Trivia, Trivia) \ |
149 | 0 | __ENUMERATE_JS_TOKEN(Try, ControlKeyword) \ |
150 | 0 | __ENUMERATE_JS_TOKEN(Typeof, Keyword) \ |
151 | 0 | __ENUMERATE_JS_TOKEN(UnsignedShiftRight, Operator) \ |
152 | 0 | __ENUMERATE_JS_TOKEN(UnsignedShiftRightEquals, Operator) \ |
153 | 80 | __ENUMERATE_JS_TOKEN(UnterminatedRegexLiteral, String) \ |
154 | 706 | __ENUMERATE_JS_TOKEN(UnterminatedStringLiteral, String) \ |
155 | 706 | __ENUMERATE_JS_TOKEN(UnterminatedTemplateLiteral, String) \ |
156 | 51 | __ENUMERATE_JS_TOKEN(Var, Keyword) \ |
157 | 51 | __ENUMERATE_JS_TOKEN(Void, Keyword) \ |
158 | 0 | __ENUMERATE_JS_TOKEN(While, ControlKeyword) \ |
159 | 320 | __ENUMERATE_JS_TOKEN(With, ControlKeyword) \ |
160 | 320 | __ENUMERATE_JS_TOKEN(Yield, ControlKeyword) |
161 | | |
162 | | enum class TokenType { |
163 | | #define __ENUMERATE_JS_TOKEN(type, category) type, |
164 | | ENUMERATE_JS_TOKENS |
165 | | #undef __ENUMERATE_JS_TOKEN |
166 | | _COUNT_OF_TOKENS |
167 | | }; |
168 | | constexpr size_t cs_num_of_js_tokens = static_cast<size_t>(TokenType::_COUNT_OF_TOKENS); |
169 | | |
170 | | enum class TokenCategory { |
171 | | Invalid, |
172 | | Trivia, |
173 | | Number, |
174 | | String, |
175 | | Punctuation, |
176 | | Operator, |
177 | | Keyword, |
178 | | ControlKeyword, |
179 | | Identifier |
180 | | }; |
181 | | |
182 | | class Token { |
183 | | public: |
184 | 1.08M | Token() = default; |
185 | | |
186 | | Token(TokenType type, StringView message, StringView trivia, StringView value, size_t line_number, size_t line_column, size_t offset) |
187 | 6.76M | : m_type(type) |
188 | 6.76M | , m_message(message) |
189 | 6.76M | , m_trivia(trivia) |
190 | 6.76M | , m_original_value(value) |
191 | 6.76M | , m_value(value) |
192 | 6.76M | , m_line_number(line_number) |
193 | 6.76M | , m_line_column(line_column) |
194 | 6.76M | , m_offset(offset) |
195 | 6.76M | { |
196 | 6.76M | } |
197 | | |
198 | 60.3M | TokenType type() const { return m_type; } |
199 | | TokenCategory category() const; |
200 | | static TokenCategory category(TokenType); |
201 | | char const* name() const; |
202 | | static char const* name(TokenType); |
203 | | |
204 | 444k | StringView message() const { return m_message; } |
205 | 2.93k | StringView trivia() const { return m_trivia; } |
206 | 177k | StringView original_value() const { return m_original_value; } |
207 | | StringView value() const |
208 | 4.00M | { |
209 | 4.00M | return m_value.visit( |
210 | 4.00M | [](StringView view) { return view; }, |
211 | 4.00M | [](DeprecatedFlyString const& identifier) { return identifier.view(); }, |
212 | 4.00M | [](Empty) -> StringView { VERIFY_NOT_REACHED(); }); |
213 | 4.00M | } |
214 | | |
215 | | DeprecatedFlyString DeprecatedFlyString_value() const |
216 | 817k | { |
217 | 817k | return m_value.visit( |
218 | 817k | [](StringView view) -> DeprecatedFlyString { return view; }, |
219 | 817k | [](DeprecatedFlyString const& identifier) -> DeprecatedFlyString { return identifier; }, |
220 | 817k | [](Empty) -> DeprecatedFlyString { VERIFY_NOT_REACHED(); }); |
221 | 817k | } |
222 | | |
223 | 15.4M | size_t line_number() const { return m_line_number; } |
224 | 15.4M | size_t line_column() const { return m_line_column; } |
225 | 15.4M | size_t offset() const { return m_offset; } |
226 | | double double_value() const; |
227 | | bool bool_value() const; |
228 | | |
229 | | enum class StringValueStatus { |
230 | | Ok, |
231 | | MalformedHexEscape, |
232 | | MalformedUnicodeEscape, |
233 | | UnicodeEscapeOverflow, |
234 | | LegacyOctalEscapeSequence, |
235 | | }; |
236 | | ByteString string_value(StringValueStatus& status) const; |
237 | | ByteString raw_template_value() const; |
238 | | |
239 | | void set_identifier_value(DeprecatedFlyString value) |
240 | 825k | { |
241 | 825k | m_value = move(value); |
242 | 825k | } |
243 | | |
244 | | bool is_identifier_name() const; |
245 | | bool trivia_contains_line_terminator() const; |
246 | | |
247 | | private: |
248 | | TokenType m_type { TokenType::Invalid }; |
249 | | StringView m_message; |
250 | | StringView m_trivia; |
251 | | StringView m_original_value; |
252 | | Variant<Empty, StringView, DeprecatedFlyString> m_value {}; |
253 | | size_t m_line_number { 0 }; |
254 | | size_t m_line_column { 0 }; |
255 | | size_t m_offset { 0 }; |
256 | | }; |
257 | | |
258 | | } |