/src/serenity/Userland/Libraries/LibSQL/AST/Token.h

Source
/*
 * Copyright (c) 2021, Tim Flynn <trflynn89@serenityos.org>
 * Copyright (c) 2021, Jan de Visser <jan@de-visser.net>
 * Copyright (c) 2021, Mahmoud Mandour <ma.mandourr@gmail.com>
 *
 * SPDX-License-Identifier: BSD-2-Clause
 */

#pragma once

#include <AK/ByteString.h>
#include <AK/HashMap.h>
#include <AK/StringView.h>

namespace SQL::AST {

// https://sqlite.org/lang_keywords.html
#define ENUMERATE_SQL_TOKENS                                              \
    __ENUMERATE_SQL_TOKEN("ABORT", Abort, Keyword)                        \
    __ENUMERATE_SQL_TOKEN("ACTION", Action, Keyword)                      \
    __ENUMERATE_SQL_TOKEN("ADD", Add, Keyword)                            \
    __ENUMERATE_SQL_TOKEN("AFTER", After, Keyword)                        \
    __ENUMERATE_SQL_TOKEN("ALL", All, Keyword)                            \
    __ENUMERATE_SQL_TOKEN("ALTER", Alter, Keyword)                        \
    __ENUMERATE_SQL_TOKEN("ALWAYS", Always, Keyword)                      \
    __ENUMERATE_SQL_TOKEN("ANALYZE", Analyze, Keyword)                    \
    __ENUMERATE_SQL_TOKEN("AND", And, Keyword)                            \
    __ENUMERATE_SQL_TOKEN("AS", As, Keyword)                              \
    __ENUMERATE_SQL_TOKEN("ASC", Asc, Keyword)                            \
    __ENUMERATE_SQL_TOKEN("ATTACH", Attach, Keyword)                      \
    __ENUMERATE_SQL_TOKEN("AUTOINCREMENT", Autoincrement, Keyword)        \
    __ENUMERATE_SQL_TOKEN("BEFORE", Before, Keyword)                      \
    __ENUMERATE_SQL_TOKEN("BEGIN", Begin, Keyword)                        \
    __ENUMERATE_SQL_TOKEN("BETWEEN", Between, Keyword)                    \
    __ENUMERATE_SQL_TOKEN("BY", By, Keyword)                              \
    __ENUMERATE_SQL_TOKEN("CASCADE", Cascade, Keyword)                    \
    __ENUMERATE_SQL_TOKEN("CASE", Case, Keyword)                          \
    __ENUMERATE_SQL_TOKEN("CAST", Cast, Keyword)                          \
    __ENUMERATE_SQL_TOKEN("CHECK", Check, Keyword)                        \
    __ENUMERATE_SQL_TOKEN("COLLATE", Collate, Keyword)                    \
    __ENUMERATE_SQL_TOKEN("COLUMN", Column, Keyword)                      \
    __ENUMERATE_SQL_TOKEN("COMMIT", Commit, Keyword)                      \
    __ENUMERATE_SQL_TOKEN("CONFLICT", Conflict, Keyword)                  \
    __ENUMERATE_SQL_TOKEN("CONSTRAINT", Constraint, Keyword)              \
    __ENUMERATE_SQL_TOKEN("CREATE", Create, Keyword)                      \
    __ENUMERATE_SQL_TOKEN("CROSS", Cross, Keyword)                        \
    __ENUMERATE_SQL_TOKEN("CURRENT", Current, Keyword)                    \
    __ENUMERATE_SQL_TOKEN("CURRENT_DATE", CurrentDate, Keyword)           \
    __ENUMERATE_SQL_TOKEN("CURRENT_TIME", CurrentTime, Keyword)           \
    __ENUMERATE_SQL_TOKEN("CURRENT_TIMESTAMP", CurrentTimestamp, Keyword) \
    __ENUMERATE_SQL_TOKEN("DATABASE", Database, Keyword)                  \
    __ENUMERATE_SQL_TOKEN("DEFAULT", Default, Keyword)                    \
    __ENUMERATE_SQL_TOKEN("DEFERRABLE", Deferrable, Keyword)              \
    __ENUMERATE_SQL_TOKEN("DEFERRED", Deferred, Keyword)                  \
    __ENUMERATE_SQL_TOKEN("DELETE", Delete, Keyword)                      \
    __ENUMERATE_SQL_TOKEN("DESC", Desc, Keyword)                          \
    __ENUMERATE_SQL_TOKEN("DESCRIBE", Describe, Keyword)                  \
    __ENUMERATE_SQL_TOKEN("DETACH", Detach, Keyword)                      \
    __ENUMERATE_SQL_TOKEN("DISTINCT", Distinct, Keyword)                  \
    __ENUMERATE_SQL_TOKEN("DO", Do, Keyword)                              \
    __ENUMERATE_SQL_TOKEN("DROP", Drop, Keyword)                          \
    __ENUMERATE_SQL_TOKEN("EACH", Each, Keyword)                          \
    __ENUMERATE_SQL_TOKEN("ELSE", Else, Keyword)                          \
    __ENUMERATE_SQL_TOKEN("END", End, Keyword)                            \
    __ENUMERATE_SQL_TOKEN("ESCAPE", Escape, Keyword)                      \
    __ENUMERATE_SQL_TOKEN("EXCEPT", Except, Keyword)                      \
    __ENUMERATE_SQL_TOKEN("EXCLUDE", Exclude, Keyword)                    \
    __ENUMERATE_SQL_TOKEN("EXCLUSIVE", Exclusive, Keyword)                \
    __ENUMERATE_SQL_TOKEN("EXISTS", Exists, Keyword)                      \
    __ENUMERATE_SQL_TOKEN("EXPLAIN", Explain, Keyword)                    \
    __ENUMERATE_SQL_TOKEN("FAIL", Fail, Keyword)                          \
    __ENUMERATE_SQL_TOKEN("FALSE", False, Keyword)                        \
    __ENUMERATE_SQL_TOKEN("FILTER", Filter, Keyword)                      \
    __ENUMERATE_SQL_TOKEN("FIRST", First, Keyword)                        \
    __ENUMERATE_SQL_TOKEN("FOLLOWING", Following, Keyword)                \
    __ENUMERATE_SQL_TOKEN("FOR", For, Keyword)                            \
    __ENUMERATE_SQL_TOKEN("FOREIGN", Foreign, Keyword)                    \
    __ENUMERATE_SQL_TOKEN("FROM", From, Keyword)                          \
    __ENUMERATE_SQL_TOKEN("FULL", Full, Keyword)                          \
    __ENUMERATE_SQL_TOKEN("GENERATED", Generated, Keyword)                \
    __ENUMERATE_SQL_TOKEN("GLOB", Glob, Keyword)                          \
    __ENUMERATE_SQL_TOKEN("GROUP", Group, Keyword)                        \
    __ENUMERATE_SQL_TOKEN("GROUPS", Groups, Keyword)                      \
    __ENUMERATE_SQL_TOKEN("HAVING", Having, Keyword)                      \
    __ENUMERATE_SQL_TOKEN("IF", If, Keyword)                              \
    __ENUMERATE_SQL_TOKEN("IGNORE", Ignore, Keyword)                      \
    __ENUMERATE_SQL_TOKEN("IMMEDIATE", Immediate, Keyword)                \
    __ENUMERATE_SQL_TOKEN("IN", In, Keyword)                              \
    __ENUMERATE_SQL_TOKEN("INDEX", Index, Keyword)                        \
    __ENUMERATE_SQL_TOKEN("INDEXED", Indexed, Keyword)                    \
    __ENUMERATE_SQL_TOKEN("INITIALLY", Initially, Keyword)                \
    __ENUMERATE_SQL_TOKEN("INNER", Inner, Keyword)                        \
    __ENUMERATE_SQL_TOKEN("INSERT", Insert, Keyword)                      \
    __ENUMERATE_SQL_TOKEN("INSTEAD", Instead, Keyword)                    \
    __ENUMERATE_SQL_TOKEN("INTERSECT", Intersect, Keyword)                \
    __ENUMERATE_SQL_TOKEN("INTO", Into, Keyword)                          \
    __ENUMERATE_SQL_TOKEN("IS", Is, Keyword)                              \
    __ENUMERATE_SQL_TOKEN("ISNULL", Isnull, Keyword)                      \
    __ENUMERATE_SQL_TOKEN("JOIN", Join, Keyword)                          \
    __ENUMERATE_SQL_TOKEN("KEY", Key, Keyword)                            \
    __ENUMERATE_SQL_TOKEN("LAST", Last, Keyword)                          \
    __ENUMERATE_SQL_TOKEN("LEFT", Left, Keyword)                          \
    __ENUMERATE_SQL_TOKEN("LIKE", Like, Keyword)                          \
    __ENUMERATE_SQL_TOKEN("LIMIT", Limit, Keyword)                        \
    __ENUMERATE_SQL_TOKEN("MATCH", Match, Keyword)                        \
    __ENUMERATE_SQL_TOKEN("MATERIALIZED", Materialized, Keyword)          \
    __ENUMERATE_SQL_TOKEN("NATURAL", Natural, Keyword)                    \
    __ENUMERATE_SQL_TOKEN("NO", No, Keyword)                              \
    __ENUMERATE_SQL_TOKEN("NOT", Not, Keyword)                            \
    __ENUMERATE_SQL_TOKEN("NOTHING", Nothing, Keyword)                    \
    __ENUMERATE_SQL_TOKEN("NOTNULL", Notnull, Keyword)                    \
    __ENUMERATE_SQL_TOKEN("NULL", Null, Keyword)                          \
    __ENUMERATE_SQL_TOKEN("NULLS", Nulls, Keyword)                        \
    __ENUMERATE_SQL_TOKEN("OF", Of, Keyword)                              \
    __ENUMERATE_SQL_TOKEN("OFFSET", Offset, Keyword)                      \
    __ENUMERATE_SQL_TOKEN("ON", On, Keyword)                              \
    __ENUMERATE_SQL_TOKEN("OR", Or, Keyword)                              \
    __ENUMERATE_SQL_TOKEN("ORDER", Order, Keyword)                        \
    __ENUMERATE_SQL_TOKEN("OTHERS", Others, Keyword)                      \
    __ENUMERATE_SQL_TOKEN("OUTER", Outer, Keyword)                        \
    __ENUMERATE_SQL_TOKEN("OVER", Over, Keyword)                          \
    __ENUMERATE_SQL_TOKEN("PARTITION", Partition, Keyword)                \
    __ENUMERATE_SQL_TOKEN("PLAN", Plan, Keyword)                          \
    __ENUMERATE_SQL_TOKEN("PRAGMA", Pragma, Keyword)                      \
    __ENUMERATE_SQL_TOKEN("PRECEDING", Preceding, Keyword)                \
    __ENUMERATE_SQL_TOKEN("PRIMARY", Primary, Keyword)                    \
    __ENUMERATE_SQL_TOKEN("QUERY", Query, Keyword)                        \
    __ENUMERATE_SQL_TOKEN("RAISE", Raise, Keyword)                        \
    __ENUMERATE_SQL_TOKEN("RANGE", Range, Keyword)                        \
    __ENUMERATE_SQL_TOKEN("RECURSIVE", Recursive, Keyword)                \
    __ENUMERATE_SQL_TOKEN("REFERENCES", References, Keyword)              \
    __ENUMERATE_SQL_TOKEN("REGEXP", Regexp, Keyword)                      \
    __ENUMERATE_SQL_TOKEN("REINDEX", Reindex, Keyword)                    \
    __ENUMERATE_SQL_TOKEN("RELEASE", Release, Keyword)                    \
    __ENUMERATE_SQL_TOKEN("RENAME", Rename, Keyword)                      \
    __ENUMERATE_SQL_TOKEN("REPLACE", Replace, Keyword)                    \
    __ENUMERATE_SQL_TOKEN("RESTRICT", Restrict, Keyword)                  \
    __ENUMERATE_SQL_TOKEN("RETURNING", Returning, Keyword)                \
    __ENUMERATE_SQL_TOKEN("RIGHT", Right, Keyword)                        \
    __ENUMERATE_SQL_TOKEN("ROLLBACK", Rollback, Keyword)                  \
    __ENUMERATE_SQL_TOKEN("ROW", Row, Keyword)                            \
    __ENUMERATE_SQL_TOKEN("ROWS", Rows, Keyword)                          \
    __ENUMERATE_SQL_TOKEN("SAVEPOINT", Savepoint, Keyword)                \
    __ENUMERATE_SQL_TOKEN("SCHEMA", Schema, Keyword)                      \
    __ENUMERATE_SQL_TOKEN("SELECT", Select, Keyword)                      \
    __ENUMERATE_SQL_TOKEN("SET", Set, Keyword)                            \
    __ENUMERATE_SQL_TOKEN("TABLE", Table, Keyword)                        \
    __ENUMERATE_SQL_TOKEN("TEMP", Temp, Keyword)                          \
    __ENUMERATE_SQL_TOKEN("TEMPORARY", Temporary, Keyword)                \
    __ENUMERATE_SQL_TOKEN("THEN", Then, Keyword)                          \
    __ENUMERATE_SQL_TOKEN("TIES", Ties, Keyword)                          \
    __ENUMERATE_SQL_TOKEN("TO", To, Keyword)                              \
    __ENUMERATE_SQL_TOKEN("TRANSACTION", Transaction, Keyword)            \
    __ENUMERATE_SQL_TOKEN("TRIGGER", Trigger, Keyword)                    \
    __ENUMERATE_SQL_TOKEN("TRUE", True, Keyword)                          \
    __ENUMERATE_SQL_TOKEN("UNBOUNDED", Unbounded, Keyword)                \
    __ENUMERATE_SQL_TOKEN("UNION", Union, Keyword)                        \
    __ENUMERATE_SQL_TOKEN("UNIQUE", Unique, Keyword)                      \
    __ENUMERATE_SQL_TOKEN("UPDATE", Update, Keyword)                      \
    __ENUMERATE_SQL_TOKEN("USING", Using, Keyword)                        \
    __ENUMERATE_SQL_TOKEN("VACUUM", Vacuum, Keyword)                      \
    __ENUMERATE_SQL_TOKEN("VALUES", Values, Keyword)                      \
    __ENUMERATE_SQL_TOKEN("VIEW", View, Keyword)                          \
    __ENUMERATE_SQL_TOKEN("VIRTUAL", Virtual, Keyword)                    \
    __ENUMERATE_SQL_TOKEN("WHEN", When, Keyword)                          \
    __ENUMERATE_SQL_TOKEN("WHERE", Where, Keyword)                        \
    __ENUMERATE_SQL_TOKEN("WINDOW", Window, Keyword)                      \
    __ENUMERATE_SQL_TOKEN("WITH", With, Keyword)                          \
    __ENUMERATE_SQL_TOKEN("WITHOUT", Without, Keyword)                    \
    __ENUMERATE_SQL_TOKEN("_identifier_", Identifier, Identifier)         \
    __ENUMERATE_SQL_TOKEN("_numeric_", NumericLiteral, Number)            \
    __ENUMERATE_SQL_TOKEN("_string_", StringLiteral, String)              \
    __ENUMERATE_SQL_TOKEN("_blob_", BlobLiteral, Blob)                    \
    __ENUMERATE_SQL_TOKEN("_eof_", Eof, Invalid)                          \
    __ENUMERATE_SQL_TOKEN("_invalid_", Invalid, Invalid)                  \
    __ENUMERATE_SQL_TOKEN("?", Placeholder, Operator)                     \
    __ENUMERATE_SQL_TOKEN("&", Ampersand, Operator)                       \
    __ENUMERATE_SQL_TOKEN("*", Asterisk, Operator)                        \
    __ENUMERATE_SQL_TOKEN(",", Comma, Punctuation)                        \
    __ENUMERATE_SQL_TOKEN("/", Divide, Operator)                          \
    __ENUMERATE_SQL_TOKEN("||", DoublePipe, Operator)                     \
    __ENUMERATE_SQL_TOKEN("=", Equals, Operator)                          \
    __ENUMERATE_SQL_TOKEN("==", EqualsEquals, Operator)                   \
    __ENUMERATE_SQL_TOKEN(">", GreaterThan, Operator)                     \
    __ENUMERATE_SQL_TOKEN(">=", GreaterThanEquals, Operator)              \
    __ENUMERATE_SQL_TOKEN("<", LessThan, Operator)                        \
    __ENUMERATE_SQL_TOKEN("<=", LessThanEquals, Operator)                 \
    __ENUMERATE_SQL_TOKEN("-", Minus, Operator)                           \
    __ENUMERATE_SQL_TOKEN("%", Modulus, Operator)                         \
    __ENUMERATE_SQL_TOKEN("!=", NotEquals1, Operator)                     \
    __ENUMERATE_SQL_TOKEN("<>", NotEquals2, Operator)                     \
    __ENUMERATE_SQL_TOKEN(")", ParenClose, Punctuation)                   \
    __ENUMERATE_SQL_TOKEN("(", ParenOpen, Punctuation)                    \
    __ENUMERATE_SQL_TOKEN(".", Period, Operator)                          \
    __ENUMERATE_SQL_TOKEN("|", Pipe, Operator)                            \
    __ENUMERATE_SQL_TOKEN("+", Plus, Operator)                            \
    __ENUMERATE_SQL_TOKEN(";", SemiColon, Punctuation)                    \
    __ENUMERATE_SQL_TOKEN("<<", ShiftLeft, Operator)                      \
    __ENUMERATE_SQL_TOKEN(">>", ShiftRight, Operator)                     \
    __ENUMERATE_SQL_TOKEN("~", Tilde, Operator)

enum class TokenType {
#define __ENUMERATE_SQL_TOKEN(value, type, category) type,
    ENUMERATE_SQL_TOKENS
#undef __ENUMERATE_SQL_TOKEN
        _COUNT_OF_TOKENS,
};

enum class TokenCategory {
    Invalid,
    Keyword,
    Identifier,
    Number,
    String,
    Blob,
    Operator,
    Punctuation,
};

struct SourcePosition {
    size_t line { 0 };
    size_t column { 0 };
};

class Token {
public:
    Token(TokenType type, ByteString value, SourcePosition start_position, SourcePosition end_position)
        : m_type(type)
        , m_value(move(value))
        , m_start_position(start_position)
        , m_end_position(end_position)
    {
    }

    static StringView name(TokenType);
    static TokenCategory category(TokenType);

    StringView name() const { return name(m_type); }
    TokenType type() const { return m_type; }
    TokenCategory category() const { return category(m_type); }

    ByteString const& value() const { return m_value; }
    double double_value() const;

    SourcePosition const& start_position() const { return m_start_position; }
    SourcePosition const& end_position() const { return m_end_position; }

private:
    TokenType m_type;
    ByteString m_value;
    SourcePosition m_start_position;
    SourcePosition m_end_position;
};

}

Coverage Report

Created: 2026-07-25 07:46

Line	Count	Source
1		/*
2		* Copyright (c) 2021, Tim Flynn <trflynn89@serenityos.org>
3		* Copyright (c) 2021, Jan de Visser <jan@de-visser.net>
4		* Copyright (c) 2021, Mahmoud Mandour <ma.mandourr@gmail.com>
5		*
6		* SPDX-License-Identifier: BSD-2-Clause
7		*/
8
9		#pragma once
10
11		#include <AK/ByteString.h>
12		#include <AK/HashMap.h>
13		#include <AK/StringView.h>
14
15		namespace SQL::AST {
16
17		// https://sqlite.org/lang_keywords.html
18		#define ENUMERATE_SQL_TOKENS \
19	211	__ENUMERATE_SQL_TOKEN("ABORT", Abort, Keyword) \
20	681	__ENUMERATE_SQL_TOKEN("ACTION", Action, Keyword) \
21	1.29k	__ENUMERATE_SQL_TOKEN("ADD", Add, Keyword) \
22	1.81k	__ENUMERATE_SQL_TOKEN("AFTER", After, Keyword) \
23	1.80k	__ENUMERATE_SQL_TOKEN("ALL", All, Keyword) \
24	50	__ENUMERATE_SQL_TOKEN("ALTER", Alter, Keyword) \
25	641	__ENUMERATE_SQL_TOKEN("ALWAYS", Always, Keyword) \
26	638	__ENUMERATE_SQL_TOKEN("ANALYZE", Analyze, Keyword) \
27	5.64k	__ENUMERATE_SQL_TOKEN("AND", And, Keyword) \
28	340k	__ENUMERATE_SQL_TOKEN("AS", As, Keyword) \
29	340k	__ENUMERATE_SQL_TOKEN("ASC", Asc, Keyword) \
30	1.15k	__ENUMERATE_SQL_TOKEN("ATTACH", Attach, Keyword) \
31	1.14k	__ENUMERATE_SQL_TOKEN("AUTOINCREMENT", Autoincrement, Keyword) \
32	81	__ENUMERATE_SQL_TOKEN("BEFORE", Before, Keyword) \
33	101	__ENUMERATE_SQL_TOKEN("BEGIN", Begin, Keyword) \
34	11.5k	__ENUMERATE_SQL_TOKEN("BETWEEN", Between, Keyword) \
35	11.5k	__ENUMERATE_SQL_TOKEN("BY", By, Keyword) \
36	1.31k	__ENUMERATE_SQL_TOKEN("CASCADE", Cascade, Keyword) \
37	70.1k	__ENUMERATE_SQL_TOKEN("CASE", Case, Keyword) \
38	70.1k	__ENUMERATE_SQL_TOKEN("CAST", Cast, Keyword) \
39	28.7k	__ENUMERATE_SQL_TOKEN("CHECK", Check, Keyword) \
40	302	__ENUMERATE_SQL_TOKEN("COLLATE", Collate, Keyword) \
41	299	__ENUMERATE_SQL_TOKEN("COLUMN", Column, Keyword) \
42	254	__ENUMERATE_SQL_TOKEN("COMMIT", Commit, Keyword) \
43	251	__ENUMERATE_SQL_TOKEN("CONFLICT", Conflict, Keyword) \
44	637	__ENUMERATE_SQL_TOKEN("CONSTRAINT", Constraint, Keyword) \
45	634	__ENUMERATE_SQL_TOKEN("CREATE", Create, Keyword) \
46	157	__ENUMERATE_SQL_TOKEN("CROSS", Cross, Keyword) \
47	264	__ENUMERATE_SQL_TOKEN("CURRENT", Current, Keyword) \
48	1.43k	__ENUMERATE_SQL_TOKEN("CURRENT_DATE", CurrentDate, Keyword) \
49	1.43k	__ENUMERATE_SQL_TOKEN("CURRENT_TIME", CurrentTime, Keyword) \
50	623	__ENUMERATE_SQL_TOKEN("CURRENT_TIMESTAMP", CurrentTimestamp, Keyword) \
51	3	__ENUMERATE_SQL_TOKEN("DATABASE", Database, Keyword) \
52	345	__ENUMERATE_SQL_TOKEN("DEFAULT", Default, Keyword) \
53	342	__ENUMERATE_SQL_TOKEN("DEFERRABLE", Deferrable, Keyword) \
54	947	__ENUMERATE_SQL_TOKEN("DEFERRED", Deferred, Keyword) \
55	1.53k	__ENUMERATE_SQL_TOKEN("DELETE", Delete, Keyword) \
56	1.53k	__ENUMERATE_SQL_TOKEN("DESC", Desc, Keyword) \
57	22	__ENUMERATE_SQL_TOKEN("DESCRIBE", Describe, Keyword) \
58	389	__ENUMERATE_SQL_TOKEN("DETACH", Detach, Keyword) \
59	415	__ENUMERATE_SQL_TOKEN("DISTINCT", Distinct, Keyword) \
60	412	__ENUMERATE_SQL_TOKEN("DO", Do, Keyword) \
61	112	__ENUMERATE_SQL_TOKEN("DROP", Drop, Keyword) \
62	112	__ENUMERATE_SQL_TOKEN("EACH", Each, Keyword) \
63	3.42k	__ENUMERATE_SQL_TOKEN("ELSE", Else, Keyword) \
64	347k	__ENUMERATE_SQL_TOKEN("END", End, Keyword) \
65	347k	__ENUMERATE_SQL_TOKEN("ESCAPE", Escape, Keyword) \
66	1.96k	__ENUMERATE_SQL_TOKEN("EXCEPT", Except, Keyword) \
67	79	__ENUMERATE_SQL_TOKEN("EXCLUDE", Exclude, Keyword) \
68	705	__ENUMERATE_SQL_TOKEN("EXCLUSIVE", Exclusive, Keyword) \
69	5.04k	__ENUMERATE_SQL_TOKEN("EXISTS", Exists, Keyword) \
70	5.04k	__ENUMERATE_SQL_TOKEN("EXPLAIN", Explain, Keyword) \
71	487	__ENUMERATE_SQL_TOKEN("FAIL", Fail, Keyword) \
72	484	__ENUMERATE_SQL_TOKEN("FALSE", False, Keyword) \
73	127	__ENUMERATE_SQL_TOKEN("FILTER", Filter, Keyword) \
74	371	__ENUMERATE_SQL_TOKEN("FIRST", First, Keyword) \
75	368	__ENUMERATE_SQL_TOKEN("FOLLOWING", Following, Keyword) \
76	285	__ENUMERATE_SQL_TOKEN("FOR", For, Keyword) \
77	282	__ENUMERATE_SQL_TOKEN("FOREIGN", Foreign, Keyword) \
78	1.01k	__ENUMERATE_SQL_TOKEN("FROM", From, Keyword) \
79	1.00k	__ENUMERATE_SQL_TOKEN("FULL", Full, Keyword) \
80	392	__ENUMERATE_SQL_TOKEN("GENERATED", Generated, Keyword) \
81	602	__ENUMERATE_SQL_TOKEN("GLOB", Glob, Keyword) \
82	599	__ENUMERATE_SQL_TOKEN("GROUP", Group, Keyword) \
83	587	__ENUMERATE_SQL_TOKEN("GROUPS", Groups, Keyword) \
84	584	__ENUMERATE_SQL_TOKEN("HAVING", Having, Keyword) \
85	345	__ENUMERATE_SQL_TOKEN("IF", If, Keyword) \
86	1.00k	__ENUMERATE_SQL_TOKEN("IGNORE", Ignore, Keyword) \
87	997	__ENUMERATE_SQL_TOKEN("IMMEDIATE", Immediate, Keyword) \
88	23.7k	__ENUMERATE_SQL_TOKEN("IN", In, Keyword) \
89	23.7k	__ENUMERATE_SQL_TOKEN("INDEX", Index, Keyword) \
90	1.93k	__ENUMERATE_SQL_TOKEN("INDEXED", Indexed, Keyword) \
91	1.93k	__ENUMERATE_SQL_TOKEN("INITIALLY", Initially, Keyword) \
92	52	__ENUMERATE_SQL_TOKEN("INNER", Inner, Keyword) \
93	1.87k	__ENUMERATE_SQL_TOKEN("INSERT", Insert, Keyword) \
94	1.87k	__ENUMERATE_SQL_TOKEN("INSTEAD", Instead, Keyword) \
95	524	__ENUMERATE_SQL_TOKEN("INTERSECT", Intersect, Keyword) \
96	2.69k	__ENUMERATE_SQL_TOKEN("INTO", Into, Keyword) \
97	3.60k	__ENUMERATE_SQL_TOKEN("IS", Is, Keyword) \
98	3.60k	__ENUMERATE_SQL_TOKEN("ISNULL", Isnull, Keyword) \
99	8	__ENUMERATE_SQL_TOKEN("JOIN", Join, Keyword) \
100	115	__ENUMERATE_SQL_TOKEN("KEY", Key, Keyword) \
101	112	__ENUMERATE_SQL_TOKEN("LAST", Last, Keyword) \
102	17	__ENUMERATE_SQL_TOKEN("LEFT", Left, Keyword) \
103	652	__ENUMERATE_SQL_TOKEN("LIKE", Like, Keyword) \
104	1.14k	__ENUMERATE_SQL_TOKEN("LIMIT", Limit, Keyword) \
105	1.13k	__ENUMERATE_SQL_TOKEN("MATCH", Match, Keyword) \
106	911	__ENUMERATE_SQL_TOKEN("MATERIALIZED", Materialized, Keyword) \
107	89	__ENUMERATE_SQL_TOKEN("NATURAL", Natural, Keyword) \
108	2.85k	__ENUMERATE_SQL_TOKEN("NO", No, Keyword) \
109	2.94k	__ENUMERATE_SQL_TOKEN("NOT", Not, Keyword) \
110	2.93k	__ENUMERATE_SQL_TOKEN("NOTHING", Nothing, Keyword) \
111	2.03k	__ENUMERATE_SQL_TOKEN("NOTNULL", Notnull, Keyword) \
112	2.02k	__ENUMERATE_SQL_TOKEN("NULL", Null, Keyword) \
113	1.59k	__ENUMERATE_SQL_TOKEN("NULLS", Nulls, Keyword) \
114	959	__ENUMERATE_SQL_TOKEN("OF", Of, Keyword) \
115	269	__ENUMERATE_SQL_TOKEN("OFFSET", Offset, Keyword) \
116	1.39k	__ENUMERATE_SQL_TOKEN("ON", On, Keyword) \
117	1.54k	__ENUMERATE_SQL_TOKEN("OR", Or, Keyword) \
118	3.39k	__ENUMERATE_SQL_TOKEN("ORDER", Order, Keyword) \
119	3.39k	__ENUMERATE_SQL_TOKEN("OTHERS", Others, Keyword) \
120	1.80k	__ENUMERATE_SQL_TOKEN("OUTER", Outer, Keyword) \
121	1.80k	__ENUMERATE_SQL_TOKEN("OVER", Over, Keyword) \
122	161	__ENUMERATE_SQL_TOKEN("PARTITION", Partition, Keyword) \
123	158	__ENUMERATE_SQL_TOKEN("PLAN", Plan, Keyword) \
124	64	__ENUMERATE_SQL_TOKEN("PRAGMA", Pragma, Keyword) \
125	5	__ENUMERATE_SQL_TOKEN("PRECEDING", Preceding, Keyword) \
126	278	__ENUMERATE_SQL_TOKEN("PRIMARY", Primary, Keyword) \
127	275	__ENUMERATE_SQL_TOKEN("QUERY", Query, Keyword) \
128	672	__ENUMERATE_SQL_TOKEN("RAISE", Raise, Keyword) \
129	787	__ENUMERATE_SQL_TOKEN("RANGE", Range, Keyword) \
130	784	__ENUMERATE_SQL_TOKEN("RECURSIVE", Recursive, Keyword) \
131	15	__ENUMERATE_SQL_TOKEN("REFERENCES", References, Keyword) \
132	32	__ENUMERATE_SQL_TOKEN("REGEXP", Regexp, Keyword) \
133	29	__ENUMERATE_SQL_TOKEN("REINDEX", Reindex, Keyword) \
134	9	__ENUMERATE_SQL_TOKEN("RELEASE", Release, Keyword) \
135	109	__ENUMERATE_SQL_TOKEN("RENAME", Rename, Keyword) \
136	172	__ENUMERATE_SQL_TOKEN("REPLACE", Replace, Keyword) \
137	354	__ENUMERATE_SQL_TOKEN("RESTRICT", Restrict, Keyword) \
138	351	__ENUMERATE_SQL_TOKEN("RETURNING", Returning, Keyword) \
139	138	__ENUMERATE_SQL_TOKEN("RIGHT", Right, Keyword) \
140	12	__ENUMERATE_SQL_TOKEN("ROLLBACK", Rollback, Keyword) \
141	37	__ENUMERATE_SQL_TOKEN("ROW", Row, Keyword) \
142	106	__ENUMERATE_SQL_TOKEN("ROWS", Rows, Keyword) \
143	103	__ENUMERATE_SQL_TOKEN("SAVEPOINT", Savepoint, Keyword) \
144	39	__ENUMERATE_SQL_TOKEN("SCHEMA", Schema, Keyword) \
145	52.6k	__ENUMERATE_SQL_TOKEN("SELECT", Select, Keyword) \
146	52.6k	__ENUMERATE_SQL_TOKEN("SET", Set, Keyword) \
147	3.31k	__ENUMERATE_SQL_TOKEN("TABLE", Table, Keyword) \
148	351	__ENUMERATE_SQL_TOKEN("TEMP", Temp, Keyword) \
149	6	__ENUMERATE_SQL_TOKEN("TEMPORARY", Temporary, Keyword) \
150	656k	__ENUMERATE_SQL_TOKEN("THEN", Then, Keyword) \
151	656k	__ENUMERATE_SQL_TOKEN("TIES", Ties, Keyword) \
152	909	__ENUMERATE_SQL_TOKEN("TO", To, Keyword) \
153	906	__ENUMERATE_SQL_TOKEN("TRANSACTION", Transaction, Keyword) \
154	231	__ENUMERATE_SQL_TOKEN("TRIGGER", Trigger, Keyword) \
155	18	__ENUMERATE_SQL_TOKEN("TRUE", True, Keyword) \
156	799	__ENUMERATE_SQL_TOKEN("UNBOUNDED", Unbounded, Keyword) \
157	796	__ENUMERATE_SQL_TOKEN("UNION", Union, Keyword) \
158	86	__ENUMERATE_SQL_TOKEN("UNIQUE", Unique, Keyword) \
159	1.37k	__ENUMERATE_SQL_TOKEN("UPDATE", Update, Keyword) \
160	1.37k	__ENUMERATE_SQL_TOKEN("USING", Using, Keyword) \
161	454	__ENUMERATE_SQL_TOKEN("VACUUM", Vacuum, Keyword) \
162	1.87k	__ENUMERATE_SQL_TOKEN("VALUES", Values, Keyword) \
163	1.87k	__ENUMERATE_SQL_TOKEN("VIEW", View, Keyword) \
164	1.23k	__ENUMERATE_SQL_TOKEN("VIRTUAL", Virtual, Keyword) \
165	347k	__ENUMERATE_SQL_TOKEN("WHEN", When, Keyword) \
166	347k	__ENUMERATE_SQL_TOKEN("WHERE", Where, Keyword) \
167	818	__ENUMERATE_SQL_TOKEN("WINDOW", Window, Keyword) \
168	318	__ENUMERATE_SQL_TOKEN("WITH", With, Keyword) \
169	315	__ENUMERATE_SQL_TOKEN("WITHOUT", Without, Keyword) \
170	787k	__ENUMERATE_SQL_TOKEN("_identifier_", Identifier, Identifier) \
171	787k	__ENUMERATE_SQL_TOKEN("_numeric_", NumericLiteral, Number) \
172	91.1k	__ENUMERATE_SQL_TOKEN("_string_", StringLiteral, String) \
173	91.1k	__ENUMERATE_SQL_TOKEN("_blob_", BlobLiteral, Blob) \
174	218k	__ENUMERATE_SQL_TOKEN("_eof_", Eof, Invalid) \
175	1.58M	__ENUMERATE_SQL_TOKEN("_invalid_", Invalid, Invalid) \
176	1.58M	__ENUMERATE_SQL_TOKEN("?", Placeholder, Operator) \
177	46.2k	__ENUMERATE_SQL_TOKEN("&", Ampersand, Operator) \
178	103k	__ENUMERATE_SQL_TOKEN("*", Asterisk, Operator) \
179	223k	__ENUMERATE_SQL_TOKEN(",", Comma, Punctuation) \
180	223k	__ENUMERATE_SQL_TOKEN("/", Divide, Operator) \
181	73.2k	__ENUMERATE_SQL_TOKEN("\|\|", DoublePipe, Operator) \
182	88.3k	__ENUMERATE_SQL_TOKEN("=", Equals, Operator) \
183	88.2k	__ENUMERATE_SQL_TOKEN("==", EqualsEquals, Operator) \
184	44.9k	__ENUMERATE_SQL_TOKEN(">", GreaterThan, Operator) \
185	3.29k	__ENUMERATE_SQL_TOKEN(">=", GreaterThanEquals, Operator) \
186	29.1k	__ENUMERATE_SQL_TOKEN("<", LessThan, Operator) \
187	29.1k	__ENUMERATE_SQL_TOKEN("<=", LessThanEquals, Operator) \
188	103k	__ENUMERATE_SQL_TOKEN("-", Minus, Operator) \
189	103k	__ENUMERATE_SQL_TOKEN("%", Modulus, Operator) \
190	52.1k	__ENUMERATE_SQL_TOKEN("!=", NotEquals1, Operator) \
191	2.81k	__ENUMERATE_SQL_TOKEN("<>", NotEquals2, Operator) \
192	970k	__ENUMERATE_SQL_TOKEN(")", ParenClose, Punctuation) \
193	1.11M	__ENUMERATE_SQL_TOKEN("(", ParenOpen, Punctuation) \
194	1.11M	__ENUMERATE_SQL_TOKEN(".", Period, Operator) \
195	29.9k	__ENUMERATE_SQL_TOKEN("\|", Pipe, Operator) \
196	332k	__ENUMERATE_SQL_TOKEN("+", Plus, Operator) \
197	332k	__ENUMERATE_SQL_TOKEN(";", SemiColon, Punctuation) \
198	95.3k	__ENUMERATE_SQL_TOKEN("<<", ShiftLeft, Operator) \
199	15.5k	__ENUMERATE_SQL_TOKEN(">>", ShiftRight, Operator) \
200	2.47k	__ENUMERATE_SQL_TOKEN("~", Tilde, Operator)
201
202		enum class TokenType {
203		#define __ENUMERATE_SQL_TOKEN(value, type, category) type,
204		ENUMERATE_SQL_TOKENS
205		#undef __ENUMERATE_SQL_TOKEN
206		_COUNT_OF_TOKENS,
207		};
208
209		enum class TokenCategory {
210		Invalid,
211		Keyword,
212		Identifier,
213		Number,
214		String,
215		Blob,
216		Operator,
217		Punctuation,
218		};
219
220		struct SourcePosition {
221		size_t line { 0 };
222		size_t column { 0 };
223		};
224
225		class Token {
226		public:
227		Token(TokenType type, ByteString value, SourcePosition start_position, SourcePosition end_position)
228	155M	: m_type(type)
229	155M	, m_value(move(value))
230	155M	, m_start_position(start_position)
231	155M	, m_end_position(end_position)
232	155M	{
233	155M	}
234
235		static StringView name(TokenType);
236		static TokenCategory category(TokenType);
237
238	4.87M	StringView name() const { return name(m_type); }
239	1.73G	TokenType type() const { return m_type; }
240	0	TokenCategory category() const { return category(m_type); }
241
242	72.0M	ByteString const& value() const { return m_value; }
243		double double_value() const;
244
245	4.92M	SourcePosition const& start_position() const { return m_start_position; }
246	0	SourcePosition const& end_position() const { return m_end_position; }
247
248		private:
249		TokenType m_type;
250		ByteString m_value;
251		SourcePosition m_start_position;
252		SourcePosition m_end_position;
253		};
254
255		}