/src/cpython/Parser/token.c

Source
/* Auto-generated by Tools/build/generate_token.py */

#include "Python.h"
#include "pycore_token.h"

/* Token names */

const char * const _PyParser_TokenNames[] = {
    "ENDMARKER",
    "NAME",
    "NUMBER",
    "STRING",
    "NEWLINE",
    "INDENT",
    "DEDENT",
    "LPAR",
    "RPAR",
    "LSQB",
    "RSQB",
    "COLON",
    "COMMA",
    "SEMI",
    "PLUS",
    "MINUS",
    "STAR",
    "SLASH",
    "VBAR",
    "AMPER",
    "LESS",
    "GREATER",
    "EQUAL",
    "DOT",
    "PERCENT",
    "LBRACE",
    "RBRACE",
    "EQEQUAL",
    "NOTEQUAL",
    "LESSEQUAL",
    "GREATEREQUAL",
    "TILDE",
    "CIRCUMFLEX",
    "LEFTSHIFT",
    "RIGHTSHIFT",
    "DOUBLESTAR",
    "PLUSEQUAL",
    "MINEQUAL",
    "STAREQUAL",
    "SLASHEQUAL",
    "PERCENTEQUAL",
    "AMPEREQUAL",
    "VBAREQUAL",
    "CIRCUMFLEXEQUAL",
    "LEFTSHIFTEQUAL",
    "RIGHTSHIFTEQUAL",
    "DOUBLESTAREQUAL",
    "DOUBLESLASH",
    "DOUBLESLASHEQUAL",
    "AT",
    "ATEQUAL",
    "RARROW",
    "ELLIPSIS",
    "COLONEQUAL",
    "EXCLAMATION",
    "OP",
    "TYPE_IGNORE",
    "TYPE_COMMENT",
    "SOFT_KEYWORD",
    "FSTRING_START",
    "FSTRING_MIDDLE",
    "FSTRING_END",
    "TSTRING_START",
    "TSTRING_MIDDLE",
    "TSTRING_END",
    "COMMENT",
    "NL",
    "<ERRORTOKEN>",
    "<ENCODING>",
    "<N_TOKENS>",
};

/* Return the token corresponding to a single character */

int
_PyToken_OneChar(int c1)
{
    switch (c1) {
    case '!': return EXCLAMATION;
    case '%': return PERCENT;
    case '&': return AMPER;
    case '(': return LPAR;
    case ')': return RPAR;
    case '*': return STAR;
    case '+': return PLUS;
    case ',': return COMMA;
    case '-': return MINUS;
    case '.': return DOT;
    case '/': return SLASH;
    case ':': return COLON;
    case ';': return SEMI;
    case '<': return LESS;
    case '=': return EQUAL;
    case '>': return GREATER;
    case '@': return AT;
    case '[': return LSQB;
    case ']': return RSQB;
    case '^': return CIRCUMFLEX;
    case '{': return LBRACE;
    case '|': return VBAR;
    case '}': return RBRACE;
    case '~': return TILDE;
    }
    return OP;
}

int
_PyToken_TwoChars(int c1, int c2, int BARRY_AS_BDFL)
{
    switch (c1) {
    case '!':
        switch (c2) {
        case '=': return NOTEQUAL;
        }
        break;
    case '%':
        switch (c2) {
        case '=': return PERCENTEQUAL;
        }
        break;
    case '&':
        switch (c2) {
        case '=': return AMPEREQUAL;
        }
        break;
    case '*':
        switch (c2) {
        case '*': return DOUBLESTAR;
        case '=': return STAREQUAL;
        }
        break;
    case '+':
        switch (c2) {
        case '=': return PLUSEQUAL;
        }
        break;
    case '-':
        switch (c2) {
        case '=': return MINEQUAL;
        case '>': return RARROW;
        }
        break;
    case '/':
        switch (c2) {
        case '/': return DOUBLESLASH;
        case '=': return SLASHEQUAL;
        }
        break;
    case ':':
        switch (c2) {
        case '=': return COLONEQUAL;
        }
        break;
    case '<':
        switch (c2) {
        case '<': return LEFTSHIFT;
        case '=': return LESSEQUAL;
        case '>': return BARRY_AS_BDFL ? NOTEQUAL : OP;
        }
        break;
    case '=':
        switch (c2) {
        case '=': return EQEQUAL;
        }
        break;
    case '>':
        switch (c2) {
        case '=': return GREATEREQUAL;
        case '>': return RIGHTSHIFT;
        }
        break;
    case '@':
        switch (c2) {
        case '=': return ATEQUAL;
        }
        break;
    case '^':
        switch (c2) {
        case '=': return CIRCUMFLEXEQUAL;
        }
        break;
    case '|':
        switch (c2) {
        case '=': return VBAREQUAL;
        }
        break;
    }
    return OP;
}

int
_PyToken_ThreeChars(int c1, int c2, int c3)
{
    switch (c1) {
    case '*':
        switch (c2) {
        case '*':
            switch (c3) {
            case '=': return DOUBLESTAREQUAL;
            }
            break;
        }
        break;
    case '.':
        switch (c2) {
        case '.':
            switch (c3) {
            case '.': return ELLIPSIS;
            }
            break;
        }
        break;
    case '/':
        switch (c2) {
        case '/':
            switch (c3) {
            case '=': return DOUBLESLASHEQUAL;
            }
            break;
        }
        break;
    case '<':
        switch (c2) {
        case '<':
            switch (c3) {
            case '=': return LEFTSHIFTEQUAL;
            }
            break;
        }
        break;
    case '>':
        switch (c2) {
        case '>':
            switch (c3) {
            case '=': return RIGHTSHIFTEQUAL;
            }
            break;
        }
        break;
    }
    return OP;
}

Coverage Report

Created: 2026-10-17 06:15

Line	Count	Source
1		/* Auto-generated by Tools/build/generate_token.py */
2
3		#include "Python.h"
4		#include "pycore_token.h"
5
6		/* Token names */
7
8		const char * const _PyParser_TokenNames[] = {
9		"ENDMARKER",
10		"NAME",
11		"NUMBER",
12		"STRING",
13		"NEWLINE",
14		"INDENT",
15		"DEDENT",
16		"LPAR",
17		"RPAR",
18		"LSQB",
19		"RSQB",
20		"COLON",
21		"COMMA",
22		"SEMI",
23		"PLUS",
24		"MINUS",
25		"STAR",
26		"SLASH",
27		"VBAR",
28		"AMPER",
29		"LESS",
30		"GREATER",
31		"EQUAL",
32		"DOT",
33		"PERCENT",
34		"LBRACE",
35		"RBRACE",
36		"EQEQUAL",
37		"NOTEQUAL",
38		"LESSEQUAL",
39		"GREATEREQUAL",
40		"TILDE",
41		"CIRCUMFLEX",
42		"LEFTSHIFT",
43		"RIGHTSHIFT",
44		"DOUBLESTAR",
45		"PLUSEQUAL",
46		"MINEQUAL",
47		"STAREQUAL",
48		"SLASHEQUAL",
49		"PERCENTEQUAL",
50		"AMPEREQUAL",
51		"VBAREQUAL",
52		"CIRCUMFLEXEQUAL",
53		"LEFTSHIFTEQUAL",
54		"RIGHTSHIFTEQUAL",
55		"DOUBLESTAREQUAL",
56		"DOUBLESLASH",
57		"DOUBLESLASHEQUAL",
58		"AT",
59		"ATEQUAL",
60		"RARROW",
61		"ELLIPSIS",
62		"COLONEQUAL",
63		"EXCLAMATION",
64		"OP",
65		"TYPE_IGNORE",
66		"TYPE_COMMENT",
67		"SOFT_KEYWORD",
68		"FSTRING_START",
69		"FSTRING_MIDDLE",
70		"FSTRING_END",
71		"TSTRING_START",
72		"TSTRING_MIDDLE",
73		"TSTRING_END",
74		"COMMENT",
75		"NL",
76		"<ERRORTOKEN>",
77		"<ENCODING>",
78		"<N_TOKENS>",
79		};
80
81		/* Return the token corresponding to a single character */
82
83		int
84		_PyToken_OneChar(int c1)
85	819k	{
86	819k	switch (c1) {
87	2.22k	case '!': return EXCLAMATION;
88	2.94k	case '%': return PERCENT;
89	2.71k	case '&': return AMPER;
90	67.7k	case '(': return LPAR;
91	38.7k	case ')': return RPAR;
92	27.8k	case '*': return STAR;
93	42.6k	case '+': return PLUS;
94	123k	case ',': return COMMA;
95	59.3k	case '-': return MINUS;
96	0	case '.': return DOT;
97	4.16k	case '/': return SLASH;
98	101k	case ':': return COLON;
99	24.4k	case ';': return SEMI;
100	47.1k	case '<': return LESS;
101	34.7k	case '=': return EQUAL;
102	47.9k	case '>': return GREATER;
103	4.66k	case '@': return AT;
104	34.2k	case '[': return LSQB;
105	5.72k	case ']': return RSQB;
106	1.45k	case '^': return CIRCUMFLEX;
107	51.0k	case '{': return LBRACE;
108	8.20k	case '\|': return VBAR;
109	27.7k	case '}': return RBRACE;
110	57.7k	case '~': return TILDE;
111	819k	}
112	960	return OP;
113	819k	}
114
115		int
116		_PyToken_TwoChars(int c1, int c2, int BARRY_AS_BDFL)
117	847k	{
118	847k	switch (c1) {
119	3.34k	case '!':
120	3.34k	switch (c2) {
121	1.12k	case '=': return NOTEQUAL;
122	3.34k	}
123	2.22k	break;
124	3.13k	case '%':
125	3.13k	switch (c2) {
126	187	case '=': return PERCENTEQUAL;
127	3.13k	}
128	2.94k	break;
129	3.00k	case '&':
130	3.00k	switch (c2) {
131	295	case '=': return AMPEREQUAL;
132	3.00k	}
133	2.71k	break;
134	40.8k	case '*':
135	40.8k	switch (c2) {
136	12.6k	case '*': return DOUBLESTAR;
137	341	case '=': return STAREQUAL;
138	40.8k	}
139	27.8k	break;
140	42.9k	case '+':
141	42.9k	switch (c2) {
142	265	case '=': return PLUSEQUAL;
143	42.9k	}
144	42.6k	break;
145	59.9k	case '-':
146	59.9k	switch (c2) {
147	169	case '=': return MINEQUAL;
148	427	case '>': return RARROW;
149	59.9k	}
150	59.3k	break;
151	59.3k	case '/':
152	6.05k	switch (c2) {
153	1.41k	case '/': return DOUBLESLASH;
154	471	case '=': return SLASHEQUAL;
155	6.05k	}
156	4.16k	break;
157	102k	case ':':
158	102k	switch (c2) {
159	1.28k	case '=': return COLONEQUAL;
160	102k	}
161	101k	break;
162	101k	case '<':
163	49.8k	switch (c2) {
164	1.80k	case '<': return LEFTSHIFT;
165	793	case '=': return LESSEQUAL;
166	302	case '>': return BARRY_AS_BDFL ? NOTEQUAL : OP;
167	49.8k	}
168	46.9k	break;
169	46.9k	case '=':
170	36.8k	switch (c2) {
171	2.11k	case '=': return EQEQUAL;
172	36.8k	}
173	34.7k	break;
174	50.3k	case '>':
175	50.3k	switch (c2) {
176	1.03k	case '=': return GREATEREQUAL;
177	1.36k	case '>': return RIGHTSHIFT;
178	50.3k	}
179	47.9k	break;
180	47.9k	case '@':
181	4.92k	switch (c2) {
182	252	case '=': return ATEQUAL;
183	4.92k	}
184	4.66k	break;
185	4.66k	case '^':
186	1.70k	switch (c2) {
187	255	case '=': return CIRCUMFLEXEQUAL;
188	1.70k	}
189	1.45k	break;
190	8.43k	case '\|':
191	8.43k	switch (c2) {
192	227	case '=': return VBAREQUAL;
193	8.43k	}
194	8.20k	break;
195	847k	}
196	820k	return OP;
197	847k	}
198
199		int
200		_PyToken_ThreeChars(int c1, int c2, int c3)
201	26.5k	{
202	26.5k	switch (c1) {
203	12.9k	case '*':
204	12.9k	switch (c2) {
205	12.6k	case '*':
206	12.6k	switch (c3) {
207	230	case '=': return DOUBLESTAREQUAL;
208	12.6k	}
209	12.4k	break;
210	12.9k	}
211	12.7k	break;
212	12.7k	case '.':
213	0	switch (c2) {
214	0	case '.':
215	0	switch (c3) {
216	0	case '.': return ELLIPSIS;
217	0	}
218	0	break;
219	0	}
220	0	break;
221	1.88k	case '/':
222	1.88k	switch (c2) {
223	1.41k	case '/':
224	1.41k	switch (c3) {
225	320	case '=': return DOUBLESLASHEQUAL;
226	1.41k	}
227	1.09k	break;
228	1.88k	}
229	1.56k	break;
230	2.69k	case '<':
231	2.69k	switch (c2) {
232	1.80k	case '<':
233	1.80k	switch (c3) {
234	305	case '=': return LEFTSHIFTEQUAL;
235	1.80k	}
236	1.50k	break;
237	2.69k	}
238	2.39k	break;
239	2.40k	case '>':
240	2.40k	switch (c2) {
241	1.36k	case '>':
242	1.36k	switch (c3) {
243	414	case '=': return RIGHTSHIFTEQUAL;
244	1.36k	}
245	951	break;
246	2.40k	}
247	1.98k	break;
248	26.5k	}
249	25.2k	return OP;
250	26.5k	}