/src/cpython/Parser/lexer/number.c

Source
#include "Python.h"
#include "pycore_token.h"

#include "lexer_internal.h"
#include "../tokenizer/helpers.h"

#define MAKE_TOKEN(token_type) _PyLexer_token_setup(tok, token, token_type, p_start, p_end)

static int
lookahead(struct tok_state *tok, const char *test)
{
    const char *s = test;
    int res = 0;
    while (1) {
        int c = tok_nextc(tok);
        if (*s == 0) {
            res = !is_potential_identifier_char(c);
        }
        else if (c == *s) {
            s++;
            continue;
        }

        tok_backup(tok, c);
        while (s != test) {
            tok_backup(tok, *--s);
        }
        return res;
    }
}

static int
verify_end_of_number(struct tok_state *tok, int c, const char *kind) {
    if (tok->tok_extra_tokens) {
        // When we are parsing extra tokens, we don't want to emit warnings
        // about invalid literals, because we want to be a bit more liberal.
        return 1;
    }
    /* Emit a deprecation warning only if the numeric literal is immediately
     * followed by one of keywords which can occur after a numeric literal
     * in valid code: "and", "else", "for", "if", "in", "is" and "or".
     * It allows to gradually deprecate existing valid code without adding
     * warning before error in most cases of invalid numeric literal (which
     * would be confusing and break existing tests).
     * Raise a syntax error with slightly better message than plain
     * "invalid syntax" if the numeric literal is immediately followed by
     * other keyword or identifier.
     */
    int r = 0;
    if (c == 'a') {
        r = lookahead(tok, "nd");
    }
    else if (c == 'e') {
        r = lookahead(tok, "lse");
    }
    else if (c == 'f') {
        r = lookahead(tok, "or");
    }
    else if (c == 'i') {
        int c2 = tok_nextc(tok);
        if (c2 == 'f' || c2 == 'n' || c2 == 's') {
            r = 1;
        }
        tok_backup(tok, c2);
    }
    else if (c == 'o') {
        r = lookahead(tok, "r");
    }
    else if (c == 'n') {
        r = lookahead(tok, "ot");
    }
    if (r) {
        tok_backup(tok, c);
        if (_PyTokenizer_parser_warn(tok, PyExc_SyntaxWarning,
                "invalid %s literal", kind))
        {
            return 0;
        }
        tok_nextc(tok);
    }
    else /* In future releases, only error will remain. */
    if (c < 128 && is_potential_identifier_char(c)) {
        tok_backup(tok, c);
        _PyTokenizer_syntaxerror(tok, "invalid %s literal", kind);
        return 0;
    }
    return 1;
}

static int
tok_decimal_tail(struct tok_state *tok)
{
    int c;

    while (1) {
        do {
            c = tok_nextc(tok);
        } while (Py_ISDIGIT(c));
        if (c != '_') {
            break;
        }
        c = tok_nextc(tok);
        if (!Py_ISDIGIT(c)) {
            tok_backup(tok, c);
            _PyTokenizer_syntaxerror(tok, "invalid decimal literal");
            return 0;
        }
    }
    return c;
}

int
_PyLexer_scan_number(struct tok_state *tok, struct token *token, int c,
                     int leading_dot)
{
    const char *p_start = NULL;
    const char *p_end = NULL;

    if (leading_dot) {
        goto fraction;
    }
    if (c == '0') {
        /* Hex, octal or binary -- maybe. */
        c = tok_nextc(tok);
        if (c == 'x' || c == 'X') {
            /* Hex */
            c = tok_nextc(tok);
            do {
                if (c == '_') {
                    c = tok_nextc(tok);
                }
                if (!Py_ISXDIGIT(c)) {
                    tok_backup(tok, c);
                    return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid hexadecimal literal"));
                }
                do {
                    c = tok_nextc(tok);
                } while (Py_ISXDIGIT(c));
            } while (c == '_');
            if (!verify_end_of_number(tok, c, "hexadecimal")) {
                return MAKE_TOKEN(ERRORTOKEN);
            }
        }
        else if (c == 'o' || c == 'O') {
            /* Octal */
            c = tok_nextc(tok);
            do {
                if (c == '_') {
                    c = tok_nextc(tok);
                }
                if (c < '0' || c >= '8') {
                    if (Py_ISDIGIT(c)) {
                        return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok,
                                "invalid digit '%c' in octal literal", c));
                    }
                    else {
                        tok_backup(tok, c);
                        return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid octal literal"));
                    }
                }
                do {
                    c = tok_nextc(tok);
                } while ('0' <= c && c < '8');
            } while (c == '_');
            if (Py_ISDIGIT(c)) {
                return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok,
                        "invalid digit '%c' in octal literal", c));
            }
            if (!verify_end_of_number(tok, c, "octal")) {
                return MAKE_TOKEN(ERRORTOKEN);
            }
        }
        else if (c == 'b' || c == 'B') {
            /* Binary */
            c = tok_nextc(tok);
            do {
                if (c == '_') {
                    c = tok_nextc(tok);
                }
                if (c != '0' && c != '1') {
                    if (Py_ISDIGIT(c)) {
                        return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid digit '%c' in binary literal", c));
                    }
                    else {
                        tok_backup(tok, c);
                        return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid binary literal"));
                    }
                }
                do {
                    c = tok_nextc(tok);
                } while (c == '0' || c == '1');
            } while (c == '_');
            if (Py_ISDIGIT(c)) {
                return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid digit '%c' in binary literal", c));
            }
            if (!verify_end_of_number(tok, c, "binary")) {
                return MAKE_TOKEN(ERRORTOKEN);
            }
        }
        else {
            int nonzero = 0;
            /* maybe old-style octal; c is first char of it */
            /* in any case, allow '0' as a literal */
            while (1) {
                if (c == '_') {
                    c = tok_nextc(tok);
                    if (!Py_ISDIGIT(c)) {
                        tok_backup(tok, c);
                        return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid decimal literal"));
                    }
                }
                if (c != '0') {
                    break;
                }
                c = tok_nextc(tok);
            }
            char* zeros_end = tok->cur;
            if (Py_ISDIGIT(c)) {
                nonzero = 1;
                c = tok_decimal_tail(tok);
                if (c == 0) {
                    return MAKE_TOKEN(ERRORTOKEN);
                }
            }
            if (c == '.') {
                c = tok_nextc(tok);
                goto fraction;
            }
            else if (c == 'e' || c == 'E') {
                goto exponent;
            }
            else if (c == 'j' || c == 'J') {
                goto imaginary;
            }
            else if (nonzero && !tok->tok_extra_tokens) {
                /* Old-style octal: now disallowed. */
                tok_backup(tok, c);
                return MAKE_TOKEN(_PyTokenizer_syntaxerror_known_range(
                        tok, (int)(tok->start + 1 - tok->line_start),
                        (int)(zeros_end - tok->line_start),
                        "leading zeros in decimal integer "
                        "literals are not permitted; "
                        "use an 0o prefix for octal integers"));
            }
            if (!verify_end_of_number(tok, c, "decimal")) {
                return MAKE_TOKEN(ERRORTOKEN);
            }
        }
    }
    else {
        /* Decimal */
        c = tok_decimal_tail(tok);
        if (c == 0) {
            return MAKE_TOKEN(ERRORTOKEN);
        }
        {
            /* Accept floating-point numbers. */
            if (c == '.') {
                c = tok_nextc(tok);
    fraction:
                /* Fraction */
                if (Py_ISDIGIT(c)) {
                    c = tok_decimal_tail(tok);
                    if (c == 0) {
                        return MAKE_TOKEN(ERRORTOKEN);
                    }
                }
            }
            if (c == 'e' || c == 'E') {
                int e;
              exponent:
                e = c;
                /* Exponent part */
                c = tok_nextc(tok);
                if (c == '+' || c == '-') {
                    c = tok_nextc(tok);
                    if (!Py_ISDIGIT(c)) {
                        tok_backup(tok, c);
                        return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid decimal literal"));
                    }
                } else if (!Py_ISDIGIT(c)) {
                    tok_backup(tok, c);
                    if (!verify_end_of_number(tok, e, "decimal")) {
                        return MAKE_TOKEN(ERRORTOKEN);
                    }
                    tok_backup(tok, e);
                    p_start = tok->start;
                    p_end = tok->cur;
                    return MAKE_TOKEN(NUMBER);
                }
                c = tok_decimal_tail(tok);
                if (c == 0) {
                    return MAKE_TOKEN(ERRORTOKEN);
                }
            }
            if (c == 'j' || c == 'J') {
                /* Imaginary part */
    imaginary:
                c = tok_nextc(tok);
                if (!verify_end_of_number(tok, c, "imaginary")) {
                    return MAKE_TOKEN(ERRORTOKEN);
                }
            }
            else if (!verify_end_of_number(tok, c, "decimal")) {
                return MAKE_TOKEN(ERRORTOKEN);
            }
        }
    }
    tok_backup(tok, c);
    p_start = tok->start;
    p_end = tok->cur;
    return MAKE_TOKEN(NUMBER);
}

Coverage Report

Created: 2026-07-14 06:16

Line	Count	Source
1		#include "Python.h"
2		#include "pycore_token.h"
3
4		#include "lexer_internal.h"
5		#include "../tokenizer/helpers.h"
6
7	99.0k	#define MAKE_TOKEN(token_type) _PyLexer_token_setup(tok, token, token_type, p_start, p_end)
8
9		static int
10		lookahead(struct tok_state tok, const char test)
11	8.87k	{
12	8.87k	const char *s = test;
13	8.87k	int res = 0;
14	23.1k	while (1) {
15	23.1k	int c = tok_nextc(tok);
16	23.1k	if (*s == 0) {
17	8.76k	res = !is_potential_identifier_char(c);
18	8.76k	}
19	14.4k	else if (c == *s) {
20	14.2k	s++;
21	14.2k	continue;
22	14.2k	}
23
24	8.87k	tok_backup(tok, c);
25	23.1k	while (s != test) {
26	14.2k	tok_backup(tok, *--s);
27	14.2k	}
28	8.87k	return res;
29	23.1k	}
30	8.87k	}
31
32		static int
33	98.8k	verify_end_of_number(struct tok_state tok, int c, const char kind) {
34	98.8k	if (tok->tok_extra_tokens) {
35		// When we are parsing extra tokens, we don't want to emit warnings
36		// about invalid literals, because we want to be a bit more liberal.
37	56	return 1;
38	56	}
39		/* Emit a deprecation warning only if the numeric literal is immediately
40		* followed by one of keywords which can occur after a numeric literal
41		* in valid code: "and", "else", "for", "if", "in", "is" and "or".
42		* It allows to gradually deprecate existing valid code without adding
43		* warning before error in most cases of invalid numeric literal (which
44		* would be confusing and break existing tests).
45		* Raise a syntax error with slightly better message than plain
46		* "invalid syntax" if the numeric literal is immediately followed by
47		* other keyword or identifier.
48		*/
49	98.8k	int r = 0;
50	98.8k	if (c == 'a') {
51	1.10k	r = lookahead(tok, "nd");
52	1.10k	}
53	97.7k	else if (c == 'e') {
54	737	r = lookahead(tok, "lse");
55	737	}
56	96.9k	else if (c == 'f') {
57	2.74k	r = lookahead(tok, "or");
58	2.74k	}
59	94.2k	else if (c == 'i') {
60	1.45k	int c2 = tok_nextc(tok);
61	1.45k	if (c2 == 'f' \|\| c2 == 'n' \|\| c2 == 's') {
62	1.43k	r = 1;
63	1.43k	}
64	1.45k	tok_backup(tok, c2);
65	1.45k	}
66	92.8k	else if (c == 'o') {
67	3.96k	r = lookahead(tok, "r");
68	3.96k	}
69	88.8k	else if (c == 'n') {
70	320	r = lookahead(tok, "ot");
71	320	}
72	98.8k	if (r) {
73	10.1k	tok_backup(tok, c);
74	10.1k	if (_PyTokenizer_parser_warn(tok, PyExc_SyntaxWarning,
75	10.1k	"invalid %s literal", kind))
76	0	{
77	0	return 0;
78	0	}
79	10.1k	tok_nextc(tok);
80	10.1k	}
81	88.6k	else /* In future releases, only error will remain. */
82	88.6k	if (c < 128 && is_potential_identifier_char(c)) {
83	269	tok_backup(tok, c);
84	269	_PyTokenizer_syntaxerror(tok, "invalid %s literal", kind);
85	269	return 0;
86	269	}
87	98.5k	return 1;
88	98.8k	}
89
90		static int
91		tok_decimal_tail(struct tok_state *tok)
92	82.3k	{
93	82.3k	int c;
94
95	83.5k	while (1) {
96	230k	do {
97	230k	c = tok_nextc(tok);
98	230k	} while (Py_ISDIGIT(c));
99	83.5k	if (c != '_') {
100	82.2k	break;
101	82.2k	}
102	1.27k	c = tok_nextc(tok);
103	1.27k	if (!Py_ISDIGIT(c)) {
104	22	tok_backup(tok, c);
105	22	_PyTokenizer_syntaxerror(tok, "invalid decimal literal");
106	22	return 0;
107	22	}
108	1.27k	}
109	82.2k	return c;
110	82.3k	}
111
112		int
113		_PyLexer_scan_number(struct tok_state tok, struct token token, int c,
114		int leading_dot)
115	99.0k	{
116	99.0k	const char *p_start = NULL;
117	99.0k	const char *p_end = NULL;
118
119	99.0k	if (leading_dot) {
120	2.80k	goto fraction;
121	2.80k	}
122	96.2k	if (c == '0') {
123		/* Hex, octal or binary -- maybe. */
124	29.7k	c = tok_nextc(tok);
125	29.7k	if (c == 'x' \|\| c == 'X') {
126		/* Hex */
127	14.0k	c = tok_nextc(tok);
128	16.2k	do {
129	16.2k	if (c == '_') {
130	2.26k	c = tok_nextc(tok);
131	2.26k	}
132	16.2k	if (!Py_ISXDIGIT(c)) {
133	17	tok_backup(tok, c);
134	17	return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid hexadecimal literal"));
135	17	}
136	74.7k	do {
137	74.7k	c = tok_nextc(tok);
138	74.7k	} while (Py_ISXDIGIT(c));
139	16.2k	} while (c == '_');
140	14.0k	if (!verify_end_of_number(tok, c, "hexadecimal")) {
141	6	return MAKE_TOKEN(ERRORTOKEN);
142	6	}
143	14.0k	}
144	15.7k	else if (c == 'o' \|\| c == 'O') {
145		/* Octal */
146	911	c = tok_nextc(tok);
147	2.31k	do {
148	2.31k	if (c == '_') {
149	1.40k	c = tok_nextc(tok);
150	1.40k	}
151	2.31k	if (c < '0' \|\| c >= '8') {
152	21	if (Py_ISDIGIT(c)) {
153	1	return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok,
154	1	"invalid digit '%c' in octal literal", c));
155	1	}
156	20	else {
157	20	tok_backup(tok, c);
158	20	return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid octal literal"));
159	20	}
160	21	}
161	10.4k	do {
162	10.4k	c = tok_nextc(tok);
163	10.4k	} while ('0' <= c && c < '8');
164	2.29k	} while (c == '_');
165	890	if (Py_ISDIGIT(c)) {
166	1	return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok,
167	1	"invalid digit '%c' in octal literal", c));
168	1	}
169	889	if (!verify_end_of_number(tok, c, "octal")) {
170	3	return MAKE_TOKEN(ERRORTOKEN);
171	3	}
172	889	}
173	14.7k	else if (c == 'b' \|\| c == 'B') {
174		/* Binary */
175	552	c = tok_nextc(tok);
176	800	do {
177	800	if (c == '_') {
178	254	c = tok_nextc(tok);
179	254	}
180	800	if (c != '0' && c != '1') {
181	22	if (Py_ISDIGIT(c)) {
182	1	return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid digit '%c' in binary literal", c));
183	1	}
184	21	else {
185	21	tok_backup(tok, c);
186	21	return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid binary literal"));
187	21	}
188	22	}
189	7.07k	do {
190	7.07k	c = tok_nextc(tok);
191	7.07k	} while (c == '0' \|\| c == '1');
192	778	} while (c == '_');
193	530	if (Py_ISDIGIT(c)) {
194	2	return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid digit '%c' in binary literal", c));
195	2	}
196	528	if (!verify_end_of_number(tok, c, "binary")) {
197	2	return MAKE_TOKEN(ERRORTOKEN);
198	2	}
199	528	}
200	14.2k	else {
201	14.2k	int nonzero = 0;
202		/* maybe old-style octal; c is first char of it */
203		/* in any case, allow '0' as a literal */
204	15.9k	while (1) {
205	15.9k	if (c == '_') {
206	313	c = tok_nextc(tok);
207	313	if (!Py_ISDIGIT(c)) {
208	3	tok_backup(tok, c);
209	3	return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid decimal literal"));
210	3	}
211	313	}
212	15.9k	if (c != '0') {
213	14.2k	break;
214	14.2k	}
215	1.70k	c = tok_nextc(tok);
216	1.70k	}
217	14.2k	char* zeros_end = tok->cur;
218	14.2k	if (Py_ISDIGIT(c)) {
219	366	nonzero = 1;
220	366	c = tok_decimal_tail(tok);
221	366	if (c == 0) {
222	4	return MAKE_TOKEN(ERRORTOKEN);
223	4	}
224	366	}
225	14.2k	if (c == '.') {
226	1.10k	c = tok_nextc(tok);
227	1.10k	goto fraction;
228	1.10k	}
229	13.1k	else if (c == 'e' \|\| c == 'E') {
230	756	goto exponent;
231	756	}
232	12.3k	else if (c == 'j' \|\| c == 'J') {
233	839	goto imaginary;
234	839	}
235	11.5k	else if (nonzero && !tok->tok_extra_tokens) {
236		/* Old-style octal: now disallowed. */
237	26	tok_backup(tok, c);
238	26	return MAKE_TOKEN(_PyTokenizer_syntaxerror_known_range(
239	26	tok, (int)(tok->start + 1 - tok->line_start),
240	26	(int)(zeros_end - tok->line_start),
241	26	"leading zeros in decimal integer "
242	26	"literals are not permitted; "
243	26	"use an 0o prefix for octal integers"));
244	26	}
245	11.5k	if (!verify_end_of_number(tok, c, "decimal")) {
246	32	return MAKE_TOKEN(ERRORTOKEN);
247	32	}
248	11.5k	}
249	29.7k	}
250	66.4k	else {
251		/* Decimal */
252	66.4k	c = tok_decimal_tail(tok);
253	66.4k	if (c == 0) {
254	12	return MAKE_TOKEN(ERRORTOKEN);
255	12	}
256	66.4k	{
257		/* Accept floating-point numbers. */
258	66.4k	if (c == '.') {
259	3.58k	c = tok_nextc(tok);
260	7.49k	fraction:
261		/* Fraction */
262	7.49k	if (Py_ISDIGIT(c)) {
263	5.58k	c = tok_decimal_tail(tok);
264	5.58k	if (c == 0) {
265	4	return MAKE_TOKEN(ERRORTOKEN);
266	4	}
267	5.58k	}
268	7.49k	}
269	70.3k	if (c == 'e' \|\| c == 'E') {
270	9.86k	int e;
271	10.6k	exponent:
272	10.6k	e = c;
273		/* Exponent part */
274	10.6k	c = tok_nextc(tok);
275	10.6k	if (c == '+' \|\| c == '-') {
276	4.11k	c = tok_nextc(tok);
277	4.11k	if (!Py_ISDIGIT(c)) {
278	11	tok_backup(tok, c);
279	11	return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid decimal literal"));
280	11	}
281	6.50k	} else if (!Py_ISDIGIT(c)) {
282	725	tok_backup(tok, c);
283	725	if (!verify_end_of_number(tok, e, "decimal")) {
284	56	return MAKE_TOKEN(ERRORTOKEN);
285	56	}
286	669	tok_backup(tok, e);
287	669	p_start = tok->start;
288	669	p_end = tok->cur;
289	669	return MAKE_TOKEN(NUMBER);
290	725	}
291	9.88k	c = tok_decimal_tail(tok);
292	9.88k	if (c == 0) {
293	2	return MAKE_TOKEN(ERRORTOKEN);
294	2	}
295	9.88k	}
296	70.3k	if (c == 'j' \|\| c == 'J') {
297		/* Imaginary part */
298	5.18k	imaginary:
299	5.18k	c = tok_nextc(tok);
300	5.18k	if (!verify_end_of_number(tok, c, "imaginary")) {
301	9	return MAKE_TOKEN(ERRORTOKEN);
302	9	}
303	5.18k	}
304	66.0k	else if (!verify_end_of_number(tok, c, "decimal")) {
305	161	return MAKE_TOKEN(ERRORTOKEN);
306	161	}
307	70.3k	}
308	70.3k	}
309	97.9k	tok_backup(tok, c);
310	97.9k	p_start = tok->start;
311	97.9k	p_end = tok->cur;
312	97.9k	return MAKE_TOKEN(NUMBER);
313	96.2k	}