/src/cpython/Parser/lexer/number.c

Source
#include "Python.h"
#include "pycore_token.h"

#include "lexer_internal.h"
#include "../tokenizer/helpers.h"

#define MAKE_TOKEN(token_type) _PyLexer_token_setup(tok, token, token_type, p_start, p_end)

static int
lookahead(struct tok_state *tok, const char *test)
{
    const char *s = test;
    int res = 0;
    while (1) {
        int c = tok_nextc(tok);
        if (*s == 0) {
            res = !is_potential_identifier_char(c);
        }
        else if (c == *s) {
            s++;
            continue;
        }

        tok_backup(tok, c);
        while (s != test) {
            tok_backup(tok, *--s);
        }
        return res;
    }
}

static int
verify_end_of_number(struct tok_state *tok, int c, const char *kind) {
    if (tok->tok_extra_tokens) {
        // When we are parsing extra tokens, we don't want to emit warnings
        // about invalid literals, because we want to be a bit more liberal.
        return 1;
    }
    /* Emit a deprecation warning only if the numeric literal is immediately
     * followed by one of keywords which can occur after a numeric literal
     * in valid code: "and", "else", "for", "if", "in", "is" and "or".
     * It allows to gradually deprecate existing valid code without adding
     * warning before error in most cases of invalid numeric literal (which
     * would be confusing and break existing tests).
     * Raise a syntax error with slightly better message than plain
     * "invalid syntax" if the numeric literal is immediately followed by
     * other keyword or identifier.
     */
    int r = 0;
    if (c == 'a') {
        r = lookahead(tok, "nd");
    }
    else if (c == 'e') {
        r = lookahead(tok, "lse");
    }
    else if (c == 'f') {
        r = lookahead(tok, "or");
    }
    else if (c == 'i') {
        int c2 = tok_nextc(tok);
        if (c2 == 'f' || c2 == 'n' || c2 == 's') {
            r = 1;
        }
        tok_backup(tok, c2);
    }
    else if (c == 'o') {
        r = lookahead(tok, "r");
    }
    else if (c == 'n') {
        r = lookahead(tok, "ot");
    }
    if (r) {
        tok_backup(tok, c);
        if (_PyTokenizer_parser_warn(tok, PyExc_SyntaxWarning,
                "invalid %s literal", kind))
        {
            return 0;
        }
        tok_nextc(tok);
    }
    else /* In future releases, only error will remain. */
    if (c < 128 && is_potential_identifier_char(c)) {
        tok_backup(tok, c);
        _PyTokenizer_syntaxerror(tok, "invalid %s literal", kind);
        return 0;
    }
    return 1;
}

static int
tok_decimal_tail(struct tok_state *tok)
{
    int c;

    while (1) {
        do {
            c = tok_nextc(tok);
        } while (Py_ISDIGIT(c));
        if (c != '_') {
            break;
        }
        c = tok_nextc(tok);
        if (!Py_ISDIGIT(c)) {
            tok_backup(tok, c);
            _PyTokenizer_syntaxerror(tok, "invalid decimal literal");
            return 0;
        }
    }
    return c;
}

int
_PyLexer_scan_number(struct tok_state *tok, struct token *token, int c,
                     int leading_dot)
{
    _PyTok_Off p_start = -1;
    _PyTok_Off p_end = -1;

    if (leading_dot) {
        goto fraction;
    }
    if (c == '0') {
        /* Hex, octal or binary -- maybe. */
        c = tok_nextc(tok);
        if (c == 'x' || c == 'X') {
            /* Hex */
            c = tok_nextc(tok);
            do {
                if (c == '_') {
                    c = tok_nextc(tok);
                }
                if (!Py_ISXDIGIT(c)) {
                    tok_backup(tok, c);
                    return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid hexadecimal literal"));
                }
                do {
                    c = tok_nextc(tok);
                } while (Py_ISXDIGIT(c));
            } while (c == '_');
            if (!verify_end_of_number(tok, c, "hexadecimal")) {
                return MAKE_TOKEN(ERRORTOKEN);
            }
        }
        else if (c == 'o' || c == 'O') {
            /* Octal */
            c = tok_nextc(tok);
            do {
                if (c == '_') {
                    c = tok_nextc(tok);
                }
                if (c < '0' || c >= '8') {
                    if (Py_ISDIGIT(c)) {
                        return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok,
                                "invalid digit '%c' in octal literal", c));
                    }
                    else {
                        tok_backup(tok, c);
                        return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid octal literal"));
                    }
                }
                do {
                    c = tok_nextc(tok);
                } while ('0' <= c && c < '8');
            } while (c == '_');
            if (Py_ISDIGIT(c)) {
                return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok,
                        "invalid digit '%c' in octal literal", c));
            }
            if (!verify_end_of_number(tok, c, "octal")) {
                return MAKE_TOKEN(ERRORTOKEN);
            }
        }
        else if (c == 'b' || c == 'B') {
            /* Binary */
            c = tok_nextc(tok);
            do {
                if (c == '_') {
                    c = tok_nextc(tok);
                }
                if (c != '0' && c != '1') {
                    if (Py_ISDIGIT(c)) {
                        return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid digit '%c' in binary literal", c));
                    }
                    else {
                        tok_backup(tok, c);
                        return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid binary literal"));
                    }
                }
                do {
                    c = tok_nextc(tok);
                } while (c == '0' || c == '1');
            } while (c == '_');
            if (Py_ISDIGIT(c)) {
                return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid digit '%c' in binary literal", c));
            }
            if (!verify_end_of_number(tok, c, "binary")) {
                return MAKE_TOKEN(ERRORTOKEN);
            }
        }
        else {
            int nonzero = 0;
            /* maybe old-style octal; c is first char of it */
            /* in any case, allow '0' as a literal */
            while (1) {
                if (c == '_') {
                    c = tok_nextc(tok);
                    if (!Py_ISDIGIT(c)) {
                        tok_backup(tok, c);
                        return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid decimal literal"));
                    }
                }
                if (c != '0') {
                    break;
                }
                c = tok_nextc(tok);
            }
            _PyTok_Off zeros_end = tok->cur;
            if (Py_ISDIGIT(c)) {
                nonzero = 1;
                c = tok_decimal_tail(tok);
                if (c == 0) {
                    return MAKE_TOKEN(ERRORTOKEN);
                }
            }
            if (c == '.') {
                c = tok_nextc(tok);
                goto fraction;
            }
            else if (c == 'e' || c == 'E') {
                goto exponent;
            }
            else if (c == 'j' || c == 'J') {
                goto imaginary;
            }
            else if (nonzero && !tok->tok_extra_tokens) {
                /* Old-style octal: now disallowed. */
                tok_backup(tok, c);
                return MAKE_TOKEN(_PyTokenizer_syntaxerror_known_range(
                        tok, (int)(tok->start + 1 - tok->line_start),
                        (int)(zeros_end - tok->line_start),
                        "leading zeros in decimal integer "
                        "literals are not permitted; "
                        "use an 0o prefix for octal integers"));
            }
            if (!verify_end_of_number(tok, c, "decimal")) {
                return MAKE_TOKEN(ERRORTOKEN);
            }
        }
    }
    else {
        /* Decimal */
        c = tok_decimal_tail(tok);
        if (c == 0) {
            return MAKE_TOKEN(ERRORTOKEN);
        }
        {
            /* Accept floating-point numbers. */
            if (c == '.') {
                c = tok_nextc(tok);
    fraction:
                /* Fraction */
                if (Py_ISDIGIT(c)) {
                    c = tok_decimal_tail(tok);
                    if (c == 0) {
                        return MAKE_TOKEN(ERRORTOKEN);
                    }
                }
            }
            if (c == 'e' || c == 'E') {
                int e;
              exponent:
                e = c;
                /* Exponent part */
                c = tok_nextc(tok);
                if (c == '+' || c == '-') {
                    c = tok_nextc(tok);
                    if (!Py_ISDIGIT(c)) {
                        tok_backup(tok, c);
                        return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid decimal literal"));
                    }
                } else if (!Py_ISDIGIT(c)) {
                    tok_backup(tok, c);
                    if (!verify_end_of_number(tok, e, "decimal")) {
                        return MAKE_TOKEN(ERRORTOKEN);
                    }
                    tok_backup(tok, e);
                    p_start = tok->start;
                    p_end = tok->cur;
                    return MAKE_TOKEN(NUMBER);
                }
                c = tok_decimal_tail(tok);
                if (c == 0) {
                    return MAKE_TOKEN(ERRORTOKEN);
                }
            }
            if (c == 'j' || c == 'J') {
                /* Imaginary part */
    imaginary:
                c = tok_nextc(tok);
                if (!verify_end_of_number(tok, c, "imaginary")) {
                    return MAKE_TOKEN(ERRORTOKEN);
                }
            }
            else if (!verify_end_of_number(tok, c, "decimal")) {
                return MAKE_TOKEN(ERRORTOKEN);
            }
        }
    }
    tok_backup(tok, c);
    p_start = tok->start;
    p_end = tok->cur;
    return MAKE_TOKEN(NUMBER);
}

Coverage Report

Created: 2026-10-17 06:15

Line	Count	Source
1		#include "Python.h"
2		#include "pycore_token.h"
3
4		#include "lexer_internal.h"
5		#include "../tokenizer/helpers.h"
6
7	102k	#define MAKE_TOKEN(token_type) _PyLexer_token_setup(tok, token, token_type, p_start, p_end)
8
9		static int
10		lookahead(struct tok_state tok, const char test)
11	8.48k	{
12	8.48k	const char *s = test;
13	8.48k	int res = 0;
14	22.4k	while (1) {
15	22.4k	int c = tok_nextc(tok);
16	22.4k	if (*s == 0) {
17	8.36k	res = !is_potential_identifier_char(c);
18	8.36k	}
19	14.1k	else if (c == *s) {
20	14.0k	s++;
21	14.0k	continue;
22	14.0k	}
23
24	8.48k	tok_backup(tok, c);
25	22.4k	while (s != test) {
26	14.0k	tok_backup(tok, *--s);
27	14.0k	}
28	8.48k	return res;
29	22.4k	}
30	8.48k	}
31
32		static int
33	102k	verify_end_of_number(struct tok_state tok, int c, const char kind) {
34	102k	if (tok->tok_extra_tokens) {
35		// When we are parsing extra tokens, we don't want to emit warnings
36		// about invalid literals, because we want to be a bit more liberal.
37	56	return 1;
38	56	}
39		/* Emit a deprecation warning only if the numeric literal is immediately
40		* followed by one of keywords which can occur after a numeric literal
41		* in valid code: "and", "else", "for", "if", "in", "is" and "or".
42		* It allows to gradually deprecate existing valid code without adding
43		* warning before error in most cases of invalid numeric literal (which
44		* would be confusing and break existing tests).
45		* Raise a syntax error with slightly better message than plain
46		* "invalid syntax" if the numeric literal is immediately followed by
47		* other keyword or identifier.
48		*/
49	102k	int r = 0;
50	102k	if (c == 'a') {
51	997	r = lookahead(tok, "nd");
52	997	}
53	101k	else if (c == 'e') {
54	921	r = lookahead(tok, "lse");
55	921	}
56	100k	else if (c == 'f') {
57	2.62k	r = lookahead(tok, "or");
58	2.62k	}
59	98.2k	else if (c == 'i') {
60	1.38k	int c2 = tok_nextc(tok);
61	1.38k	if (c2 == 'f' \|\| c2 == 'n' \|\| c2 == 's') {
62	1.37k	r = 1;
63	1.37k	}
64	1.38k	tok_backup(tok, c2);
65	1.38k	}
66	96.8k	else if (c == 'o') {
67	3.62k	r = lookahead(tok, "r");
68	3.62k	}
69	93.2k	else if (c == 'n') {
70	316	r = lookahead(tok, "ot");
71	316	}
72	102k	if (r) {
73	9.72k	tok_backup(tok, c);
74	9.72k	if (_PyTokenizer_parser_warn(tok, PyExc_SyntaxWarning,
75	9.72k	"invalid %s literal", kind))
76	0	{
77	0	return 0;
78	0	}
79	9.72k	tok_nextc(tok);
80	9.72k	}
81	93.0k	else /* In future releases, only error will remain. */
82	93.0k	if (c < 128 && is_potential_identifier_char(c)) {
83	253	tok_backup(tok, c);
84	253	_PyTokenizer_syntaxerror(tok, "invalid %s literal", kind);
85	253	return 0;
86	253	}
87	102k	return 1;
88	102k	}
89
90		static int
91		tok_decimal_tail(struct tok_state *tok)
92	89.5k	{
93	89.5k	int c;
94
95	90.4k	while (1) {
96	314k	do {
97	314k	c = tok_nextc(tok);
98	314k	} while (Py_ISDIGIT(c));
99	90.4k	if (c != '_') {
100	89.5k	break;
101	89.5k	}
102	866	c = tok_nextc(tok);
103	866	if (!Py_ISDIGIT(c)) {
104	11	tok_backup(tok, c);
105	11	_PyTokenizer_syntaxerror(tok, "invalid decimal literal");
106	11	return 0;
107	11	}
108	866	}
109	89.5k	return c;
110	89.5k	}
111
112		int
113		_PyLexer_scan_number(struct tok_state tok, struct token token, int c,
114		int leading_dot)
115	102k	{
116	102k	_PyTok_Off p_start = -1;
117	102k	_PyTok_Off p_end = -1;
118
119	102k	if (leading_dot) {
120	3.95k	goto fraction;
121	3.95k	}
122	99.0k	if (c == '0') {
123		/* Hex, octal or binary -- maybe. */
124	30.4k	c = tok_nextc(tok);
125	30.4k	if (c == 'x' \|\| c == 'X') {
126		/* Hex */
127	14.1k	c = tok_nextc(tok);
128	14.3k	do {
129	14.3k	if (c == '_') {
130	206	c = tok_nextc(tok);
131	206	}
132	14.3k	if (!Py_ISXDIGIT(c)) {
133	16	tok_backup(tok, c);
134	16	return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid hexadecimal literal"));
135	16	}
136	76.4k	do {
137	76.4k	c = tok_nextc(tok);
138	76.4k	} while (Py_ISXDIGIT(c));
139	14.2k	} while (c == '_');
140	14.0k	if (!verify_end_of_number(tok, c, "hexadecimal")) {
141	1	return MAKE_TOKEN(ERRORTOKEN);
142	1	}
143	14.0k	}
144	16.3k	else if (c == 'o' \|\| c == 'O') {
145		/* Octal */
146	1.21k	c = tok_nextc(tok);
147	3.66k	do {
148	3.66k	if (c == '_') {
149	2.46k	c = tok_nextc(tok);
150	2.46k	}
151	3.66k	if (c < '0' \|\| c >= '8') {
152	18	if (Py_ISDIGIT(c)) {
153	1	return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok,
154	1	"invalid digit '%c' in octal literal", c));
155	1	}
156	17	else {
157	17	tok_backup(tok, c);
158	17	return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid octal literal"));
159	17	}
160	18	}
161	22.8k	do {
162	22.8k	c = tok_nextc(tok);
163	22.8k	} while ('0' <= c && c < '8');
164	3.64k	} while (c == '_');
165	1.19k	if (Py_ISDIGIT(c)) {
166	1	return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok,
167	1	"invalid digit '%c' in octal literal", c));
168	1	}
169	1.19k	if (!verify_end_of_number(tok, c, "octal")) {
170	4	return MAKE_TOKEN(ERRORTOKEN);
171	4	}
172	1.19k	}
173	15.1k	else if (c == 'b' \|\| c == 'B') {
174		/* Binary */
175	496	c = tok_nextc(tok);
176	856	do {
177	856	if (c == '_') {
178	368	c = tok_nextc(tok);
179	368	}
180	856	if (c != '0' && c != '1') {
181	23	if (Py_ISDIGIT(c)) {
182	1	return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid digit '%c' in binary literal", c));
183	1	}
184	22	else {
185	22	tok_backup(tok, c);
186	22	return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid binary literal"));
187	22	}
188	23	}
189	6.85k	do {
190	6.85k	c = tok_nextc(tok);
191	6.85k	} while (c == '0' \|\| c == '1');
192	833	} while (c == '_');
193	473	if (Py_ISDIGIT(c)) {
194	1	return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid digit '%c' in binary literal", c));
195	1	}
196	472	if (!verify_end_of_number(tok, c, "binary")) {
197	3	return MAKE_TOKEN(ERRORTOKEN);
198	3	}
199	472	}
200	14.6k	else {
201	14.6k	int nonzero = 0;
202		/* maybe old-style octal; c is first char of it */
203		/* in any case, allow '0' as a literal */
204	16.9k	while (1) {
205	16.9k	if (c == '_') {
206	303	c = tok_nextc(tok);
207	303	if (!Py_ISDIGIT(c)) {
208	3	tok_backup(tok, c);
209	3	return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid decimal literal"));
210	3	}
211	303	}
212	16.9k	if (c != '0') {
213	14.6k	break;
214	14.6k	}
215	2.36k	c = tok_nextc(tok);
216	2.36k	}
217	14.6k	_PyTok_Off zeros_end = tok->cur;
218	14.6k	if (Py_ISDIGIT(c)) {
219	804	nonzero = 1;
220	804	c = tok_decimal_tail(tok);
221	804	if (c == 0) {
222	1	return MAKE_TOKEN(ERRORTOKEN);
223	1	}
224	804	}
225	14.6k	if (c == '.') {
226	1.13k	c = tok_nextc(tok);
227	1.13k	goto fraction;
228	1.13k	}
229	13.4k	else if (c == 'e' \|\| c == 'E') {
230	955	goto exponent;
231	955	}
232	12.5k	else if (c == 'j' \|\| c == 'J') {
233	1.03k	goto imaginary;
234	1.03k	}
235	11.4k	else if (nonzero && !tok->tok_extra_tokens) {
236		/* Old-style octal: now disallowed. */
237	51	tok_backup(tok, c);
238	51	return MAKE_TOKEN(_PyTokenizer_syntaxerror_known_range(
239	51	tok, (int)(tok->start + 1 - tok->line_start),
240	51	(int)(zeros_end - tok->line_start),
241	51	"leading zeros in decimal integer "
242	51	"literals are not permitted; "
243	51	"use an 0o prefix for octal integers"));
244	51	}
245	11.4k	if (!verify_end_of_number(tok, c, "decimal")) {
246	30	return MAKE_TOKEN(ERRORTOKEN);
247	30	}
248	11.4k	}
249	30.4k	}
250	68.5k	else {
251		/* Decimal */
252	68.5k	c = tok_decimal_tail(tok);
253	68.5k	if (c == 0) {
254	6	return MAKE_TOKEN(ERRORTOKEN);
255	6	}
256	68.5k	{
257		/* Accept floating-point numbers. */
258	68.5k	if (c == '.') {
259	4.43k	c = tok_nextc(tok);
260	9.52k	fraction:
261		/* Fraction */
262	9.52k	if (Py_ISDIGIT(c)) {
263	7.12k	c = tok_decimal_tail(tok);
264	7.12k	if (c == 0) {
265	2	return MAKE_TOKEN(ERRORTOKEN);
266	2	}
267	7.12k	}
268	9.52k	}
269	73.6k	if (c == 'e' \|\| c == 'E') {
270	13.0k	int e;
271	13.9k	exponent:
272	13.9k	e = c;
273		/* Exponent part */
274	13.9k	c = tok_nextc(tok);
275	13.9k	if (c == '+' \|\| c == '-') {
276	6.21k	c = tok_nextc(tok);
277	6.21k	if (!Py_ISDIGIT(c)) {
278	12	tok_backup(tok, c);
279	12	return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid decimal literal"));
280	12	}
281	7.77k	} else if (!Py_ISDIGIT(c)) {
282	911	tok_backup(tok, c);
283	911	if (!verify_end_of_number(tok, e, "decimal")) {
284	63	return MAKE_TOKEN(ERRORTOKEN);
285	63	}
286	848	tok_backup(tok, e);
287	848	p_start = tok->start;
288	848	p_end = tok->cur;
289	848	return MAKE_TOKEN(NUMBER);
290	911	}
291	13.0k	c = tok_decimal_tail(tok);
292	13.0k	if (c == 0) {
293	2	return MAKE_TOKEN(ERRORTOKEN);
294	2	}
295	13.0k	}
296	73.6k	if (c == 'j' \|\| c == 'J') {
297		/* Imaginary part */
298	5.64k	imaginary:
299	5.64k	c = tok_nextc(tok);
300	5.64k	if (!verify_end_of_number(tok, c, "imaginary")) {
301	13	return MAKE_TOKEN(ERRORTOKEN);
302	13	}
303	5.64k	}
304	69.0k	else if (!verify_end_of_number(tok, c, "decimal")) {
305	139	return MAKE_TOKEN(ERRORTOKEN);
306	139	}
307	73.6k	}
308	73.6k	}
309	101k	tok_backup(tok, c);
310	101k	p_start = tok->start;
311	101k	p_end = tok->cur;
312	101k	return MAKE_TOKEN(NUMBER);
313	99.0k	}