/src/cpython3/Parser/lexer/number.c

Source
#include "Python.h"
#include "pycore_token.h"

#include "lexer_internal.h"
#include "../tokenizer/helpers.h"

#define MAKE_TOKEN(token_type) _PyLexer_token_setup(tok, token, token_type, p_start, p_end)

static int
lookahead(struct tok_state *tok, const char *test)
{
    const char *s = test;
    int res = 0;
    while (1) {
        int c = tok_nextc(tok);
        if (*s == 0) {
            res = !is_potential_identifier_char(c);
        }
        else if (c == *s) {
            s++;
            continue;
        }

        tok_backup(tok, c);
        while (s != test) {
            tok_backup(tok, *--s);
        }
        return res;
    }
}

static int
verify_end_of_number(struct tok_state *tok, int c, const char *kind) {
    if (tok->tok_extra_tokens) {
        // When we are parsing extra tokens, we don't want to emit warnings
        // about invalid literals, because we want to be a bit more liberal.
        return 1;
    }
    /* Emit a deprecation warning only if the numeric literal is immediately
     * followed by one of keywords which can occur after a numeric literal
     * in valid code: "and", "else", "for", "if", "in", "is" and "or".
     * It allows to gradually deprecate existing valid code without adding
     * warning before error in most cases of invalid numeric literal (which
     * would be confusing and break existing tests).
     * Raise a syntax error with slightly better message than plain
     * "invalid syntax" if the numeric literal is immediately followed by
     * other keyword or identifier.
     */
    int r = 0;
    if (c == 'a') {
        r = lookahead(tok, "nd");
    }
    else if (c == 'e') {
        r = lookahead(tok, "lse");
    }
    else if (c == 'f') {
        r = lookahead(tok, "or");
    }
    else if (c == 'i') {
        int c2 = tok_nextc(tok);
        if (c2 == 'f' || c2 == 'n' || c2 == 's') {
            r = 1;
        }
        tok_backup(tok, c2);
    }
    else if (c == 'o') {
        r = lookahead(tok, "r");
    }
    else if (c == 'n') {
        r = lookahead(tok, "ot");
    }
    if (r) {
        tok_backup(tok, c);
        if (_PyTokenizer_parser_warn(tok, PyExc_SyntaxWarning,
                "invalid %s literal", kind))
        {
            return 0;
        }
        tok_nextc(tok);
    }
    else /* In future releases, only error will remain. */
    if (c < 128 && is_potential_identifier_char(c)) {
        tok_backup(tok, c);
        _PyTokenizer_syntaxerror(tok, "invalid %s literal", kind);
        return 0;
    }
    return 1;
}

static int
tok_decimal_tail(struct tok_state *tok)
{
    int c;

    while (1) {
        do {
            c = tok_nextc(tok);
        } while (Py_ISDIGIT(c));
        if (c != '_') {
            break;
        }
        c = tok_nextc(tok);
        if (!Py_ISDIGIT(c)) {
            tok_backup(tok, c);
            _PyTokenizer_syntaxerror(tok, "invalid decimal literal");
            return 0;
        }
    }
    return c;
}

int
_PyLexer_scan_number(struct tok_state *tok, struct token *token, int c,
                     int leading_dot)
{
    const char *p_start = NULL;
    const char *p_end = NULL;

    if (leading_dot) {
        goto fraction;
    }
    if (c == '0') {
        /* Hex, octal or binary -- maybe. */
        c = tok_nextc(tok);
        if (c == 'x' || c == 'X') {
            /* Hex */
            c = tok_nextc(tok);
            do {
                if (c == '_') {
                    c = tok_nextc(tok);
                }
                if (!Py_ISXDIGIT(c)) {
                    tok_backup(tok, c);
                    return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid hexadecimal literal"));
                }
                do {
                    c = tok_nextc(tok);
                } while (Py_ISXDIGIT(c));
            } while (c == '_');
            if (!verify_end_of_number(tok, c, "hexadecimal")) {
                return MAKE_TOKEN(ERRORTOKEN);
            }
        }
        else if (c == 'o' || c == 'O') {
            /* Octal */
            c = tok_nextc(tok);
            do {
                if (c == '_') {
                    c = tok_nextc(tok);
                }
                if (c < '0' || c >= '8') {
                    if (Py_ISDIGIT(c)) {
                        return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok,
                                "invalid digit '%c' in octal literal", c));
                    }
                    else {
                        tok_backup(tok, c);
                        return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid octal literal"));
                    }
                }
                do {
                    c = tok_nextc(tok);
                } while ('0' <= c && c < '8');
            } while (c == '_');
            if (Py_ISDIGIT(c)) {
                return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok,
                        "invalid digit '%c' in octal literal", c));
            }
            if (!verify_end_of_number(tok, c, "octal")) {
                return MAKE_TOKEN(ERRORTOKEN);
            }
        }
        else if (c == 'b' || c == 'B') {
            /* Binary */
            c = tok_nextc(tok);
            do {
                if (c == '_') {
                    c = tok_nextc(tok);
                }
                if (c != '0' && c != '1') {
                    if (Py_ISDIGIT(c)) {
                        return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid digit '%c' in binary literal", c));
                    }
                    else {
                        tok_backup(tok, c);
                        return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid binary literal"));
                    }
                }
                do {
                    c = tok_nextc(tok);
                } while (c == '0' || c == '1');
            } while (c == '_');
            if (Py_ISDIGIT(c)) {
                return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid digit '%c' in binary literal", c));
            }
            if (!verify_end_of_number(tok, c, "binary")) {
                return MAKE_TOKEN(ERRORTOKEN);
            }
        }
        else {
            int nonzero = 0;
            /* maybe old-style octal; c is first char of it */
            /* in any case, allow '0' as a literal */
            while (1) {
                if (c == '_') {
                    c = tok_nextc(tok);
                    if (!Py_ISDIGIT(c)) {
                        tok_backup(tok, c);
                        return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid decimal literal"));
                    }
                }
                if (c != '0') {
                    break;
                }
                c = tok_nextc(tok);
            }
            char* zeros_end = tok->cur;
            if (Py_ISDIGIT(c)) {
                nonzero = 1;
                c = tok_decimal_tail(tok);
                if (c == 0) {
                    return MAKE_TOKEN(ERRORTOKEN);
                }
            }
            if (c == '.') {
                c = tok_nextc(tok);
                goto fraction;
            }
            else if (c == 'e' || c == 'E') {
                goto exponent;
            }
            else if (c == 'j' || c == 'J') {
                goto imaginary;
            }
            else if (nonzero && !tok->tok_extra_tokens) {
                /* Old-style octal: now disallowed. */
                tok_backup(tok, c);
                return MAKE_TOKEN(_PyTokenizer_syntaxerror_known_range(
                        tok, (int)(tok->start + 1 - tok->line_start),
                        (int)(zeros_end - tok->line_start),
                        "leading zeros in decimal integer "
                        "literals are not permitted; "
                        "use an 0o prefix for octal integers"));
            }
            if (!verify_end_of_number(tok, c, "decimal")) {
                return MAKE_TOKEN(ERRORTOKEN);
            }
        }
    }
    else {
        /* Decimal */
        c = tok_decimal_tail(tok);
        if (c == 0) {
            return MAKE_TOKEN(ERRORTOKEN);
        }
        {
            /* Accept floating-point numbers. */
            if (c == '.') {
                c = tok_nextc(tok);
    fraction:
                /* Fraction */
                if (Py_ISDIGIT(c)) {
                    c = tok_decimal_tail(tok);
                    if (c == 0) {
                        return MAKE_TOKEN(ERRORTOKEN);
                    }
                }
            }
            if (c == 'e' || c == 'E') {
                int e;
              exponent:
                e = c;
                /* Exponent part */
                c = tok_nextc(tok);
                if (c == '+' || c == '-') {
                    c = tok_nextc(tok);
                    if (!Py_ISDIGIT(c)) {
                        tok_backup(tok, c);
                        return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid decimal literal"));
                    }
                } else if (!Py_ISDIGIT(c)) {
                    tok_backup(tok, c);
                    if (!verify_end_of_number(tok, e, "decimal")) {
                        return MAKE_TOKEN(ERRORTOKEN);
                    }
                    tok_backup(tok, e);
                    p_start = tok->start;
                    p_end = tok->cur;
                    return MAKE_TOKEN(NUMBER);
                }
                c = tok_decimal_tail(tok);
                if (c == 0) {
                    return MAKE_TOKEN(ERRORTOKEN);
                }
            }
            if (c == 'j' || c == 'J') {
                /* Imaginary part */
    imaginary:
                c = tok_nextc(tok);
                if (!verify_end_of_number(tok, c, "imaginary")) {
                    return MAKE_TOKEN(ERRORTOKEN);
                }
            }
            else if (!verify_end_of_number(tok, c, "decimal")) {
                return MAKE_TOKEN(ERRORTOKEN);
            }
        }
    }
    tok_backup(tok, c);
    p_start = tok->start;
    p_end = tok->cur;
    return MAKE_TOKEN(NUMBER);
}

Coverage Report

Created: 2026-08-31 07:18

Line	Count	Source
1		#include "Python.h"
2		#include "pycore_token.h"
3
4		#include "lexer_internal.h"
5		#include "../tokenizer/helpers.h"
6
7	1.07M	#define MAKE_TOKEN(token_type) _PyLexer_token_setup(tok, token, token_type, p_start, p_end)
8
9		static int
10		lookahead(struct tok_state tok, const char test)
11	19.4k	{
12	19.4k	const char *s = test;
13	19.4k	int res = 0;
14	56.4k	while (1) {
15	56.4k	int c = tok_nextc(tok);
16	56.4k	if (*s == 0) {
17	19.2k	res = !is_potential_identifier_char(c);
18	19.2k	}
19	37.2k	else if (c == *s) {
20	37.0k	s++;
21	37.0k	continue;
22	37.0k	}
23
24	19.4k	tok_backup(tok, c);
25	56.4k	while (s != test) {
26	37.0k	tok_backup(tok, *--s);
27	37.0k	}
28	19.4k	return res;
29	56.4k	}
30	19.4k	}
31
32		static int
33	1.07M	verify_end_of_number(struct tok_state tok, int c, const char kind) {
34	1.07M	if (tok->tok_extra_tokens) {
35		// When we are parsing extra tokens, we don't want to emit warnings
36		// about invalid literals, because we want to be a bit more liberal.
37	0	return 1;
38	0	}
39		/* Emit a deprecation warning only if the numeric literal is immediately
40		* followed by one of keywords which can occur after a numeric literal
41		* in valid code: "and", "else", "for", "if", "in", "is" and "or".
42		* It allows to gradually deprecate existing valid code without adding
43		* warning before error in most cases of invalid numeric literal (which
44		* would be confusing and break existing tests).
45		* Raise a syntax error with slightly better message than plain
46		* "invalid syntax" if the numeric literal is immediately followed by
47		* other keyword or identifier.
48		*/
49	1.07M	int r = 0;
50	1.07M	if (c == 'a') {
51	7.23k	r = lookahead(tok, "nd");
52	7.23k	}
53	1.06M	else if (c == 'e') {
54	3.37k	r = lookahead(tok, "lse");
55	3.37k	}
56	1.06M	else if (c == 'f') {
57	3.58k	r = lookahead(tok, "or");
58	3.58k	}
59	1.05M	else if (c == 'i') {
60	12.5k	int c2 = tok_nextc(tok);
61	12.5k	if (c2 == 'f' \|\| c2 == 'n' \|\| c2 == 's') {
62	12.5k	r = 1;
63	12.5k	}
64	12.5k	tok_backup(tok, c2);
65	12.5k	}
66	1.04M	else if (c == 'o') {
67	4.83k	r = lookahead(tok, "r");
68	4.83k	}
69	1.03M	else if (c == 'n') {
70	393	r = lookahead(tok, "ot");
71	393	}
72	1.07M	if (r) {
73	31.7k	tok_backup(tok, c);
74	31.7k	if (_PyTokenizer_parser_warn(tok, PyExc_SyntaxWarning,
75	31.7k	"invalid %s literal", kind))
76	0	{
77	0	return 0;
78	0	}
79	31.7k	tok_nextc(tok);
80	31.7k	}
81	1.03M	else /* In future releases, only error will remain. */
82	1.03M	if (c < 128 && is_potential_identifier_char(c)) {
83	502	tok_backup(tok, c);
84	502	_PyTokenizer_syntaxerror(tok, "invalid %s literal", kind);
85	502	return 0;
86	502	}
87	1.07M	return 1;
88	1.07M	}
89
90		static int
91		tok_decimal_tail(struct tok_state *tok)
92	1.08M	{
93	1.08M	int c;
94
95	1.10M	while (1) {
96	5.58M	do {
97	5.58M	c = tok_nextc(tok);
98	5.58M	} while (Py_ISDIGIT(c));
99	1.10M	if (c != '_') {
100	1.08M	break;
101	1.08M	}
102	13.6k	c = tok_nextc(tok);
103	13.6k	if (!Py_ISDIGIT(c)) {
104	20	tok_backup(tok, c);
105	20	_PyTokenizer_syntaxerror(tok, "invalid decimal literal");
106	20	return 0;
107	20	}
108	13.6k	}
109	1.08M	return c;
110	1.08M	}
111
112		int
113		_PyLexer_scan_number(struct tok_state tok, struct token token, int c,
114		int leading_dot)
115	1.07M	{
116	1.07M	const char *p_start = NULL;
117	1.07M	const char *p_end = NULL;
118
119	1.07M	if (leading_dot) {
120	15.9k	goto fraction;
121	15.9k	}
122	1.05M	if (c == '0') {
123		/* Hex, octal or binary -- maybe. */
124	69.2k	c = tok_nextc(tok);
125	69.2k	if (c == 'x' \|\| c == 'X') {
126		/* Hex */
127	2.10k	c = tok_nextc(tok);
128	3.44k	do {
129	3.44k	if (c == '_') {
130	1.34k	c = tok_nextc(tok);
131	1.34k	}
132	3.44k	if (!Py_ISXDIGIT(c)) {
133	9	tok_backup(tok, c);
134	9	return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid hexadecimal literal"));
135	9	}
136	93.0k	do {
137	93.0k	c = tok_nextc(tok);
138	93.0k	} while (Py_ISXDIGIT(c));
139	3.43k	} while (c == '_');
140	2.09k	if (!verify_end_of_number(tok, c, "hexadecimal")) {
141	2	return MAKE_TOKEN(ERRORTOKEN);
142	2	}
143	2.09k	}
144	67.1k	else if (c == 'o' \|\| c == 'O') {
145		/* Octal */
146	1.51k	c = tok_nextc(tok);
147	2.20k	do {
148	2.20k	if (c == '_') {
149	692	c = tok_nextc(tok);
150	692	}
151	2.20k	if (c < '0' \|\| c >= '8') {
152	12	if (Py_ISDIGIT(c)) {
153	1	return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok,
154	1	"invalid digit '%c' in octal literal", c));
155	1	}
156	11	else {
157	11	tok_backup(tok, c);
158	11	return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid octal literal"));
159	11	}
160	12	}
161	19.7k	do {
162	19.7k	c = tok_nextc(tok);
163	19.7k	} while ('0' <= c && c < '8');
164	2.19k	} while (c == '_');
165	1.50k	if (Py_ISDIGIT(c)) {
166	1	return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok,
167	1	"invalid digit '%c' in octal literal", c));
168	1	}
169	1.50k	if (!verify_end_of_number(tok, c, "octal")) {
170	7	return MAKE_TOKEN(ERRORTOKEN);
171	7	}
172	1.50k	}
173	65.6k	else if (c == 'b' \|\| c == 'B') {
174		/* Binary */
175	1.69k	c = tok_nextc(tok);
176	2.40k	do {
177	2.40k	if (c == '_') {
178	1.66k	c = tok_nextc(tok);
179	1.66k	}
180	2.40k	if (c != '0' && c != '1') {
181	8	if (Py_ISDIGIT(c)) {
182	1	return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid digit '%c' in binary literal", c));
183	1	}
184	7	else {
185	7	tok_backup(tok, c);
186	7	return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid binary literal"));
187	7	}
188	8	}
189	57.1k	do {
190	57.1k	c = tok_nextc(tok);
191	57.1k	} while (c == '0' \|\| c == '1');
192	2.39k	} while (c == '_');
193	1.69k	if (Py_ISDIGIT(c)) {
194	3	return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid digit '%c' in binary literal", c));
195	3	}
196	1.68k	if (!verify_end_of_number(tok, c, "binary")) {
197	4	return MAKE_TOKEN(ERRORTOKEN);
198	4	}
199	1.68k	}
200	63.9k	else {
201	63.9k	int nonzero = 0;
202		/* maybe old-style octal; c is first char of it */
203		/* in any case, allow '0' as a literal */
204	132k	while (1) {
205	132k	if (c == '_') {
206	427	c = tok_nextc(tok);
207	427	if (!Py_ISDIGIT(c)) {
208	4	tok_backup(tok, c);
209	4	return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid decimal literal"));
210	4	}
211	427	}
212	132k	if (c != '0') {
213	63.9k	break;
214	63.9k	}
215	68.5k	c = tok_nextc(tok);
216	68.5k	}
217	63.9k	char* zeros_end = tok->cur;
218	63.9k	if (Py_ISDIGIT(c)) {
219	2.97k	nonzero = 1;
220	2.97k	c = tok_decimal_tail(tok);
221	2.97k	if (c == 0) {
222	2	return MAKE_TOKEN(ERRORTOKEN);
223	2	}
224	2.97k	}
225	63.9k	if (c == '.') {
226	7.69k	c = tok_nextc(tok);
227	7.69k	goto fraction;
228	7.69k	}
229	56.2k	else if (c == 'e' \|\| c == 'E') {
230	1.93k	goto exponent;
231	1.93k	}
232	54.3k	else if (c == 'j' \|\| c == 'J') {
233	11.9k	goto imaginary;
234	11.9k	}
235	42.3k	else if (nonzero && !tok->tok_extra_tokens) {
236		/* Old-style octal: now disallowed. */
237	84	tok_backup(tok, c);
238	84	return MAKE_TOKEN(_PyTokenizer_syntaxerror_known_range(
239	84	tok, (int)(tok->start + 1 - tok->line_start),
240	84	(int)(zeros_end - tok->line_start),
241	84	"leading zeros in decimal integer "
242	84	"literals are not permitted; "
243	84	"use an 0o prefix for octal integers"));
244	84	}
245	42.2k	if (!verify_end_of_number(tok, c, "decimal")) {
246	69	return MAKE_TOKEN(ERRORTOKEN);
247	69	}
248	42.2k	}
249	69.2k	}
250	985k	else {
251		/* Decimal */
252	985k	c = tok_decimal_tail(tok);
253	985k	if (c == 0) {
254	12	return MAKE_TOKEN(ERRORTOKEN);
255	12	}
256	985k	{
257		/* Accept floating-point numbers. */
258	985k	if (c == '.') {
259	49.9k	c = tok_nextc(tok);
260	73.5k	fraction:
261		/* Fraction */
262	73.5k	if (Py_ISDIGIT(c)) {
263	41.3k	c = tok_decimal_tail(tok);
264	41.3k	if (c == 0) {
265	4	return MAKE_TOKEN(ERRORTOKEN);
266	4	}
267	41.3k	}
268	73.5k	}
269	1.00M	if (c == 'e' \|\| c == 'E') {
270	59.4k	int e;
271	61.3k	exponent:
272	61.3k	e = c;
273		/* Exponent part */
274	61.3k	c = tok_nextc(tok);
275	61.3k	if (c == '+' \|\| c == '-') {
276	10.3k	c = tok_nextc(tok);
277	10.3k	if (!Py_ISDIGIT(c)) {
278	8	tok_backup(tok, c);
279	8	return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid decimal literal"));
280	8	}
281	51.0k	} else if (!Py_ISDIGIT(c)) {
282	3.37k	tok_backup(tok, c);
283	3.37k	if (!verify_end_of_number(tok, e, "decimal")) {
284	58	return MAKE_TOKEN(ERRORTOKEN);
285	58	}
286	3.31k	tok_backup(tok, e);
287	3.31k	p_start = tok->start;
288	3.31k	p_end = tok->cur;
289	3.31k	return MAKE_TOKEN(NUMBER);
290	3.37k	}
291	58.0k	c = tok_decimal_tail(tok);
292	58.0k	if (c == 0) {
293	2	return MAKE_TOKEN(ERRORTOKEN);
294	2	}
295	58.0k	}
296	1.00M	if (c == 'j' \|\| c == 'J') {
297		/* Imaginary part */
298	93.1k	imaginary:
299	93.1k	c = tok_nextc(tok);
300	93.1k	if (!verify_end_of_number(tok, c, "imaginary")) {
301	21	return MAKE_TOKEN(ERRORTOKEN);
302	21	}
303	93.1k	}
304	926k	else if (!verify_end_of_number(tok, c, "decimal")) {
305	341	return MAKE_TOKEN(ERRORTOKEN);
306	341	}
307	1.00M	}
308	1.00M	}
309	1.06M	tok_backup(tok, c);
310	1.06M	p_start = tok->start;
311	1.06M	p_end = tok->cur;
312	1.06M	return MAKE_TOKEN(NUMBER);
313	1.05M	}