/src/cpython/Parser/lexer/number.c

Source
#include "Python.h"
#include "pycore_token.h"

#include "lexer_internal.h"
#include "../tokenizer/helpers.h"

#define MAKE_TOKEN(token_type) _PyLexer_token_setup(tok, token, token_type, p_start, p_end)

static int
lookahead(struct tok_state *tok, const char *test)
{
    const char *s = test;
    int res = 0;
    while (1) {
        int c = tok_nextc(tok);
        if (*s == 0) {
            res = !is_potential_identifier_char(c);
        }
        else if (c == *s) {
            s++;
            continue;
        }

        tok_backup(tok, c);
        while (s != test) {
            tok_backup(tok, *--s);
        }
        return res;
    }
}

static int
verify_end_of_number(struct tok_state *tok, int c, const char *kind) {
    if (tok->tok_extra_tokens) {
        // When we are parsing extra tokens, we don't want to emit warnings
        // about invalid literals, because we want to be a bit more liberal.
        return 1;
    }
    /* Emit a deprecation warning only if the numeric literal is immediately
     * followed by one of keywords which can occur after a numeric literal
     * in valid code: "and", "else", "for", "if", "in", "is" and "or".
     * It allows to gradually deprecate existing valid code without adding
     * warning before error in most cases of invalid numeric literal (which
     * would be confusing and break existing tests).
     * Raise a syntax error with slightly better message than plain
     * "invalid syntax" if the numeric literal is immediately followed by
     * other keyword or identifier.
     */
    int r = 0;
    if (c == 'a') {
        r = lookahead(tok, "nd");
    }
    else if (c == 'e') {
        r = lookahead(tok, "lse");
    }
    else if (c == 'f') {
        r = lookahead(tok, "or");
    }
    else if (c == 'i') {
        int c2 = tok_nextc(tok);
        if (c2 == 'f' || c2 == 'n' || c2 == 's') {
            r = 1;
        }
        tok_backup(tok, c2);
    }
    else if (c == 'o') {
        r = lookahead(tok, "r");
    }
    else if (c == 'n') {
        r = lookahead(tok, "ot");
    }
    if (r) {
        tok_backup(tok, c);
        if (_PyTokenizer_parser_warn(tok, PyExc_SyntaxWarning,
                "invalid %s literal", kind))
        {
            return 0;
        }
        tok_nextc(tok);
    }
    else /* In future releases, only error will remain. */
    if (c < 128 && is_potential_identifier_char(c)) {
        tok_backup(tok, c);
        _PyTokenizer_syntaxerror(tok, "invalid %s literal", kind);
        return 0;
    }
    return 1;
}

static int
tok_decimal_tail(struct tok_state *tok)
{
    int c;

    while (1) {
        do {
            c = tok_nextc(tok);
        } while (Py_ISDIGIT(c));
        if (c != '_') {
            break;
        }
        c = tok_nextc(tok);
        if (!Py_ISDIGIT(c)) {
            tok_backup(tok, c);
            _PyTokenizer_syntaxerror(tok, "invalid decimal literal");
            return 0;
        }
    }
    return c;
}

int
_PyLexer_scan_number(struct tok_state *tok, struct token *token, int c,
                     int leading_dot)
{
    _PyTok_Off p_start = -1;
    _PyTok_Off p_end = -1;

    if (leading_dot) {
        goto fraction;
    }
    if (c == '0') {
        /* Hex, octal or binary -- maybe. */
        c = tok_nextc(tok);
        if (c == 'x' || c == 'X') {
            /* Hex */
            c = tok_nextc(tok);
            do {
                if (c == '_') {
                    c = tok_nextc(tok);
                }
                if (!Py_ISXDIGIT(c)) {
                    tok_backup(tok, c);
                    return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid hexadecimal literal"));
                }
                do {
                    c = tok_nextc(tok);
                } while (Py_ISXDIGIT(c));
            } while (c == '_');
            if (!verify_end_of_number(tok, c, "hexadecimal")) {
                return MAKE_TOKEN(ERRORTOKEN);
            }
        }
        else if (c == 'o' || c == 'O') {
            /* Octal */
            c = tok_nextc(tok);
            do {
                if (c == '_') {
                    c = tok_nextc(tok);
                }
                if (c < '0' || c >= '8') {
                    if (Py_ISDIGIT(c)) {
                        return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok,
                                "invalid digit '%c' in octal literal", c));
                    }
                    else {
                        tok_backup(tok, c);
                        return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid octal literal"));
                    }
                }
                do {
                    c = tok_nextc(tok);
                } while ('0' <= c && c < '8');
            } while (c == '_');
            if (Py_ISDIGIT(c)) {
                return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok,
                        "invalid digit '%c' in octal literal", c));
            }
            if (!verify_end_of_number(tok, c, "octal")) {
                return MAKE_TOKEN(ERRORTOKEN);
            }
        }
        else if (c == 'b' || c == 'B') {
            /* Binary */
            c = tok_nextc(tok);
            do {
                if (c == '_') {
                    c = tok_nextc(tok);
                }
                if (c != '0' && c != '1') {
                    if (Py_ISDIGIT(c)) {
                        return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid digit '%c' in binary literal", c));
                    }
                    else {
                        tok_backup(tok, c);
                        return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid binary literal"));
                    }
                }
                do {
                    c = tok_nextc(tok);
                } while (c == '0' || c == '1');
            } while (c == '_');
            if (Py_ISDIGIT(c)) {
                return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid digit '%c' in binary literal", c));
            }
            if (!verify_end_of_number(tok, c, "binary")) {
                return MAKE_TOKEN(ERRORTOKEN);
            }
        }
        else {
            int nonzero = 0;
            /* maybe old-style octal; c is first char of it */
            /* in any case, allow '0' as a literal */
            while (1) {
                if (c == '_') {
                    c = tok_nextc(tok);
                    if (!Py_ISDIGIT(c)) {
                        tok_backup(tok, c);
                        return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid decimal literal"));
                    }
                }
                if (c != '0') {
                    break;
                }
                c = tok_nextc(tok);
            }
            _PyTok_Off zeros_end = tok->cur;
            if (Py_ISDIGIT(c)) {
                nonzero = 1;
                c = tok_decimal_tail(tok);
                if (c == 0) {
                    return MAKE_TOKEN(ERRORTOKEN);
                }
            }
            if (c == '.') {
                c = tok_nextc(tok);
                goto fraction;
            }
            else if (c == 'e' || c == 'E') {
                goto exponent;
            }
            else if (c == 'j' || c == 'J') {
                goto imaginary;
            }
            else if (nonzero && !tok->tok_extra_tokens) {
                /* Old-style octal: now disallowed. */
                tok_backup(tok, c);
                return MAKE_TOKEN(_PyTokenizer_syntaxerror_known_range(
                        tok, (int)(tok->start + 1 - tok->line_start),
                        (int)(zeros_end - tok->line_start),
                        "leading zeros in decimal integer "
                        "literals are not permitted; "
                        "use an 0o prefix for octal integers"));
            }
            if (!verify_end_of_number(tok, c, "decimal")) {
                return MAKE_TOKEN(ERRORTOKEN);
            }
        }
    }
    else {
        /* Decimal */
        c = tok_decimal_tail(tok);
        if (c == 0) {
            return MAKE_TOKEN(ERRORTOKEN);
        }
        {
            /* Accept floating-point numbers. */
            if (c == '.') {
                c = tok_nextc(tok);
    fraction:
                /* Fraction */
                if (Py_ISDIGIT(c)) {
                    c = tok_decimal_tail(tok);
                    if (c == 0) {
                        return MAKE_TOKEN(ERRORTOKEN);
                    }
                }
            }
            if (c == 'e' || c == 'E') {
                int e;
              exponent:
                e = c;
                /* Exponent part */
                c = tok_nextc(tok);
                if (c == '+' || c == '-') {
                    c = tok_nextc(tok);
                    if (!Py_ISDIGIT(c)) {
                        tok_backup(tok, c);
                        return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid decimal literal"));
                    }
                } else if (!Py_ISDIGIT(c)) {
                    tok_backup(tok, c);
                    if (!verify_end_of_number(tok, e, "decimal")) {
                        return MAKE_TOKEN(ERRORTOKEN);
                    }
                    tok_backup(tok, e);
                    p_start = tok->start;
                    p_end = tok->cur;
                    return MAKE_TOKEN(NUMBER);
                }
                c = tok_decimal_tail(tok);
                if (c == 0) {
                    return MAKE_TOKEN(ERRORTOKEN);
                }
            }
            if (c == 'j' || c == 'J') {
                /* Imaginary part */
    imaginary:
                c = tok_nextc(tok);
                if (!verify_end_of_number(tok, c, "imaginary")) {
                    return MAKE_TOKEN(ERRORTOKEN);
                }
            }
            else if (!verify_end_of_number(tok, c, "decimal")) {
                return MAKE_TOKEN(ERRORTOKEN);
            }
        }
    }
    tok_backup(tok, c);
    p_start = tok->start;
    p_end = tok->cur;
    return MAKE_TOKEN(NUMBER);
}

Coverage Report

Created: 2026-10-31 06:21

Line	Count	Source
1		#include "Python.h"
2		#include "pycore_token.h"
3
4		#include "lexer_internal.h"
5		#include "../tokenizer/helpers.h"
6
7	107k	#define MAKE_TOKEN(token_type) _PyLexer_token_setup(tok, token, token_type, p_start, p_end)
8
9		static int
10		lookahead(struct tok_state tok, const char test)
11	8.90k	{
12	8.90k	const char *s = test;
13	8.90k	int res = 0;
14	23.4k	while (1) {
15	23.4k	int c = tok_nextc(tok);
16	23.4k	if (*s == 0) {
17	8.78k	res = !is_potential_identifier_char(c);
18	8.78k	}
19	14.6k	else if (c == *s) {
20	14.5k	s++;
21	14.5k	continue;
22	14.5k	}
23
24	8.90k	tok_backup(tok, c);
25	23.4k	while (s != test) {
26	14.5k	tok_backup(tok, *--s);
27	14.5k	}
28	8.90k	return res;
29	23.4k	}
30	8.90k	}
31
32		static int
33	107k	verify_end_of_number(struct tok_state tok, int c, const char kind) {
34	107k	if (tok->tok_extra_tokens) {
35		// When we are parsing extra tokens, we don't want to emit warnings
36		// about invalid literals, because we want to be a bit more liberal.
37	56	return 1;
38	56	}
39		/* Emit a deprecation warning only if the numeric literal is immediately
40		* followed by one of keywords which can occur after a numeric literal
41		* in valid code: "and", "else", "for", "if", "in", "is" and "or".
42		* It allows to gradually deprecate existing valid code without adding
43		* warning before error in most cases of invalid numeric literal (which
44		* would be confusing and break existing tests).
45		* Raise a syntax error with slightly better message than plain
46		* "invalid syntax" if the numeric literal is immediately followed by
47		* other keyword or identifier.
48		*/
49	107k	int r = 0;
50	107k	if (c == 'a') {
51	1.03k	r = lookahead(tok, "nd");
52	1.03k	}
53	106k	else if (c == 'e') {
54	932	r = lookahead(tok, "lse");
55	932	}
56	105k	else if (c == 'f') {
57	2.67k	r = lookahead(tok, "or");
58	2.67k	}
59	102k	else if (c == 'i') {
60	1.62k	int c2 = tok_nextc(tok);
61	1.62k	if (c2 == 'f' \|\| c2 == 'n' \|\| c2 == 's') {
62	1.60k	r = 1;
63	1.60k	}
64	1.62k	tok_backup(tok, c2);
65	1.62k	}
66	101k	else if (c == 'o') {
67	3.94k	r = lookahead(tok, "r");
68	3.94k	}
69	97.1k	else if (c == 'n') {
70	316	r = lookahead(tok, "ot");
71	316	}
72	107k	if (r) {
73	10.3k	tok_backup(tok, c);
74	10.3k	if (_PyTokenizer_parser_warn(tok, PyExc_SyntaxWarning,
75	10.3k	"invalid %s literal", kind))
76	0	{
77	0	return 0;
78	0	}
79	10.3k	tok_nextc(tok);
80	10.3k	}
81	96.9k	else /* In future releases, only error will remain. */
82	96.9k	if (c < 128 && is_potential_identifier_char(c)) {
83	274	tok_backup(tok, c);
84	274	_PyTokenizer_syntaxerror(tok, "invalid %s literal", kind);
85	274	return 0;
86	274	}
87	107k	return 1;
88	107k	}
89
90		static int
91		tok_decimal_tail(struct tok_state *tok)
92	92.3k	{
93	92.3k	int c;
94
95	92.9k	while (1) {
96	321k	do {
97	321k	c = tok_nextc(tok);
98	321k	} while (Py_ISDIGIT(c));
99	92.9k	if (c != '_') {
100	92.3k	break;
101	92.3k	}
102	540	c = tok_nextc(tok);
103	540	if (!Py_ISDIGIT(c)) {
104	15	tok_backup(tok, c);
105	15	_PyTokenizer_syntaxerror(tok, "invalid decimal literal");
106	15	return 0;
107	15	}
108	540	}
109	92.3k	return c;
110	92.3k	}
111
112		int
113		_PyLexer_scan_number(struct tok_state tok, struct token token, int c,
114		int leading_dot)
115	107k	{
116	107k	_PyTok_Off p_start = -1;
117	107k	_PyTok_Off p_end = -1;
118
119	107k	if (leading_dot) {
120	4.18k	goto fraction;
121	4.18k	}
122	103k	if (c == '0') {
123		/* Hex, octal or binary -- maybe. */
124	32.8k	c = tok_nextc(tok);
125	32.8k	if (c == 'x' \|\| c == 'X') {
126		/* Hex */
127	15.7k	c = tok_nextc(tok);
128	15.9k	do {
129	15.9k	if (c == '_') {
130	203	c = tok_nextc(tok);
131	203	}
132	15.9k	if (!Py_ISXDIGIT(c)) {
133	17	tok_backup(tok, c);
134	17	return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid hexadecimal literal"));
135	17	}
136	78.8k	do {
137	78.8k	c = tok_nextc(tok);
138	78.8k	} while (Py_ISXDIGIT(c));
139	15.9k	} while (c == '_');
140	15.7k	if (!verify_end_of_number(tok, c, "hexadecimal")) {
141	1	return MAKE_TOKEN(ERRORTOKEN);
142	1	}
143	15.7k	}
144	17.1k	else if (c == 'o' \|\| c == 'O') {
145		/* Octal */
146	1.12k	c = tok_nextc(tok);
147	3.16k	do {
148	3.16k	if (c == '_') {
149	2.04k	c = tok_nextc(tok);
150	2.04k	}
151	3.16k	if (c < '0' \|\| c >= '8') {
152	20	if (Py_ISDIGIT(c)) {
153	1	return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok,
154	1	"invalid digit '%c' in octal literal", c));
155	1	}
156	19	else {
157	19	tok_backup(tok, c);
158	19	return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid octal literal"));
159	19	}
160	20	}
161	20.4k	do {
162	20.4k	c = tok_nextc(tok);
163	20.4k	} while ('0' <= c && c < '8');
164	3.14k	} while (c == '_');
165	1.10k	if (Py_ISDIGIT(c)) {
166	1	return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok,
167	1	"invalid digit '%c' in octal literal", c));
168	1	}
169	1.09k	if (!verify_end_of_number(tok, c, "octal")) {
170	4	return MAKE_TOKEN(ERRORTOKEN);
171	4	}
172	1.09k	}
173	16.0k	else if (c == 'b' \|\| c == 'B') {
174		/* Binary */
175	711	c = tok_nextc(tok);
176	1.07k	do {
177	1.07k	if (c == '_') {
178	370	c = tok_nextc(tok);
179	370	}
180	1.07k	if (c != '0' && c != '1') {
181	27	if (Py_ISDIGIT(c)) {
182	2	return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid digit '%c' in binary literal", c));
183	2	}
184	25	else {
185	25	tok_backup(tok, c);
186	25	return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid binary literal"));
187	25	}
188	27	}
189	7.88k	do {
190	7.88k	c = tok_nextc(tok);
191	7.88k	} while (c == '0' \|\| c == '1');
192	1.04k	} while (c == '_');
193	684	if (Py_ISDIGIT(c)) {
194	1	return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid digit '%c' in binary literal", c));
195	1	}
196	683	if (!verify_end_of_number(tok, c, "binary")) {
197	2	return MAKE_TOKEN(ERRORTOKEN);
198	2	}
199	683	}
200	15.3k	else {
201	15.3k	int nonzero = 0;
202		/* maybe old-style octal; c is first char of it */
203		/* in any case, allow '0' as a literal */
204	17.7k	while (1) {
205	17.7k	if (c == '_') {
206	323	c = tok_nextc(tok);
207	323	if (!Py_ISDIGIT(c)) {
208	4	tok_backup(tok, c);
209	4	return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid decimal literal"));
210	4	}
211	323	}
212	17.6k	if (c != '0') {
213	15.3k	break;
214	15.3k	}
215	2.37k	c = tok_nextc(tok);
216	2.37k	}
217	15.3k	_PyTok_Off zeros_end = tok->cur;
218	15.3k	if (Py_ISDIGIT(c)) {
219	784	nonzero = 1;
220	784	c = tok_decimal_tail(tok);
221	784	if (c == 0) {
222	1	return MAKE_TOKEN(ERRORTOKEN);
223	1	}
224	784	}
225	15.3k	if (c == '.') {
226	1.18k	c = tok_nextc(tok);
227	1.18k	goto fraction;
228	1.18k	}
229	14.1k	else if (c == 'e' \|\| c == 'E') {
230	949	goto exponent;
231	949	}
232	13.1k	else if (c == 'j' \|\| c == 'J') {
233	1.04k	goto imaginary;
234	1.04k	}
235	12.1k	else if (nonzero && !tok->tok_extra_tokens) {
236		/* Old-style octal: now disallowed. */
237	49	tok_backup(tok, c);
238	49	return MAKE_TOKEN(_PyTokenizer_syntaxerror_known_range(
239	49	tok, (int)(tok->start + 1 - tok->line_start),
240	49	(int)(zeros_end - tok->line_start),
241	49	"leading zeros in decimal integer "
242	49	"literals are not permitted; "
243	49	"use an 0o prefix for octal integers"));
244	49	}
245	12.0k	if (!verify_end_of_number(tok, c, "decimal")) {
246	30	return MAKE_TOKEN(ERRORTOKEN);
247	30	}
248	12.0k	}
249	32.8k	}
250	70.4k	else {
251		/* Decimal */
252	70.4k	c = tok_decimal_tail(tok);
253	70.4k	if (c == 0) {
254	11	return MAKE_TOKEN(ERRORTOKEN);
255	11	}
256	70.4k	{
257		/* Accept floating-point numbers. */
258	70.4k	if (c == '.') {
259	5.08k	c = tok_nextc(tok);
260	10.4k	fraction:
261		/* Fraction */
262	10.4k	if (Py_ISDIGIT(c)) {
263	7.60k	c = tok_decimal_tail(tok);
264	7.60k	if (c == 0) {
265	2	return MAKE_TOKEN(ERRORTOKEN);
266	2	}
267	7.60k	}
268	10.4k	}
269	75.8k	if (c == 'e' \|\| c == 'E') {
270	13.5k	int e;
271	14.4k	exponent:
272	14.4k	e = c;
273		/* Exponent part */
274	14.4k	c = tok_nextc(tok);
275	14.4k	if (c == '+' \|\| c == '-') {
276	6.38k	c = tok_nextc(tok);
277	6.38k	if (!Py_ISDIGIT(c)) {
278	11	tok_backup(tok, c);
279	11	return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid decimal literal"));
280	11	}
281	8.09k	} else if (!Py_ISDIGIT(c)) {
282	920	tok_backup(tok, c);
283	920	if (!verify_end_of_number(tok, e, "decimal")) {
284	59	return MAKE_TOKEN(ERRORTOKEN);
285	59	}
286	861	tok_backup(tok, e);
287	861	p_start = tok->start;
288	861	p_end = tok->cur;
289	861	return MAKE_TOKEN(NUMBER);
290	920	}
291	13.5k	c = tok_decimal_tail(tok);
292	13.5k	if (c == 0) {
293	1	return MAKE_TOKEN(ERRORTOKEN);
294	1	}
295	13.5k	}
296	75.8k	if (c == 'j' \|\| c == 'J') {
297		/* Imaginary part */
298	5.41k	imaginary:
299	5.41k	c = tok_nextc(tok);
300	5.41k	if (!verify_end_of_number(tok, c, "imaginary")) {
301	15	return MAKE_TOKEN(ERRORTOKEN);
302	15	}
303	5.41k	}
304	71.4k	else if (!verify_end_of_number(tok, c, "decimal")) {
305	163	return MAKE_TOKEN(ERRORTOKEN);
306	163	}
307	75.8k	}
308	75.8k	}
309	106k	tok_backup(tok, c);
310	106k	p_start = tok->start;
311	106k	p_end = tok->cur;
312	106k	return MAKE_TOKEN(NUMBER);
313	103k	}