/src/cpython/Parser/lexer/lexer.c

Source
#include "Python.h"
#include "pycore_token.h"
#include "pycore_unicodeobject.h"
#include "errcode.h"

#include "lexer_internal.h"
#include "../tokenizer/helpers.h"
#include "../tokenizer/reader.h"

#define MAKE_TOKEN(token_type) _PyLexer_token_setup(tok, token, token_type, p_start, p_end)

/* Spaces in this constant are treated as "zero or more spaces or tabs" when
   tokenizing. */
static const char* type_comment_prefix = "# type: ";

int
_PyLexer_refill(struct tok_state *tok)
{
    if (tok->done != E_OK) {
        return 0;
    }
    int rc = _PyTok_ReaderUnderflow(tok);
#if defined(Py_DEBUG)
    if (tok->debug) {
        fprintf(stderr, "line[%d] = ", tok->lineno);
        _PyTokenizer_print_escape(
            stderr, _PyTok_SourcePointer(&tok->source, tok->cur),
            tok->inp - tok->cur);
        fprintf(stderr, "  tok->done = %d\n", tok->done);
    }
#endif
    if (!rc) {
        tok->cur = tok->inp;
        return 0;
    }
    tok->line_start = tok->cur;
    _PyTok_Span line = {tok->line_start, tok->inp};
    if (_PyTok_SourceFindByte(&tok->source, line, '\0') >= 0) {
        _PyTokenizer_syntaxerror(tok, "source code cannot contain null bytes");
        tok->cur = tok->inp;
        return 0;
    }
    return 1;
}

/* Back-up one character */
void
_PyLexer_backup(struct tok_state *tok, int c)
{
    if (c != EOF) {
        if (--tok->cur < tok->buf_offset) {
            Py_FatalError("tokenizer beginning of buffer");
        }
        if (_PyTok_SourceByte(&tok->source, tok->cur) != Py_CHARMASK(c)) {
            Py_FatalError("tok_backup: wrong character");
        }
    }
}



/* Verify that the identifier follows PEP 3131. */
static int
verify_identifier(struct tok_state *tok)
{
    if (tok->tok_extra_tokens) {
        return 1;
    }
    PyObject *s;
    if (tok_failed(tok))
        return 0;
    s = PyUnicode_DecodeUTF8(_PyTok_SourcePointer(&tok->source, tok->start),
                             tok->cur - tok->start, NULL);
    if (s == NULL) {
        if (PyErr_ExceptionMatches(PyExc_UnicodeDecodeError)) {
            tok->done = E_DECODE;
        }
        else {
            tok->done = E_ERROR;
        }
        return 0;
    }
    Py_ssize_t invalid = _PyUnicode_ScanIdentifier(s);
    assert(invalid >= 0);
    assert(PyUnicode_GET_LENGTH(s) > 0);
    if (invalid < PyUnicode_GET_LENGTH(s)) {
        Py_UCS4 ch = PyUnicode_READ_CHAR(s, invalid);
        _PyTok_Off error_cursor = tok->cur;
        if (invalid + 1 < PyUnicode_GET_LENGTH(s)) {
            /* Determine the offset in UTF-8 encoded input */
            Py_SETREF(s, PyUnicode_Substring(s, 0, invalid + 1));
            if (s != NULL) {
                Py_SETREF(s, PyUnicode_AsUTF8String(s));
            }
            if (s == NULL) {
                tok->done = E_ERROR;
                return 0;
            }
            error_cursor = tok->start + PyBytes_GET_SIZE(s);
        }
        Py_DECREF(s);
        if (Py_UNICODE_ISPRINTABLE(ch)) {
            _PyTokenizer_syntaxerror_at(
                tok, _PyTok_SourcePointer(&tok->source, tok->line_start),
                error_cursor - tok->line_start, tok->lineno, -1, -1,
                "invalid character '%c' (U+%04X)", ch, ch);
        }
        else {
            _PyTokenizer_syntaxerror_at(
                tok, _PyTok_SourcePointer(&tok->source, tok->line_start),
                error_cursor - tok->line_start, tok->lineno, -1, -1,
                "invalid non-printable character U+%04X", ch);
        }
        return 0;
    }
    Py_DECREF(s);
    return 1;
}

int
_PyLexer_get_normal(struct tok_state *tok, ftstring_state *current, struct token *token)
{
    assert(current == NULL ||
           (current->mode == FTSTRING_MODE_EXPRESSION &&
            current->replacement_depth > 0));
    int c;
    int blankline, nonascii;

    _PyTok_Off p_start = -1;
    _PyTok_Off p_end = -1;
  nextline:
    tok->start = -1;
    tok->start_loc = (_PyTok_Loc){tok->lineno, -1};
    blankline = 0;


    if (tok->layout.at_bol) {
        blankline = _PyLexer_BeginLine(tok);
        if (blankline < 0) {
            return MAKE_TOKEN(ERRORTOKEN);
        }
    }

    tok->start = tok->cur;
    tok->start_loc = (_PyTok_Loc){
        tok->lineno, tok->line_start >= 0 ? _PyLexer_ByteColumn(tok) : -1};

    if (tok->layout.pending != 0) {
        return _PyLexer_IndentationToken(tok, token);
    }

    /* Peek ahead at the next character */
    c = tok_nextc(tok);
    tok_backup(tok, c);

 again:
    tok->start = -1;
    /* Skip spaces */
    do {
        c = tok_nextc(tok);
    } while (c == ' ' || c == '\t' || c == '\014');

    /* Set start of current token */
    tok->start = tok->cur - 1;
    tok->start_loc = (_PyTok_Loc){
        tok->lineno, tok->line_start >= 0 ? _PyLexer_ByteColumn(tok) - 1 : -1};

    /* Skip comment, unless it's a type comment */
    if (c == '#') {

        while (c != EOF && c != '\n' && c != '\r') {
            c = tok_nextc(tok);
        }

        if (current != NULL) {
            _PyTok_Off comment_end = tok->cur;
            if (c == '\n' || c == '\r') {
                comment_end--;
            }
            if (_PyLexer_record_ftstring_comment(
                    tok, current, tok->start, comment_end) < 0) {
                tok->done = E_NOMEM;
                return MAKE_TOKEN(ERRORTOKEN);
            }
        }

        _PyTok_Off comment = tok->start;
        if (tok->type_comments) {
            const char *prefix = type_comment_prefix;
            while (*prefix && comment < tok->cur) {
                if (*prefix == ' ') {
                    while (comment < tok->cur) {
                        int ch = _PyTok_SourceByte(&tok->source, comment);
                        if (ch != ' ' && ch != '\t') {
                            break;
                        }
                        comment++;
                    }
                } else if (*prefix == _PyTok_SourceByte(&tok->source, comment)) {
                    comment++;
                } else {
                    break;
                }

                prefix++;
            }

            /* This is a type comment if we matched all of type_comment_prefix. */
            if (!*prefix) {
                tok_backup(tok, c);  /* don't eat the newline or EOF */
                /* A TYPE_IGNORE is "type: ignore" followed by the end of the token
                 * or anything ASCII and non-alphanumeric. */
                int is_type_ignore = tok->cur - comment >= 6 &&
                    memcmp(_PyTok_SourcePointer(&tok->source, comment),
                           "ignore", 6) == 0;
                if (is_type_ignore && comment + 6 < tok->cur) {
                    int ch = _PyTok_SourceByte(&tok->source, comment + 6);
                    is_type_ignore = ch < 128 && !Py_ISALNUM(ch);
                }

                int type = is_type_ignore ? TYPE_IGNORE : TYPE_COMMENT;
                p_start = comment + (is_type_ignore ? 6 : 0);
                p_end = tok->cur;
                int start_col_offset = tok->start_loc.byte_col +
                    (int)(p_start - tok->start);
                /* Consume the newline after a standalone type ignore. */
                if (is_type_ignore && blankline) {
                    tok_nextc(tok);
                    tok->layout.at_bol = 1;
                }
                _PyLexer_token_setup(tok, token, type, p_start, p_end);
                token->start_loc = (_PyTok_Loc){tok->lineno, start_col_offset};
                token->end_loc = (_PyTok_Loc){tok->lineno,
                                              _PyLexer_ByteColumn(tok)};
                return type;
            }
        }
        if (tok->implicit_newline == _PyTokenizer_IMPLY_NEWLINE &&
            tok->layout.depth > 0) {
            // Ending this comment lets the next line dedent the suite.
            tok->needs_newline |= _PyTokenizer_NEWLINE_STATEMENT;
        }
        if (tok->tok_extra_tokens) {
            tok_backup(tok, c);  /* don't eat the newline or EOF */
            p_start = comment;
            p_end = tok->cur;
            tok->layout.comment_newline = blankline;
            return MAKE_TOKEN(COMMENT);
        }
    }

    if (tok->done == E_INTERACT_STOP) {
        return MAKE_TOKEN(ENDMARKER);
    }

    /* Check for EOF and errors now */
    if (c == EOF) {
        if (tok->level) {
            return MAKE_TOKEN(ERRORTOKEN);
        }
        return MAKE_TOKEN(tok->done == E_EOF ? ENDMARKER : ERRORTOKEN);
    }

    /* Identifier (most frequent token!) */
    nonascii = 0;
    if (is_potential_identifier_start(c)) {
        /* Process the various legal combinations of b"", r"", u"", and f"". */
        int saw_b = 0, saw_r = 0, saw_u = 0, saw_f = 0, saw_t = 0;
        while (1) {
            if (!saw_b && (c == 'b' || c == 'B')) {
                saw_b = 1;
            }
            /* Since this is a backwards compatibility support literal we don't
               want to support it in arbitrary order like byte literals. */
            else if (!saw_u && (c == 'u'|| c == 'U')) {
                saw_u = 1;
            }
            /* ur"" and ru"" are not supported */
            else if (!saw_r && (c == 'r' || c == 'R')) {
                saw_r = 1;
            }
            else if (!saw_f && (c == 'f' || c == 'F')) {
                saw_f = 1;
            }
            else if (!saw_t && (c == 't' || c == 'T')) {
                saw_t = 1;
            }
            else {
                break;
            }
            c = tok_nextc(tok);
            if (c == '"' || c == '\'') {
                // Raise error on incompatible string prefixes:
                int status = _PyLexer_check_string_prefixes(
                    tok, saw_b, saw_r, saw_u, saw_f, saw_t);
                if (status < 0) {
                    return MAKE_TOKEN(ERRORTOKEN);
                }

                // Handle valid f or t string creation:
                if (saw_f || saw_t) {
                    ftstring_kind kind = saw_t
                        ? (saw_r ? RAW_TSTRING : TSTRING)
                        : (saw_r ? RAW_FSTRING : FSTRING);
                    return _PyLexer_scan_fstring_start(tok, token, c, kind);
                }
                return _PyLexer_scan_string(tok, token, c);
            }
        }
        while (is_potential_identifier_char(c)) {
            if (c >= 128) {
                nonascii = 1;
            }
            c = tok_nextc(tok);
        }
        tok_backup(tok, c);
        if (nonascii && !verify_identifier(tok)) {
            return MAKE_TOKEN(ERRORTOKEN);
        }

        p_start = tok->start;
        p_end = tok->cur;

        return MAKE_TOKEN(NAME);
    }

    if (c == '\r') {
        c = tok_nextc(tok);
    }

    if (c == '\n') {
        int type = _PyLexer_Newline(tok, token, blankline);
        if (type == 0) {
            goto nextline;
        }
        return type;
    }

    /* Period or number starting with period? */
    if (c == '.') {
        c = tok_nextc(tok);
        if (Py_ISDIGIT(c)) {
            return _PyLexer_scan_number(tok, token, c, 1);
        } else if (c == '.') {
            c = tok_nextc(tok);
            if (c == '.') {
                p_start = tok->start;
                p_end = tok->cur;
                return MAKE_TOKEN(ELLIPSIS);
            }
            else {
                tok_backup(tok, c);
            }
            tok_backup(tok, '.');
        }
        else {
            tok_backup(tok, c);
        }
        p_start = tok->start;
        p_end = tok->cur;
        return MAKE_TOKEN(DOT);
    }


    /* Number */
    if (Py_ISDIGIT(c)) {
        return _PyLexer_scan_number(tok, token, c, 0);
    }

    /* String */
    if (c == '\'' || c == '"') {
        return _PyLexer_scan_string(tok, token, c);
    }

    /* Line continuation */
    if (c == '\\') {
        if ((c = _PyLexer_ContinueLine(tok)) == -1) {
            return MAKE_TOKEN(ERRORTOKEN);
        }
        goto again; /* Read next line */
    }

    /* Punctuation character */
    int is_punctuation = (c == ':' || c == '}' || c == '!');
    if (is_punctuation && current != NULL) {
        int type = _PyLexer_ftstring_punctuation(tok, current, token, c);
        if (type < 0) {
            return MAKE_TOKEN(ERRORTOKEN);
        }
        if (type != 0) {
            p_start = tok->start;
            p_end = tok->cur;
            return MAKE_TOKEN(type);
        }
    }

    /* Check for two-character token */
    {
        int c2 = tok_nextc(tok);
        int current_token = _PyToken_TwoChars(c, c2, tok->barry_as_bdfl);
        if (current_token != OP) {
            int c3 = tok_nextc(tok);
            int current_token3 = _PyToken_ThreeChars(c, c2, c3);
            if (current_token3 != OP) {
                current_token = current_token3;
            }
            else {
                tok_backup(tok, c3);
            }
            p_start = tok->start;
            p_end = tok->cur;
            return MAKE_TOKEN(current_token);
        }
        tok_backup(tok, c2);
    }

    /* Keep track of parentheses nesting level */
    switch (c) {
    case '(':
    case '[':
    case '{':
        if (tok->level >= MAXLEVEL) {
            return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "too many nested parentheses"));
        }
        tok->parenstack[tok->level] = c;
        tok->parenlinenostack[tok->level] = tok->lineno;
        tok->parencolstack[tok->level] = (int)(tok->start - tok->line_start);
        tok->level++;
        break;
    case ')':
    case ']':
    case '}':
        if (current != NULL &&
                _PyLexer_FTStringBracketDepth(tok, current) == 0) {
            if (c == '}') {
                return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok,
                    "%c-string: single '}' is not allowed",
                    _PyLexer_StringPrefix(current->kind)));
            }
            return MAKE_TOKEN(_PyTokenizer_syntaxerror(
                tok, "%c-string: unmatched '%c'",
                _PyLexer_StringPrefix(current->kind), c));
        }
        if (!tok->tok_extra_tokens && !tok->level) {
            return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "unmatched '%c'", c));
        }
        if (tok->level > 0) {
            tok->level--;
            int opening = tok->parenstack[tok->level];
            if (!((opening == '(' && c == ')') ||
                  (opening == '[' && c == ']') ||
                  (opening == '{' && c == '}'))) {
                /* Do not match a closer against the brace that opened the
                 * current replacement field.  The f-string state relies on
                 * that brace, so this is checked even with extra tokens. */
                if (current != NULL && opening == '{') {
                    int bracket_depth =
                        _PyLexer_FTStringBracketDepth(tok, current);
                    if (bracket_depth == current->replacement_depth - 1) {
                        return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok,
                            "%c-string: unmatched '%c'",
                            _PyLexer_StringPrefix(current->kind), c));
                    }
                }
                if (tok->tok_extra_tokens) {
                    /* Allow other mismatched brackets. */
                }
                else if (tok->parenlinenostack[tok->level] != tok->lineno) {
                    return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok,
                            "closing parenthesis '%c' does not match "
                            "opening parenthesis '%c' on line %d",
                            c, opening, tok->parenlinenostack[tok->level]));
                }
                else {
                    return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok,
                            "closing parenthesis '%c' does not match "
                            "opening parenthesis '%c'",
                            c, opening));
                }
            }
        }
        if (current != NULL &&
                _PyLexer_close_ftstring_expr(tok, current, c) < 0) {
            return MAKE_TOKEN(ERRORTOKEN);
        }
        break;
    default:
        break;
    }

    if (!Py_UNICODE_ISPRINTABLE(c)) {
        return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid non-printable character U+%04X", c));
    }

    if (c == '=' && current != NULL) {
        _PyLexer_mark_ftstring_debug(tok, current);
    }

    /* Punctuation character */
    p_start = tok->start;
    p_end = tok->cur;
    return MAKE_TOKEN(_PyToken_OneChar(c));
}


void
_PyTokenizer_Get(struct tok_state *tok, struct token *token)
{
    _PyToken_Free(token);
    ftstring_state *current = _PyLexer_CurrentFTString(tok);
    int result;
    if (current == NULL) {
        result = _PyLexer_get_normal(tok, NULL, token);
    }
    else {
        switch (current->mode) {
        case FTSTRING_MODE_EXPRESSION:
            result = _PyLexer_get_normal(tok, current, token);
            break;
        case FTSTRING_MODE_MIDDLE:
        case FTSTRING_MODE_FORMAT_SPEC:
            result = _PyLexer_get_ftstring(tok, current, token);
            break;
        default:
            Py_UNREACHABLE();
        }
    }
    if (tok_failed(tok) && tok->done != E_INCOMPLETE) {
        result = ERRORTOKEN;
    }
    token->type = result;
}

Coverage Report

Created: 2026-10-31 06:21

Line	Count	Source
1		#include "Python.h"
2		#include "pycore_token.h"
3		#include "pycore_unicodeobject.h"
4		#include "errcode.h"
5
6		#include "lexer_internal.h"
7		#include "../tokenizer/helpers.h"
8		#include "../tokenizer/reader.h"
9
10	1.64M	#define MAKE_TOKEN(token_type) _PyLexer_token_setup(tok, token, token_type, p_start, p_end)
11
12		/* Spaces in this constant are treated as "zero or more spaces or tabs" when
13		tokenizing. */
14		static const char* type_comment_prefix = "# type: ";
15
16		int
17		_PyLexer_refill(struct tok_state *tok)
18	484k	{
19	484k	if (tok->done != E_OK) {
20	141k	return 0;
21	141k	}
22	343k	int rc = _PyTok_ReaderUnderflow(tok);
23		#if defined(Py_DEBUG)
24		if (tok->debug) {
25		fprintf(stderr, "line[%d] = ", tok->lineno);
26		_PyTokenizer_print_escape(
27		stderr, _PyTok_SourcePointer(&tok->source, tok->cur),
28		tok->inp - tok->cur);
29		fprintf(stderr, " tok->done = %d\n", tok->done);
30		}
31		#endif
32	343k	if (!rc) {
33	70.7k	tok->cur = tok->inp;
34	70.7k	return 0;
35	70.7k	}
36	272k	tok->line_start = tok->cur;
37	272k	_PyTok_Span line = {tok->line_start, tok->inp};
38	272k	if (_PyTok_SourceFindByte(&tok->source, line, '\0') >= 0) {
39	88	_PyTokenizer_syntaxerror(tok, "source code cannot contain null bytes");
40	88	tok->cur = tok->inp;
41	88	return 0;
42	88	}
43	272k	return 1;
44	272k	}
45
46		/* Back-up one character */
47		void
48		_PyLexer_backup(struct tok_state *tok, int c)
49	4.25M	{
50	4.25M	if (c != EOF) {
51	4.11M	if (--tok->cur < tok->buf_offset) {
52	0	Py_FatalError("tokenizer beginning of buffer");
53	0	}
54	4.11M	if (_PyTok_SourceByte(&tok->source, tok->cur) != Py_CHARMASK(c)) {
55	0	Py_FatalError("tok_backup: wrong character");
56	0	}
57	4.11M	}
58	4.25M	}
59
60
61
62		/* Verify that the identifier follows PEP 3131. */
63		static int
64		verify_identifier(struct tok_state *tok)
65	12.9k	{
66	12.9k	if (tok->tok_extra_tokens) {
67	0	return 1;
68	0	}
69	12.9k	PyObject *s;
70	12.9k	if (tok_failed(tok))
71	0	return 0;
72	12.9k	s = PyUnicode_DecodeUTF8(_PyTok_SourcePointer(&tok->source, tok->start),
73	12.9k	tok->cur - tok->start, NULL);
74	12.9k	if (s == NULL) {
75	0	if (PyErr_ExceptionMatches(PyExc_UnicodeDecodeError)) {
76	0	tok->done = E_DECODE;
77	0	}
78	0	else {
79	0	tok->done = E_ERROR;
80	0	}
81	0	return 0;
82	0	}
83	12.9k	Py_ssize_t invalid = _PyUnicode_ScanIdentifier(s);
84	12.9k	assert(invalid >= 0);
85	12.9k	assert(PyUnicode_GET_LENGTH(s) > 0);
86	12.9k	if (invalid < PyUnicode_GET_LENGTH(s)) {
87	582	Py_UCS4 ch = PyUnicode_READ_CHAR(s, invalid);
88	582	_PyTok_Off error_cursor = tok->cur;
89	582	if (invalid + 1 < PyUnicode_GET_LENGTH(s)) {
90		/* Determine the offset in UTF-8 encoded input */
91	402	Py_SETREF(s, PyUnicode_Substring(s, 0, invalid + 1));
92	402	if (s != NULL) {
93	402	Py_SETREF(s, PyUnicode_AsUTF8String(s));
94	402	}
95	402	if (s == NULL) {
96	0	tok->done = E_ERROR;
97	0	return 0;
98	0	}
99	402	error_cursor = tok->start + PyBytes_GET_SIZE(s);
100	402	}
101	582	Py_DECREF(s);
102	582	if (Py_UNICODE_ISPRINTABLE(ch)) {
103	330	_PyTokenizer_syntaxerror_at(
104	330	tok, _PyTok_SourcePointer(&tok->source, tok->line_start),
105	330	error_cursor - tok->line_start, tok->lineno, -1, -1,
106	330	"invalid character '%c' (U+%04X)", ch, ch);
107	330	}
108	252	else {
109	252	_PyTokenizer_syntaxerror_at(
110	252	tok, _PyTok_SourcePointer(&tok->source, tok->line_start),
111	252	error_cursor - tok->line_start, tok->lineno, -1, -1,
112	252	"invalid non-printable character U+%04X", ch);
113	252	}
114	582	return 0;
115	582	}
116	12.3k	Py_DECREF(s);
117	12.3k	return 1;
118	12.9k	}
119
120		int
121		_PyLexer_get_normal(struct tok_state tok, ftstring_state current, struct token *token)
122	1.96M	{
123	1.96M	assert(current == NULL \|\|
124	1.96M	(current->mode == FTSTRING_MODE_EXPRESSION &&
125	1.96M	current->replacement_depth > 0));
126	1.96M	int c;
127	1.96M	int blankline, nonascii;
128
129	1.96M	_PyTok_Off p_start = -1;
130	1.96M	_PyTok_Off p_end = -1;
131	2.08M	nextline:
132	2.08M	tok->start = -1;
133	2.08M	tok->start_loc = (_PyTok_Loc){tok->lineno, -1};
134	2.08M	blankline = 0;
135
136
137	2.08M	if (tok->layout.at_bol) {
138	329k	blankline = _PyLexer_BeginLine(tok);
139	329k	if (blankline < 0) {
140	133	return MAKE_TOKEN(ERRORTOKEN);
141	133	}
142	329k	}
143
144	2.08M	tok->start = tok->cur;
145	2.08M	tok->start_loc = (_PyTok_Loc){
146	2.08M	tok->lineno, tok->line_start >= 0 ? _PyLexer_ByteColumn(tok) : -1};
147
148	2.08M	if (tok->layout.pending != 0) {
149	18.7k	return _PyLexer_IndentationToken(tok, token);
150	18.7k	}
151
152		/* Peek ahead at the next character */
153	2.06M	c = tok_nextc(tok);
154	2.06M	tok_backup(tok, c);
155
156	2.06M	again:
157	2.06M	tok->start = -1;
158		/* Skip spaces */
159	2.45M	do {
160	2.45M	c = tok_nextc(tok);
161	2.45M	} while (c == ' ' \|\| c == '\t' \|\| c == '\014');
162
163		/* Set start of current token */
164	2.06M	tok->start = tok->cur - 1;
165	2.06M	tok->start_loc = (_PyTok_Loc){
166	2.06M	tok->lineno, tok->line_start >= 0 ? _PyLexer_ByteColumn(tok) - 1 : -1};
167
168		/* Skip comment, unless it's a type comment */
169	2.06M	if (c == '#') {
170
171	1.10M	while (c != EOF && c != '\n' && c != '\r') {
172	1.06M	c = tok_nextc(tok);
173	1.06M	}
174
175	36.5k	if (current != NULL) {
176	1.53k	_PyTok_Off comment_end = tok->cur;
177	1.53k	if (c == '\n' \|\| c == '\r') {
178	1.53k	comment_end--;
179	1.53k	}
180	1.53k	if (_PyLexer_record_ftstring_comment(
181	1.53k	tok, current, tok->start, comment_end) < 0) {
182	0	tok->done = E_NOMEM;
183	0	return MAKE_TOKEN(ERRORTOKEN);
184	0	}
185	1.53k	}
186
187	36.5k	_PyTok_Off comment = tok->start;
188	36.5k	if (tok->type_comments) {
189	0	const char *prefix = type_comment_prefix;
190	0	while (*prefix && comment < tok->cur) {
191	0	if (*prefix == ' ') {
192	0	while (comment < tok->cur) {
193	0	int ch = _PyTok_SourceByte(&tok->source, comment);
194	0	if (ch != ' ' && ch != '\t') {
195	0	break;
196	0	}
197	0	comment++;
198	0	}
199	0	} else if (*prefix == _PyTok_SourceByte(&tok->source, comment)) {
200	0	comment++;
201	0	} else {
202	0	break;
203	0	}
204
205	0	prefix++;
206	0	}
207
208		/* This is a type comment if we matched all of type_comment_prefix. */
209	0	if (!*prefix) {
210	0	tok_backup(tok, c); /* don't eat the newline or EOF */
211		/* A TYPE_IGNORE is "type: ignore" followed by the end of the token
212		* or anything ASCII and non-alphanumeric. */
213	0	int is_type_ignore = tok->cur - comment >= 6 &&
214	0	memcmp(_PyTok_SourcePointer(&tok->source, comment),
215	0	"ignore", 6) == 0;
216	0	if (is_type_ignore && comment + 6 < tok->cur) {
217	0	int ch = _PyTok_SourceByte(&tok->source, comment + 6);
218	0	is_type_ignore = ch < 128 && !Py_ISALNUM(ch);
219	0	}
220
221	0	int type = is_type_ignore ? TYPE_IGNORE : TYPE_COMMENT;
222	0	p_start = comment + (is_type_ignore ? 6 : 0);
223	0	p_end = tok->cur;
224	0	int start_col_offset = tok->start_loc.byte_col +
225	0	(int)(p_start - tok->start);
226		/* Consume the newline after a standalone type ignore. */
227	0	if (is_type_ignore && blankline) {
228	0	tok_nextc(tok);
229	0	tok->layout.at_bol = 1;
230	0	}
231	0	_PyLexer_token_setup(tok, token, type, p_start, p_end);
232	0	token->start_loc = (_PyTok_Loc){tok->lineno, start_col_offset};
233	0	token->end_loc = (_PyTok_Loc){tok->lineno,
234	0	_PyLexer_ByteColumn(tok)};
235	0	return type;
236	0	}
237	0	}
238	36.5k	if (tok->implicit_newline == _PyTokenizer_IMPLY_NEWLINE &&
239	1.61k	tok->layout.depth > 0) {
240		// Ending this comment lets the next line dedent the suite.
241	51	tok->needs_newline \|= _PyTokenizer_NEWLINE_STATEMENT;
242	51	}
243	36.5k	if (tok->tok_extra_tokens) {
244	44	tok_backup(tok, c); /* don't eat the newline or EOF */
245	44	p_start = comment;
246	44	p_end = tok->cur;
247	44	tok->layout.comment_newline = blankline;
248	44	return MAKE_TOKEN(COMMENT);
249	44	}
250	36.5k	}
251
252	2.06M	if (tok->done == E_INTERACT_STOP) {
253	0	return MAKE_TOKEN(ENDMARKER);
254	0	}
255
256		/* Check for EOF and errors now */
257	2.06M	if (c == EOF) {
258	70.6k	if (tok->level) {
259	4.21k	return MAKE_TOKEN(ERRORTOKEN);
260	4.21k	}
261	66.4k	return MAKE_TOKEN(tok->done == E_EOF ? ENDMARKER : ERRORTOKEN);
262	70.6k	}
263
264		/* Identifier (most frequent token!) */
265	1.99M	nonascii = 0;
266	1.99M	if (is_potential_identifier_start(c)) {
267		/* Process the various legal combinations of b"", r"", u"", and f"". */
268	652k	int saw_b = 0, saw_r = 0, saw_u = 0, saw_f = 0, saw_t = 0;
269	812k	while (1) {
270	812k	if (!saw_b && (c == 'b' \|\| c == 'B')) {
271	17.2k	saw_b = 1;
272	17.2k	}
273		/* Since this is a backwards compatibility support literal we don't
274		want to support it in arbitrary order like byte literals. */
275	795k	else if (!saw_u && (c == 'u'\|\| c == 'U')) {
276	59.4k	saw_u = 1;
277	59.4k	}
278		/* ur"" and ru"" are not supported */
279	735k	else if (!saw_r && (c == 'r' \|\| c == 'R')) {
280	28.0k	saw_r = 1;
281	28.0k	}
282	707k	else if (!saw_f && (c == 'f' \|\| c == 'F')) {
283	47.3k	saw_f = 1;
284	47.3k	}
285	660k	else if (!saw_t && (c == 't' \|\| c == 'T')) {
286	34.5k	saw_t = 1;
287	34.5k	}
288	625k	else {
289	625k	break;
290	625k	}
291	186k	c = tok_nextc(tok);
292	186k	if (c == '"' \|\| c == '\'') {
293		// Raise error on incompatible string prefixes:
294	26.6k	int status = _PyLexer_check_string_prefixes(
295	26.6k	tok, saw_b, saw_r, saw_u, saw_f, saw_t);
296	26.6k	if (status < 0) {
297	8	return MAKE_TOKEN(ERRORTOKEN);
298	8	}
299
300		// Handle valid f or t string creation:
301	26.6k	if (saw_f \|\| saw_t) {
302	21.7k	ftstring_kind kind = saw_t
303	21.7k	? (saw_r ? RAW_TSTRING : TSTRING)
304	21.7k	: (saw_r ? RAW_FSTRING : FSTRING);
305	21.7k	return _PyLexer_scan_fstring_start(tok, token, c, kind);
306	21.7k	}
307	4.93k	return _PyLexer_scan_string(tok, token, c);
308	26.6k	}
309	186k	}
310	2.75M	while (is_potential_identifier_char(c)) {
311	2.12M	if (c >= 128) {
312	224k	nonascii = 1;
313	224k	}
314	2.12M	c = tok_nextc(tok);
315	2.12M	}
316	625k	tok_backup(tok, c);
317	625k	if (nonascii && !verify_identifier(tok)) {
318	582	return MAKE_TOKEN(ERRORTOKEN);
319	582	}
320
321	625k	p_start = tok->start;
322	625k	p_end = tok->cur;
323
324	625k	return MAKE_TOKEN(NAME);
325	625k	}
326
327	1.34M	if (c == '\r') {
328	0	c = tok_nextc(tok);
329	0	}
330
331	1.34M	if (c == '\n') {
332	256k	int type = _PyLexer_Newline(tok, token, blankline);
333	256k	if (type == 0) {
334	118k	goto nextline;
335	118k	}
336	137k	return type;
337	256k	}
338
339		/* Period or number starting with period? */
340	1.08M	if (c == '.') {
341	34.2k	c = tok_nextc(tok);
342	34.2k	if (Py_ISDIGIT(c)) {
343	4.18k	return _PyLexer_scan_number(tok, token, c, 1);
344	30.0k	} else if (c == '.') {
345	5.60k	c = tok_nextc(tok);
346	5.60k	if (c == '.') {
347	3.65k	p_start = tok->start;
348	3.65k	p_end = tok->cur;
349	3.65k	return MAKE_TOKEN(ELLIPSIS);
350	3.65k	}
351	1.94k	else {
352	1.94k	tok_backup(tok, c);
353	1.94k	}
354	1.94k	tok_backup(tok, '.');
355	1.94k	}
356	24.4k	else {
357	24.4k	tok_backup(tok, c);
358	24.4k	}
359	26.3k	p_start = tok->start;
360	26.3k	p_end = tok->cur;
361	26.3k	return MAKE_TOKEN(DOT);
362	34.2k	}
363
364
365		/* Number */
366	1.05M	if (Py_ISDIGIT(c)) {
367	103k	return _PyLexer_scan_number(tok, token, c, 0);
368	103k	}
369
370		/* String */
371	951k	if (c == '\'' \|\| c == '"') {
372	34.3k	return _PyLexer_scan_string(tok, token, c);
373	34.3k	}
374
375		/* Line continuation */
376	917k	if (c == '\\') {
377	371	if ((c = _PyLexer_ContinueLine(tok)) == -1) {
378	88	return MAKE_TOKEN(ERRORTOKEN);
379	88	}
380	283	goto again; /* Read next line */
381	371	}
382
383		/* Punctuation character */
384	917k	int is_punctuation = (c == ':' \|\| c == '}' \|\| c == '!');
385	917k	if (is_punctuation && current != NULL) {
386	33.5k	int type = _PyLexer_ftstring_punctuation(tok, current, token, c);
387	33.5k	if (type < 0) {
388	0	return MAKE_TOKEN(ERRORTOKEN);
389	0	}
390	33.5k	if (type != 0) {
391	5.26k	p_start = tok->start;
392	5.26k	p_end = tok->cur;
393	5.26k	return MAKE_TOKEN(type);
394	5.26k	}
395	33.5k	}
396
397		/* Check for two-character token */
398	911k	{
399	911k	int c2 = tok_nextc(tok);
400	911k	int current_token = _PyToken_TwoChars(c, c2, tok->barry_as_bdfl);
401	911k	if (current_token != OP) {
402	28.4k	int c3 = tok_nextc(tok);
403	28.4k	int current_token3 = _PyToken_ThreeChars(c, c2, c3);
404	28.4k	if (current_token3 != OP) {
405	1.31k	current_token = current_token3;
406	1.31k	}
407	27.1k	else {
408	27.1k	tok_backup(tok, c3);
409	27.1k	}
410	28.4k	p_start = tok->start;
411	28.4k	p_end = tok->cur;
412	28.4k	return MAKE_TOKEN(current_token);
413	28.4k	}
414	883k	tok_backup(tok, c2);
415	883k	}
416
417		/* Keep track of parentheses nesting level */
418	0	switch (c) {
419	67.9k	case '(':
420	104k	case '[':
421	160k	case '{':
422	160k	if (tok->level >= MAXLEVEL) {
423	15	return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "too many nested parentheses"));
424	15	}
425	160k	tok->parenstack[tok->level] = c;
426	160k	tok->parenlinenostack[tok->level] = tok->lineno;
427	160k	tok->parencolstack[tok->level] = (int)(tok->start - tok->line_start);
428	160k	tok->level++;
429	160k	break;
430	38.3k	case ')':
431	45.2k	case ']':
432	75.7k	case '}':
433	75.7k	if (current != NULL &&
434	26.8k	_PyLexer_FTStringBracketDepth(tok, current) == 0) {
435	0	if (c == '}') {
436	0	return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok,
437	0	"%c-string: single '}' is not allowed",
438	0	_PyLexer_StringPrefix(current->kind)));
439	0	}
440	0	return MAKE_TOKEN(_PyTokenizer_syntaxerror(
441	0	tok, "%c-string: unmatched '%c'",
442	0	_PyLexer_StringPrefix(current->kind), c));
443	0	}
444	75.7k	if (!tok->tok_extra_tokens && !tok->level) {
445	219	return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "unmatched '%c'", c));
446	219	}
447	75.5k	if (tok->level > 0) {
448	75.5k	tok->level--;
449	75.5k	int opening = tok->parenstack[tok->level];
450	75.5k	if (!((opening == '(' && c == ')') \|\|
451	37.3k	(opening == '[' && c == ']') \|\|
452	30.5k	(opening == '{' && c == '}'))) {
453		/* Do not match a closer against the brace that opened the
454		* current replacement field. The f-string state relies on
455		* that brace, so this is checked even with extra tokens. */
456	67	if (current != NULL && opening == '{') {
457	10	int bracket_depth =
458	10	_PyLexer_FTStringBracketDepth(tok, current);
459	10	if (bracket_depth == current->replacement_depth - 1) {
460	6	return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok,
461	6	"%c-string: unmatched '%c'",
462	6	_PyLexer_StringPrefix(current->kind), c));
463	6	}
464	10	}
465	61	if (tok->tok_extra_tokens) {
466		/* Allow other mismatched brackets. */
467	0	}
468	61	else if (tok->parenlinenostack[tok->level] != tok->lineno) {
469	3	return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok,
470	3	"closing parenthesis '%c' does not match "
471	3	"opening parenthesis '%c' on line %d",
472	3	c, opening, tok->parenlinenostack[tok->level]));
473	3	}
474	58	else {
475	58	return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok,
476	58	"closing parenthesis '%c' does not match "
477	58	"opening parenthesis '%c'",
478	58	c, opening));
479	58	}
480	61	}
481	75.5k	}
482	75.4k	if (current != NULL &&
483	26.8k	_PyLexer_close_ftstring_expr(tok, current, c) < 0) {
484	0	return MAKE_TOKEN(ERRORTOKEN);
485	0	}
486	75.4k	break;
487	647k	default:
488	647k	break;
489	883k	}
490
491	883k	if (!Py_UNICODE_ISPRINTABLE(c)) {
492	391	return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid non-printable character U+%04X", c));
493	391	}
494
495	882k	if (c == '=' && current != NULL) {
496	8.47k	_PyLexer_mark_ftstring_debug(tok, current);
497	8.47k	}
498
499		/* Punctuation character */
500	882k	p_start = tok->start;
501	882k	p_end = tok->cur;
502	882k	return MAKE_TOKEN(_PyToken_OneChar(c));
503	883k	}
504
505
506		void
507		_PyTokenizer_Get(struct tok_state tok, struct token token)
508	2.01M	{
509	2.01M	_PyToken_Free(token);
510	2.01M	ftstring_state *current = _PyLexer_CurrentFTString(tok);
511	2.01M	int result;
512	2.01M	if (current == NULL) {
513	1.81M	result = _PyLexer_get_normal(tok, NULL, token);
514	1.81M	}
515	199k	else {
516	199k	switch (current->mode) {
517	136k	case FTSTRING_MODE_EXPRESSION:
518	136k	result = _PyLexer_get_normal(tok, current, token);
519	136k	break;
520	56.9k	case FTSTRING_MODE_MIDDLE:
521	62.2k	case FTSTRING_MODE_FORMAT_SPEC:
522	62.2k	result = _PyLexer_get_ftstring(tok, current, token);
523	62.2k	break;
524	0	default:
525	0	Py_UNREACHABLE();
526	199k	}
527	199k	}
528	2.01M	if (tok_failed(tok) && tok->done != E_INCOMPLETE) {
529	2.65k	result = ERRORTOKEN;
530	2.65k	}
531	2.01M	token->type = result;
532	2.01M	}