/src/cpython/Parser/lexer/lexer.c

Source
#include "Python.h"
#include "pycore_token.h"
#include "pycore_unicodeobject.h"
#include "errcode.h"

#include "lexer_internal.h"
#include "../tokenizer/helpers.h"
#include "../tokenizer/reader.h"

#define MAKE_TOKEN(token_type) _PyLexer_token_setup(tok, token, token_type, p_start, p_end)

/* Spaces in this constant are treated as "zero or more spaces or tabs" when
   tokenizing. */
static const char* type_comment_prefix = "# type: ";

int
_PyLexer_refill(struct tok_state *tok)
{
    if (tok->done != E_OK) {
        return 0;
    }
    int rc = _PyTok_ReaderUnderflow(tok);
#if defined(Py_DEBUG)
    if (tok->debug) {
        fprintf(stderr, "line[%d] = ", tok->lineno);
        _PyTokenizer_print_escape(
            stderr, _PyTok_SourcePointer(&tok->source, tok->cur),
            tok->inp - tok->cur);
        fprintf(stderr, "  tok->done = %d\n", tok->done);
    }
#endif
    if (!rc) {
        tok->cur = tok->inp;
        return 0;
    }
    tok->line_start = tok->cur;
    _PyTok_Span line = {tok->line_start, tok->inp};
    if (_PyTok_SourceFindByte(&tok->source, line, '\0') >= 0) {
        _PyTokenizer_syntaxerror(tok, "source code cannot contain null bytes");
        tok->cur = tok->inp;
        return 0;
    }
    return 1;
}

/* Back-up one character */
void
_PyLexer_backup(struct tok_state *tok, int c)
{
    if (c != EOF) {
        if (--tok->cur < tok->buf_offset) {
            Py_FatalError("tokenizer beginning of buffer");
        }
        if (_PyTok_SourceByte(&tok->source, tok->cur) != Py_CHARMASK(c)) {
            Py_FatalError("tok_backup: wrong character");
        }
    }
}



/* Verify that the identifier follows PEP 3131. */
static int
verify_identifier(struct tok_state *tok)
{
    if (tok->tok_extra_tokens) {
        return 1;
    }
    PyObject *s;
    if (tok_failed(tok))
        return 0;
    s = PyUnicode_DecodeUTF8(_PyTok_SourcePointer(&tok->source, tok->start),
                             tok->cur - tok->start, NULL);
    if (s == NULL) {
        if (PyErr_ExceptionMatches(PyExc_UnicodeDecodeError)) {
            tok->done = E_DECODE;
        }
        else {
            tok->done = E_ERROR;
        }
        return 0;
    }
    Py_ssize_t invalid = _PyUnicode_ScanIdentifier(s);
    assert(invalid >= 0);
    assert(PyUnicode_GET_LENGTH(s) > 0);
    if (invalid < PyUnicode_GET_LENGTH(s)) {
        Py_UCS4 ch = PyUnicode_READ_CHAR(s, invalid);
        _PyTok_Off error_cursor = tok->cur;
        if (invalid + 1 < PyUnicode_GET_LENGTH(s)) {
            /* Determine the offset in UTF-8 encoded input */
            Py_SETREF(s, PyUnicode_Substring(s, 0, invalid + 1));
            if (s != NULL) {
                Py_SETREF(s, PyUnicode_AsUTF8String(s));
            }
            if (s == NULL) {
                tok->done = E_ERROR;
                return 0;
            }
            error_cursor = tok->start + PyBytes_GET_SIZE(s);
        }
        Py_DECREF(s);
        if (Py_UNICODE_ISPRINTABLE(ch)) {
            _PyTokenizer_syntaxerror_at(
                tok, _PyTok_SourcePointer(&tok->source, tok->line_start),
                error_cursor - tok->line_start, tok->lineno, -1, -1,
                "invalid character '%c' (U+%04X)", ch, ch);
        }
        else {
            _PyTokenizer_syntaxerror_at(
                tok, _PyTok_SourcePointer(&tok->source, tok->line_start),
                error_cursor - tok->line_start, tok->lineno, -1, -1,
                "invalid non-printable character U+%04X", ch);
        }
        return 0;
    }
    Py_DECREF(s);
    return 1;
}

int
_PyLexer_get_normal(struct tok_state *tok, ftstring_state *current, struct token *token)
{
    assert(current == NULL ||
           (current->mode == FTSTRING_MODE_EXPRESSION &&
            current->replacement_depth > 0));
    int c;
    int blankline, nonascii;

    _PyTok_Off p_start = -1;
    _PyTok_Off p_end = -1;
  nextline:
    tok->start = -1;
    tok->start_loc = (_PyTok_Loc){tok->lineno, -1};
    blankline = 0;


    if (tok->layout.at_bol) {
        blankline = _PyLexer_BeginLine(tok);
        if (blankline < 0) {
            return MAKE_TOKEN(ERRORTOKEN);
        }
    }

    tok->start = tok->cur;
    tok->start_loc = (_PyTok_Loc){
        tok->lineno, tok->line_start >= 0 ? _PyLexer_ByteColumn(tok) : -1};

    if (tok->layout.pending != 0) {
        return _PyLexer_IndentationToken(tok, token);
    }

    /* Peek ahead at the next character */
    c = tok_nextc(tok);
    tok_backup(tok, c);

 again:
    tok->start = -1;
    /* Skip spaces */
    do {
        c = tok_nextc(tok);
    } while (c == ' ' || c == '\t' || c == '\014');

    /* Set start of current token */
    tok->start = tok->cur - 1;
    tok->start_loc = (_PyTok_Loc){
        tok->lineno, tok->line_start >= 0 ? _PyLexer_ByteColumn(tok) - 1 : -1};

    /* Skip comment, unless it's a type comment */
    if (c == '#') {

        while (c != EOF && c != '\n' && c != '\r') {
            c = tok_nextc(tok);
        }

        if (current != NULL) {
            _PyTok_Off comment_end = tok->cur;
            if (c == '\n' || c == '\r') {
                comment_end--;
            }
            if (_PyLexer_record_ftstring_comment(
                    tok, current, tok->start, comment_end) < 0) {
                tok->done = E_NOMEM;
                return MAKE_TOKEN(ERRORTOKEN);
            }
        }

        _PyTok_Off comment = tok->start;
        if (tok->type_comments) {
            const char *prefix = type_comment_prefix;
            while (*prefix && comment < tok->cur) {
                if (*prefix == ' ') {
                    while (comment < tok->cur) {
                        int ch = _PyTok_SourceByte(&tok->source, comment);
                        if (ch != ' ' && ch != '\t') {
                            break;
                        }
                        comment++;
                    }
                } else if (*prefix == _PyTok_SourceByte(&tok->source, comment)) {
                    comment++;
                } else {
                    break;
                }

                prefix++;
            }

            /* This is a type comment if we matched all of type_comment_prefix. */
            if (!*prefix) {
                tok_backup(tok, c);  /* don't eat the newline or EOF */
                /* A TYPE_IGNORE is "type: ignore" followed by the end of the token
                 * or anything ASCII and non-alphanumeric. */
                int is_type_ignore = tok->cur - comment >= 6 &&
                    memcmp(_PyTok_SourcePointer(&tok->source, comment),
                           "ignore", 6) == 0;
                if (is_type_ignore && comment + 6 < tok->cur) {
                    int ch = _PyTok_SourceByte(&tok->source, comment + 6);
                    is_type_ignore = ch < 128 && !Py_ISALNUM(ch);
                }

                int type = is_type_ignore ? TYPE_IGNORE : TYPE_COMMENT;
                p_start = comment + (is_type_ignore ? 6 : 0);
                p_end = tok->cur;
                int start_col_offset = tok->start_loc.byte_col +
                    (int)(p_start - tok->start);
                /* Consume the newline after a standalone type ignore. */
                if (is_type_ignore && blankline) {
                    tok_nextc(tok);
                    tok->layout.at_bol = 1;
                }
                _PyLexer_token_setup(tok, token, type, p_start, p_end);
                token->start_loc = (_PyTok_Loc){tok->lineno, start_col_offset};
                token->end_loc = (_PyTok_Loc){tok->lineno,
                                              _PyLexer_ByteColumn(tok)};
                return type;
            }
        }
        if (tok->tok_extra_tokens) {
            tok_backup(tok, c);  /* don't eat the newline or EOF */
            p_start = comment;
            p_end = tok->cur;
            tok->layout.comment_newline = blankline;
            return MAKE_TOKEN(COMMENT);
        }
    }

    if (tok->done == E_INTERACT_STOP) {
        return MAKE_TOKEN(ENDMARKER);
    }

    /* Check for EOF and errors now */
    if (c == EOF) {
        if (tok->level) {
            return MAKE_TOKEN(ERRORTOKEN);
        }
        return MAKE_TOKEN(tok->done == E_EOF ? ENDMARKER : ERRORTOKEN);
    }

    /* Identifier (most frequent token!) */
    nonascii = 0;
    if (is_potential_identifier_start(c)) {
        /* Process the various legal combinations of b"", r"", u"", and f"". */
        int saw_b = 0, saw_r = 0, saw_u = 0, saw_f = 0, saw_t = 0;
        while (1) {
            if (!saw_b && (c == 'b' || c == 'B')) {
                saw_b = 1;
            }
            /* Since this is a backwards compatibility support literal we don't
               want to support it in arbitrary order like byte literals. */
            else if (!saw_u && (c == 'u'|| c == 'U')) {
                saw_u = 1;
            }
            /* ur"" and ru"" are not supported */
            else if (!saw_r && (c == 'r' || c == 'R')) {
                saw_r = 1;
            }
            else if (!saw_f && (c == 'f' || c == 'F')) {
                saw_f = 1;
            }
            else if (!saw_t && (c == 't' || c == 'T')) {
                saw_t = 1;
            }
            else {
                break;
            }
            c = tok_nextc(tok);
            if (c == '"' || c == '\'') {
                // Raise error on incompatible string prefixes:
                int status = _PyLexer_check_string_prefixes(
                    tok, saw_b, saw_r, saw_u, saw_f, saw_t);
                if (status < 0) {
                    return MAKE_TOKEN(ERRORTOKEN);
                }

                // Handle valid f or t string creation:
                if (saw_f || saw_t) {
                    ftstring_kind kind = saw_t
                        ? (saw_r ? RAW_TSTRING : TSTRING)
                        : (saw_r ? RAW_FSTRING : FSTRING);
                    return _PyLexer_scan_fstring_start(tok, token, c, kind);
                }
                return _PyLexer_scan_string(tok, token, c);
            }
        }
        while (is_potential_identifier_char(c)) {
            if (c >= 128) {
                nonascii = 1;
            }
            c = tok_nextc(tok);
        }
        tok_backup(tok, c);
        if (nonascii && !verify_identifier(tok)) {
            return MAKE_TOKEN(ERRORTOKEN);
        }

        p_start = tok->start;
        p_end = tok->cur;

        return MAKE_TOKEN(NAME);
    }

    if (c == '\r') {
        c = tok_nextc(tok);
    }

    if (c == '\n') {
        int type = _PyLexer_Newline(tok, token, blankline);
        if (type == 0) {
            goto nextline;
        }
        return type;
    }

    /* Period or number starting with period? */
    if (c == '.') {
        c = tok_nextc(tok);
        if (Py_ISDIGIT(c)) {
            return _PyLexer_scan_number(tok, token, c, 1);
        } else if (c == '.') {
            c = tok_nextc(tok);
            if (c == '.') {
                p_start = tok->start;
                p_end = tok->cur;
                return MAKE_TOKEN(ELLIPSIS);
            }
            else {
                tok_backup(tok, c);
            }
            tok_backup(tok, '.');
        }
        else {
            tok_backup(tok, c);
        }
        p_start = tok->start;
        p_end = tok->cur;
        return MAKE_TOKEN(DOT);
    }


    /* Number */
    if (Py_ISDIGIT(c)) {
        return _PyLexer_scan_number(tok, token, c, 0);
    }

    /* String */
    if (c == '\'' || c == '"') {
        return _PyLexer_scan_string(tok, token, c);
    }

    /* Line continuation */
    if (c == '\\') {
        if ((c = _PyLexer_ContinueLine(tok)) == -1) {
            return MAKE_TOKEN(ERRORTOKEN);
        }
        goto again; /* Read next line */
    }

    /* Punctuation character */
    int is_punctuation = (c == ':' || c == '}' || c == '!');
    if (is_punctuation && current != NULL) {
        int type = _PyLexer_ftstring_punctuation(tok, current, token, c);
        if (type < 0) {
            return MAKE_TOKEN(ERRORTOKEN);
        }
        if (type != 0) {
            p_start = tok->start;
            p_end = tok->cur;
            return MAKE_TOKEN(type);
        }
    }

    /* Check for two-character token */
    {
        int c2 = tok_nextc(tok);
        int current_token = _PyToken_TwoChars(c, c2, tok->barry_as_bdfl);
        if (current_token != OP) {
            int c3 = tok_nextc(tok);
            int current_token3 = _PyToken_ThreeChars(c, c2, c3);
            if (current_token3 != OP) {
                current_token = current_token3;
            }
            else {
                tok_backup(tok, c3);
            }
            p_start = tok->start;
            p_end = tok->cur;
            return MAKE_TOKEN(current_token);
        }
        tok_backup(tok, c2);
    }

    /* Keep track of parentheses nesting level */
    switch (c) {
    case '(':
    case '[':
    case '{':
        if (tok->level >= MAXLEVEL) {
            return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "too many nested parentheses"));
        }
        tok->parenstack[tok->level] = c;
        tok->parenlinenostack[tok->level] = tok->lineno;
        tok->parencolstack[tok->level] = (int)(tok->start - tok->line_start);
        tok->level++;
        break;
    case ')':
    case ']':
    case '}':
        if (current != NULL &&
                _PyLexer_FTStringBracketDepth(tok, current) == 0) {
            if (c == '}') {
                return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok,
                    "%c-string: single '}' is not allowed",
                    _PyLexer_StringPrefix(current->kind)));
            }
            return MAKE_TOKEN(_PyTokenizer_syntaxerror(
                tok, "%c-string: unmatched '%c'",
                _PyLexer_StringPrefix(current->kind), c));
        }
        if (!tok->tok_extra_tokens && !tok->level) {
            return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "unmatched '%c'", c));
        }
        if (tok->level > 0) {
            tok->level--;
            int opening = tok->parenstack[tok->level];
            if (!tok->tok_extra_tokens && !((opening == '(' && c == ')') ||
                                            (opening == '[' && c == ']') ||
                                            (opening == '{' && c == '}'))) {
                /* Do not match a closer against the brace that opened the
                 * current replacement field. */
                if (current != NULL && opening == '{') {
                    int bracket_depth =
                        _PyLexer_FTStringBracketDepth(tok, current);
                    if (bracket_depth == current->replacement_depth - 1) {
                        return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok,
                            "%c-string: unmatched '%c'",
                            _PyLexer_StringPrefix(current->kind), c));
                    }
                }
                if (tok->parenlinenostack[tok->level] != tok->lineno) {
                    return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok,
                            "closing parenthesis '%c' does not match "
                            "opening parenthesis '%c' on line %d",
                            c, opening, tok->parenlinenostack[tok->level]));
                }
                else {
                    return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok,
                            "closing parenthesis '%c' does not match "
                            "opening parenthesis '%c'",
                            c, opening));
                }
            }
        }
        if (current != NULL &&
                _PyLexer_close_ftstring_expr(tok, current, c) < 0) {
            return MAKE_TOKEN(ERRORTOKEN);
        }
        break;
    default:
        break;
    }

    if (!Py_UNICODE_ISPRINTABLE(c)) {
        return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid non-printable character U+%04X", c));
    }

    if (c == '=' && current != NULL) {
        _PyLexer_mark_ftstring_debug(tok, current);
    }

    /* Punctuation character */
    p_start = tok->start;
    p_end = tok->cur;
    return MAKE_TOKEN(_PyToken_OneChar(c));
}


void
_PyTokenizer_Get(struct tok_state *tok, struct token *token)
{
    _PyToken_Free(token);
    ftstring_state *current = _PyLexer_CurrentFTString(tok);
    int result;
    if (current == NULL) {
        result = _PyLexer_get_normal(tok, NULL, token);
    }
    else {
        switch (current->mode) {
        case FTSTRING_MODE_EXPRESSION:
            result = _PyLexer_get_normal(tok, current, token);
            break;
        case FTSTRING_MODE_MIDDLE:
        case FTSTRING_MODE_FORMAT_SPEC:
            result = _PyLexer_get_ftstring(tok, current, token);
            break;
        default:
            Py_UNREACHABLE();
        }
    }
    if (tok_failed(tok)) {
        result = ERRORTOKEN;
    }
    token->type = result;
}

Coverage Report

Created: 2026-10-17 06:15

Line	Count	Source
1		#include "Python.h"
2		#include "pycore_token.h"
3		#include "pycore_unicodeobject.h"
4		#include "errcode.h"
5
6		#include "lexer_internal.h"
7		#include "../tokenizer/helpers.h"
8		#include "../tokenizer/reader.h"
9
10	1.51M	#define MAKE_TOKEN(token_type) _PyLexer_token_setup(tok, token, token_type, p_start, p_end)
11
12		/* Spaces in this constant are treated as "zero or more spaces or tabs" when
13		tokenizing. */
14		static const char* type_comment_prefix = "# type: ";
15
16		int
17		_PyLexer_refill(struct tok_state *tok)
18	435k	{
19	435k	if (tok->done != E_OK) {
20	119k	return 0;
21	119k	}
22	316k	int rc = _PyTok_ReaderUnderflow(tok);
23		#if defined(Py_DEBUG)
24		if (tok->debug) {
25		fprintf(stderr, "line[%d] = ", tok->lineno);
26		_PyTokenizer_print_escape(
27		stderr, _PyTok_SourcePointer(&tok->source, tok->cur),
28		tok->inp - tok->cur);
29		fprintf(stderr, " tok->done = %d\n", tok->done);
30		}
31		#endif
32	316k	if (!rc) {
33	59.9k	tok->cur = tok->inp;
34	59.9k	return 0;
35	59.9k	}
36	256k	tok->line_start = tok->cur;
37	256k	_PyTok_Span line = {tok->line_start, tok->inp};
38	256k	if (_PyTok_SourceFindByte(&tok->source, line, '\0') >= 0) {
39	85	_PyTokenizer_syntaxerror(tok, "source code cannot contain null bytes");
40	85	tok->cur = tok->inp;
41	85	return 0;
42	85	}
43	256k	return 1;
44	256k	}
45
46		/* Back-up one character */
47		void
48		_PyLexer_backup(struct tok_state *tok, int c)
49	3.94M	{
50	3.94M	if (c != EOF) {
51	3.82M	if (--tok->cur < tok->buf_offset) {
52	0	Py_FatalError("tokenizer beginning of buffer");
53	0	}
54	3.82M	if (_PyTok_SourceByte(&tok->source, tok->cur) != Py_CHARMASK(c)) {
55	0	Py_FatalError("tok_backup: wrong character");
56	0	}
57	3.82M	}
58	3.94M	}
59
60
61
62		/* Verify that the identifier follows PEP 3131. */
63		static int
64		verify_identifier(struct tok_state *tok)
65	12.9k	{
66	12.9k	if (tok->tok_extra_tokens) {
67	0	return 1;
68	0	}
69	12.9k	PyObject *s;
70	12.9k	if (tok_failed(tok))
71	0	return 0;
72	12.9k	s = PyUnicode_DecodeUTF8(_PyTok_SourcePointer(&tok->source, tok->start),
73	12.9k	tok->cur - tok->start, NULL);
74	12.9k	if (s == NULL) {
75	0	if (PyErr_ExceptionMatches(PyExc_UnicodeDecodeError)) {
76	0	tok->done = E_DECODE;
77	0	}
78	0	else {
79	0	tok->done = E_ERROR;
80	0	}
81	0	return 0;
82	0	}
83	12.9k	Py_ssize_t invalid = _PyUnicode_ScanIdentifier(s);
84	12.9k	assert(invalid >= 0);
85	12.9k	assert(PyUnicode_GET_LENGTH(s) > 0);
86	12.9k	if (invalid < PyUnicode_GET_LENGTH(s)) {
87	580	Py_UCS4 ch = PyUnicode_READ_CHAR(s, invalid);
88	580	_PyTok_Off error_cursor = tok->cur;
89	580	if (invalid + 1 < PyUnicode_GET_LENGTH(s)) {
90		/* Determine the offset in UTF-8 encoded input */
91	397	Py_SETREF(s, PyUnicode_Substring(s, 0, invalid + 1));
92	397	if (s != NULL) {
93	397	Py_SETREF(s, PyUnicode_AsUTF8String(s));
94	397	}
95	397	if (s == NULL) {
96	0	tok->done = E_ERROR;
97	0	return 0;
98	0	}
99	397	error_cursor = tok->start + PyBytes_GET_SIZE(s);
100	397	}
101	580	Py_DECREF(s);
102	580	if (Py_UNICODE_ISPRINTABLE(ch)) {
103	323	_PyTokenizer_syntaxerror_at(
104	323	tok, _PyTok_SourcePointer(&tok->source, tok->line_start),
105	323	error_cursor - tok->line_start, tok->lineno, -1, -1,
106	323	"invalid character '%c' (U+%04X)", ch, ch);
107	323	}
108	257	else {
109	257	_PyTokenizer_syntaxerror_at(
110	257	tok, _PyTok_SourcePointer(&tok->source, tok->line_start),
111	257	error_cursor - tok->line_start, tok->lineno, -1, -1,
112	257	"invalid non-printable character U+%04X", ch);
113	257	}
114	580	return 0;
115	580	}
116	12.4k	Py_DECREF(s);
117	12.4k	return 1;
118	12.9k	}
119
120		int
121		_PyLexer_get_normal(struct tok_state tok, ftstring_state current, struct token *token)
122	1.82M	{
123	1.82M	assert(current == NULL \|\|
124	1.82M	(current->mode == FTSTRING_MODE_EXPRESSION &&
125	1.82M	current->replacement_depth > 0));
126	1.82M	int c;
127	1.82M	int blankline, nonascii;
128
129	1.82M	_PyTok_Off p_start = -1;
130	1.82M	_PyTok_Off p_end = -1;
131	1.93M	nextline:
132	1.93M	tok->start = -1;
133	1.93M	tok->start_loc = (_PyTok_Loc){tok->lineno, -1};
134	1.93M	blankline = 0;
135
136
137	1.93M	if (tok->layout.at_bol) {
138	303k	blankline = _PyLexer_BeginLine(tok);
139	303k	if (blankline < 0) {
140	124	return MAKE_TOKEN(ERRORTOKEN);
141	124	}
142	303k	}
143
144	1.93M	tok->start = tok->cur;
145	1.93M	tok->start_loc = (_PyTok_Loc){
146	1.93M	tok->lineno, tok->line_start >= 0 ? _PyLexer_ByteColumn(tok) : -1};
147
148	1.93M	if (tok->layout.pending != 0) {
149	19.6k	return _PyLexer_IndentationToken(tok, token);
150	19.6k	}
151
152		/* Peek ahead at the next character */
153	1.91M	c = tok_nextc(tok);
154	1.91M	tok_backup(tok, c);
155
156	1.91M	again:
157	1.91M	tok->start = -1;
158		/* Skip spaces */
159	2.27M	do {
160	2.27M	c = tok_nextc(tok);
161	2.27M	} while (c == ' ' \|\| c == '\t' \|\| c == '\014');
162
163		/* Set start of current token */
164	1.91M	tok->start = tok->cur - 1;
165	1.91M	tok->start_loc = (_PyTok_Loc){
166	1.91M	tok->lineno, tok->line_start >= 0 ? _PyLexer_ByteColumn(tok) - 1 : -1};
167
168		/* Skip comment, unless it's a type comment */
169	1.91M	if (c == '#') {
170
171	1.08M	while (c != EOF && c != '\n' && c != '\r') {
172	1.04M	c = tok_nextc(tok);
173	1.04M	}
174
175	35.1k	if (current != NULL) {
176	1.50k	_PyTok_Off comment_end = tok->cur;
177	1.50k	if (c == '\n' \|\| c == '\r') {
178	1.50k	comment_end--;
179	1.50k	}
180	1.50k	if (_PyLexer_record_ftstring_comment(
181	1.50k	tok, current, tok->start, comment_end) < 0) {
182	0	tok->done = E_NOMEM;
183	0	return MAKE_TOKEN(ERRORTOKEN);
184	0	}
185	1.50k	}
186
187	35.1k	_PyTok_Off comment = tok->start;
188	35.1k	if (tok->type_comments) {
189	0	const char *prefix = type_comment_prefix;
190	0	while (*prefix && comment < tok->cur) {
191	0	if (*prefix == ' ') {
192	0	while (comment < tok->cur) {
193	0	int ch = _PyTok_SourceByte(&tok->source, comment);
194	0	if (ch != ' ' && ch != '\t') {
195	0	break;
196	0	}
197	0	comment++;
198	0	}
199	0	} else if (*prefix == _PyTok_SourceByte(&tok->source, comment)) {
200	0	comment++;
201	0	} else {
202	0	break;
203	0	}
204
205	0	prefix++;
206	0	}
207
208		/* This is a type comment if we matched all of type_comment_prefix. */
209	0	if (!*prefix) {
210	0	tok_backup(tok, c); /* don't eat the newline or EOF */
211		/* A TYPE_IGNORE is "type: ignore" followed by the end of the token
212		* or anything ASCII and non-alphanumeric. */
213	0	int is_type_ignore = tok->cur - comment >= 6 &&
214	0	memcmp(_PyTok_SourcePointer(&tok->source, comment),
215	0	"ignore", 6) == 0;
216	0	if (is_type_ignore && comment + 6 < tok->cur) {
217	0	int ch = _PyTok_SourceByte(&tok->source, comment + 6);
218	0	is_type_ignore = ch < 128 && !Py_ISALNUM(ch);
219	0	}
220
221	0	int type = is_type_ignore ? TYPE_IGNORE : TYPE_COMMENT;
222	0	p_start = comment + (is_type_ignore ? 6 : 0);
223	0	p_end = tok->cur;
224	0	int start_col_offset = tok->start_loc.byte_col +
225	0	(int)(p_start - tok->start);
226		/* Consume the newline after a standalone type ignore. */
227	0	if (is_type_ignore && blankline) {
228	0	tok_nextc(tok);
229	0	tok->layout.at_bol = 1;
230	0	}
231	0	_PyLexer_token_setup(tok, token, type, p_start, p_end);
232	0	token->start_loc = (_PyTok_Loc){tok->lineno, start_col_offset};
233	0	token->end_loc = (_PyTok_Loc){tok->lineno,
234	0	_PyLexer_ByteColumn(tok)};
235	0	return type;
236	0	}
237	0	}
238	35.1k	if (tok->tok_extra_tokens) {
239	44	tok_backup(tok, c); /* don't eat the newline or EOF */
240	44	p_start = comment;
241	44	p_end = tok->cur;
242	44	tok->layout.comment_newline = blankline;
243	44	return MAKE_TOKEN(COMMENT);
244	44	}
245	35.1k	}
246
247	1.91M	if (tok->done == E_INTERACT_STOP) {
248	0	return MAKE_TOKEN(ENDMARKER);
249	0	}
250
251		/* Check for EOF and errors now */
252	1.91M	if (c == EOF) {
253	59.8k	if (tok->level) {
254	4.06k	return MAKE_TOKEN(ERRORTOKEN);
255	4.06k	}
256	55.7k	return MAKE_TOKEN(tok->done == E_EOF ? ENDMARKER : ERRORTOKEN);
257	59.8k	}
258
259		/* Identifier (most frequent token!) */
260	1.85M	nonascii = 0;
261	1.85M	if (is_potential_identifier_start(c)) {
262		/* Process the various legal combinations of b"", r"", u"", and f"". */
263	599k	int saw_b = 0, saw_r = 0, saw_u = 0, saw_f = 0, saw_t = 0;
264	746k	while (1) {
265	746k	if (!saw_b && (c == 'b' \|\| c == 'B')) {
266	17.0k	saw_b = 1;
267	17.0k	}
268		/* Since this is a backwards compatibility support literal we don't
269		want to support it in arbitrary order like byte literals. */
270	729k	else if (!saw_u && (c == 'u'\|\| c == 'U')) {
271	48.7k	saw_u = 1;
272	48.7k	}
273		/* ur"" and ru"" are not supported */
274	681k	else if (!saw_r && (c == 'r' \|\| c == 'R')) {
275	26.9k	saw_r = 1;
276	26.9k	}
277	654k	else if (!saw_f && (c == 'f' \|\| c == 'F')) {
278	45.1k	saw_f = 1;
279	45.1k	}
280	608k	else if (!saw_t && (c == 't' \|\| c == 'T')) {
281	34.4k	saw_t = 1;
282	34.4k	}
283	574k	else {
284	574k	break;
285	574k	}
286	172k	c = tok_nextc(tok);
287	172k	if (c == '"' \|\| c == '\'') {
288		// Raise error on incompatible string prefixes:
289	24.6k	int status = _PyLexer_check_string_prefixes(
290	24.6k	tok, saw_b, saw_r, saw_u, saw_f, saw_t);
291	24.6k	if (status < 0) {
292	11	return MAKE_TOKEN(ERRORTOKEN);
293	11	}
294
295		// Handle valid f or t string creation:
296	24.6k	if (saw_f \|\| saw_t) {
297	20.2k	ftstring_kind kind = saw_t
298	20.2k	? (saw_r ? RAW_TSTRING : TSTRING)
299	20.2k	: (saw_r ? RAW_FSTRING : FSTRING);
300	20.2k	return _PyLexer_scan_fstring_start(tok, token, c, kind);
301	20.2k	}
302	4.34k	return _PyLexer_scan_string(tok, token, c);
303	24.6k	}
304	172k	}
305	2.60M	while (is_potential_identifier_char(c)) {
306	2.03M	if (c >= 128) {
307	260k	nonascii = 1;
308	260k	}
309	2.03M	c = tok_nextc(tok);
310	2.03M	}
311	574k	tok_backup(tok, c);
312	574k	if (nonascii && !verify_identifier(tok)) {
313	580	return MAKE_TOKEN(ERRORTOKEN);
314	580	}
315
316	573k	p_start = tok->start;
317	573k	p_end = tok->cur;
318
319	573k	return MAKE_TOKEN(NAME);
320	574k	}
321
322	1.25M	if (c == '\r') {
323	0	c = tok_nextc(tok);
324	0	}
325
326	1.25M	if (c == '\n') {
327	241k	int type = _PyLexer_Newline(tok, token, blankline);
328	241k	if (type == 0) {
329	114k	goto nextline;
330	114k	}
331	126k	return type;
332	241k	}
333
334		/* Period or number starting with period? */
335	1.01M	if (c == '.') {
336	32.0k	c = tok_nextc(tok);
337	32.0k	if (Py_ISDIGIT(c)) {
338	3.95k	return _PyLexer_scan_number(tok, token, c, 1);
339	28.1k	} else if (c == '.') {
340	4.02k	c = tok_nextc(tok);
341	4.02k	if (c == '.') {
342	2.53k	p_start = tok->start;
343	2.53k	p_end = tok->cur;
344	2.53k	return MAKE_TOKEN(ELLIPSIS);
345	2.53k	}
346	1.49k	else {
347	1.49k	tok_backup(tok, c);
348	1.49k	}
349	1.49k	tok_backup(tok, '.');
350	1.49k	}
351	24.0k	else {
352	24.0k	tok_backup(tok, c);
353	24.0k	}
354	25.5k	p_start = tok->start;
355	25.5k	p_end = tok->cur;
356	25.5k	return MAKE_TOKEN(DOT);
357	32.0k	}
358
359
360		/* Number */
361	985k	if (Py_ISDIGIT(c)) {
362	99.0k	return _PyLexer_scan_number(tok, token, c, 0);
363	99.0k	}
364
365		/* String */
366	886k	if (c == '\'' \|\| c == '"') {
367	34.0k	return _PyLexer_scan_string(tok, token, c);
368	34.0k	}
369
370		/* Line continuation */
371	852k	if (c == '\\') {
372	364	if ((c = _PyLexer_ContinueLine(tok)) == -1) {
373	79	return MAKE_TOKEN(ERRORTOKEN);
374	79	}
375	285	goto again; /* Read next line */
376	364	}
377
378		/* Punctuation character */
379	852k	int is_punctuation = (c == ':' \|\| c == '}' \|\| c == '!');
380	852k	if (is_punctuation && current != NULL) {
381	31.2k	int type = _PyLexer_ftstring_punctuation(tok, current, token, c);
382	31.2k	if (type < 0) {
383	0	return MAKE_TOKEN(ERRORTOKEN);
384	0	}
385	31.2k	if (type != 0) {
386	5.26k	p_start = tok->start;
387	5.26k	p_end = tok->cur;
388	5.26k	return MAKE_TOKEN(type);
389	5.26k	}
390	31.2k	}
391
392		/* Check for two-character token */
393	847k	{
394	847k	int c2 = tok_nextc(tok);
395	847k	int current_token = _PyToken_TwoChars(c, c2, tok->barry_as_bdfl);
396	847k	if (current_token != OP) {
397	26.5k	int c3 = tok_nextc(tok);
398	26.5k	int current_token3 = _PyToken_ThreeChars(c, c2, c3);
399	26.5k	if (current_token3 != OP) {
400	1.26k	current_token = current_token3;
401	1.26k	}
402	25.2k	else {
403	25.2k	tok_backup(tok, c3);
404	25.2k	}
405	26.5k	p_start = tok->start;
406	26.5k	p_end = tok->cur;
407	26.5k	return MAKE_TOKEN(current_token);
408	26.5k	}
409	820k	tok_backup(tok, c2);
410	820k	}
411
412		/* Keep track of parentheses nesting level */
413	0	switch (c) {
414	67.7k	case '(':
415	102k	case '[':
416	153k	case '{':
417	153k	if (tok->level >= MAXLEVEL) {
418	11	return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "too many nested parentheses"));
419	11	}
420	153k	tok->parenstack[tok->level] = c;
421	153k	tok->parenlinenostack[tok->level] = tok->lineno;
422	153k	tok->parencolstack[tok->level] = (int)(tok->start - tok->line_start);
423	153k	tok->level++;
424	153k	break;
425	38.8k	case ')':
426	44.6k	case ']':
427	72.5k	case '}':
428	72.5k	if (current != NULL &&
429	24.2k	_PyLexer_FTStringBracketDepth(tok, current) == 0) {
430	0	if (c == '}') {
431	0	return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok,
432	0	"%c-string: single '}' is not allowed",
433	0	_PyLexer_StringPrefix(current->kind)));
434	0	}
435	0	return MAKE_TOKEN(_PyTokenizer_syntaxerror(
436	0	tok, "%c-string: unmatched '%c'",
437	0	_PyLexer_StringPrefix(current->kind), c));
438	0	}
439	72.5k	if (!tok->tok_extra_tokens && !tok->level) {
440	227	return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "unmatched '%c'", c));
441	227	}
442	72.2k	if (tok->level > 0) {
443	72.2k	tok->level--;
444	72.2k	int opening = tok->parenstack[tok->level];
445	72.2k	if (!tok->tok_extra_tokens && !((opening == '(' && c == ')') \|\|
446	33.5k	(opening == '[' && c == ']') \|\|
447	27.8k	(opening == '{' && c == '}'))) {
448		/* Do not match a closer against the brace that opened the
449		* current replacement field. */
450	66	if (current != NULL && opening == '{') {
451	9	int bracket_depth =
452	9	_PyLexer_FTStringBracketDepth(tok, current);
453	9	if (bracket_depth == current->replacement_depth - 1) {
454	6	return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok,
455	6	"%c-string: unmatched '%c'",
456	6	_PyLexer_StringPrefix(current->kind), c));
457	6	}
458	9	}
459	60	if (tok->parenlinenostack[tok->level] != tok->lineno) {
460	4	return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok,
461	4	"closing parenthesis '%c' does not match "
462	4	"opening parenthesis '%c' on line %d",
463	4	c, opening, tok->parenlinenostack[tok->level]));
464	4	}
465	56	else {
466	56	return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok,
467	56	"closing parenthesis '%c' does not match "
468	56	"opening parenthesis '%c'",
469	56	c, opening));
470	56	}
471	60	}
472	72.2k	}
473	72.2k	if (current != NULL &&
474	24.2k	_PyLexer_close_ftstring_expr(tok, current, c) < 0) {
475	0	return MAKE_TOKEN(ERRORTOKEN);
476	0	}
477	72.2k	break;
478	595k	default:
479	595k	break;
480	820k	}
481
482	820k	if (!Py_UNICODE_ISPRINTABLE(c)) {
483	372	return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid non-printable character U+%04X", c));
484	372	}
485
486	819k	if (c == '=' && current != NULL) {
487	8.05k	_PyLexer_mark_ftstring_debug(tok, current);
488	8.05k	}
489
490		/* Punctuation character */
491	819k	p_start = tok->start;
492	819k	p_end = tok->cur;
493	819k	return MAKE_TOKEN(_PyToken_OneChar(c));
494	820k	}
495
496
497		void
498		_PyTokenizer_Get(struct tok_state tok, struct token token)
499	1.86M	{
500	1.86M	_PyToken_Free(token);
501	1.86M	ftstring_state *current = _PyLexer_CurrentFTString(tok);
502	1.86M	int result;
503	1.86M	if (current == NULL) {
504	1.67M	result = _PyLexer_get_normal(tok, NULL, token);
505	1.67M	}
506	184k	else {
507	184k	switch (current->mode) {
508	126k	case FTSTRING_MODE_EXPRESSION:
509	126k	result = _PyLexer_get_normal(tok, current, token);
510	126k	break;
511	52.8k	case FTSTRING_MODE_MIDDLE:
512	58.1k	case FTSTRING_MODE_FORMAT_SPEC:
513	58.1k	result = _PyLexer_get_ftstring(tok, current, token);
514	58.1k	break;
515	0	default:
516	0	Py_UNREACHABLE();
517	184k	}
518	184k	}
519	1.86M	if (tok_failed(tok)) {
520	2.57k	result = ERRORTOKEN;
521	2.57k	}
522	1.86M	token->type = result;
523	1.86M	}