/src/cpython/Parser/lexer/lexer.c

Source
#include "Python.h"
#include "pycore_token.h"
#include "pycore_unicodeobject.h"
#include "errcode.h"

#include "lexer_internal.h"
#include "../tokenizer/helpers.h"

/* Alternate tab spacing */
#define ALTTABSIZE 1


#define MAKE_TOKEN(token_type) _PyLexer_token_setup(tok, token, token_type, p_start, p_end)
#define MAKE_TYPE_COMMENT_TOKEN(token_type, col_offset, end_col_offset) (\
                _PyLexer_type_comment_token_setup(tok, token, token_type, col_offset, end_col_offset, p_start, p_end))

/* Spaces in this constant are treated as "zero or more spaces or tabs" when
   tokenizing. */
static const char* type_comment_prefix = "# type: ";

static inline int
contains_null_bytes(const char* str, size_t size)
{
    return memchr(str, 0, size) != NULL;
}

/* Get next char, updating state; error code goes into tok->done */
int
_PyLexer_nextc(struct tok_state *tok)
{
    int rc;
    for (;;) {
        if (tok->cur != tok->inp) {
            if ((unsigned int) tok->col_offset >= (unsigned int) INT_MAX) {
                tok->done = E_COLUMNOVERFLOW;
                return EOF;
            }
            tok->col_offset++;
            return Py_CHARMASK(*tok->cur++); /* Fast path */
        }
        if (tok->done != E_OK) {
            return EOF;
        }
        rc = tok->underflow(tok);
#if defined(Py_DEBUG)
        if (tok->debug) {
            fprintf(stderr, "line[%d] = ", tok->lineno);
            _PyTokenizer_print_escape(stderr, tok->cur, tok->inp - tok->cur);
            fprintf(stderr, "  tok->done = %d\n", tok->done);
        }
#endif
        if (!rc) {
            tok->cur = tok->inp;
            return EOF;
        }
        tok->line_start = tok->cur;

        if (contains_null_bytes(tok->line_start, tok->inp - tok->line_start)) {
            _PyTokenizer_syntaxerror(tok, "source code cannot contain null bytes");
            tok->cur = tok->inp;
            return EOF;
        }
    }
    Py_UNREACHABLE();
}

/* Back-up one character */
void
_PyLexer_backup(struct tok_state *tok, int c)
{
    if (c != EOF) {
        if (--tok->cur < tok->buf) {
            Py_FatalError("tokenizer beginning of buffer");
        }
        if ((int)(unsigned char)*tok->cur != Py_CHARMASK(c)) {
            Py_FatalError("tok_backup: wrong character");
        }
        tok->col_offset--;
    }
}



/* Verify that the identifier follows PEP 3131. */
static int
verify_identifier(struct tok_state *tok)
{
    if (tok->tok_extra_tokens) {
        return 1;
    }
    PyObject *s;
    if (tok->decoding_erred)
        return 0;
    s = PyUnicode_DecodeUTF8(tok->start, tok->cur - tok->start, NULL);
    if (s == NULL) {
        if (PyErr_ExceptionMatches(PyExc_UnicodeDecodeError)) {
            tok->done = E_DECODE;
        }
        else {
            tok->done = E_ERROR;
        }
        return 0;
    }
    Py_ssize_t invalid = _PyUnicode_ScanIdentifier(s);
    assert(invalid >= 0);
    assert(PyUnicode_GET_LENGTH(s) > 0);
    if (invalid < PyUnicode_GET_LENGTH(s)) {
        Py_UCS4 ch = PyUnicode_READ_CHAR(s, invalid);
        if (invalid + 1 < PyUnicode_GET_LENGTH(s)) {
            /* Determine the offset in UTF-8 encoded input */
            Py_SETREF(s, PyUnicode_Substring(s, 0, invalid + 1));
            if (s != NULL) {
                Py_SETREF(s, PyUnicode_AsUTF8String(s));
            }
            if (s == NULL) {
                tok->done = E_ERROR;
                return 0;
            }
            tok->cur = (char *)tok->start + PyBytes_GET_SIZE(s);
        }
        Py_DECREF(s);
        if (Py_UNICODE_ISPRINTABLE(ch)) {
            _PyTokenizer_syntaxerror(tok, "invalid character '%c' (U+%04X)", ch, ch);
        }
        else {
            _PyTokenizer_syntaxerror(tok, "invalid non-printable character U+%04X", ch);
        }
        return 0;
    }
    Py_DECREF(s);
    return 1;
}



static inline int
tok_continuation_line(struct tok_state *tok) {
    int c = tok_nextc(tok);
    if (c == '\r') {
        c = tok_nextc(tok);
    }
    if (c != '\n') {
        tok->done = E_LINECONT;
        return -1;
    }
    c = tok_nextc(tok);
    if (c == EOF) {
        tok->done = E_EOF;
        tok->cur = tok->inp;
        return -1;
    } else {
        tok_backup(tok, c);
    }
    return c;
}



int
_PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct token *token)
{
    int c;
    int blankline, nonascii;

    const char *p_start = NULL;
    const char *p_end = NULL;
  nextline:
    tok->start = NULL;
    tok->starting_col_offset = -1;
    blankline = 0;


    /* Get indentation level */
    if (tok->atbol) {
        int col = 0;
        int altcol = 0;
        tok->atbol = 0;
        int cont_line_col = 0;
        for (;;) {
            c = tok_nextc(tok);
            if (c == ' ') {
                col++, altcol++;
            }
            else if (c == '\t') {
                col = (col / tok->tabsize + 1) * tok->tabsize;
                altcol = (altcol / ALTTABSIZE + 1) * ALTTABSIZE;
            }
            else if (c == '\014')  {/* Control-L (formfeed) */
                col = altcol = 0; /* For Emacs users */
            }
            else if (c == '\\') {
                // Indentation cannot be split over multiple physical lines
                // using backslashes. This means that if we found a backslash
                // preceded by whitespace, **the first one we find** determines
                // the level of indentation of whatever comes next.
                cont_line_col = cont_line_col ? cont_line_col : col;
                if ((c = tok_continuation_line(tok)) == -1) {
                    return MAKE_TOKEN(ERRORTOKEN);
                }
            }
            else if (c == EOF && PyErr_Occurred()) {
                return MAKE_TOKEN(ERRORTOKEN);
            }
            else {
                break;
            }
        }
        tok_backup(tok, c);
        if (c == '#' || c == '\n' || c == '\r') {
            /* Lines with only whitespace and/or comments
               shouldn't affect the indentation and are
               not passed to the parser as NEWLINE tokens,
               except *totally* empty lines in interactive
               mode, which signal the end of a command group. */
            if (col == 0 && c == '\n' && tok->prompt != NULL) {
                blankline = 0; /* Let it through */
            }
            else if (tok->prompt != NULL && tok->lineno == 1) {
                /* In interactive mode, if the first line contains
                   only spaces and/or a comment, let it through. */
                blankline = 0;
                col = altcol = 0;
            }
            else {
                blankline = 1; /* Ignore completely */
            }
            /* We can't jump back right here since we still
               may need to skip to the end of a comment */
        }
        if (!blankline && tok->level == 0) {
            col = cont_line_col ? cont_line_col : col;
            altcol = cont_line_col ? cont_line_col : altcol;
            if (col == tok->indstack[tok->indent]) {
                /* No change */
                if (altcol != tok->altindstack[tok->indent]) {
                    return MAKE_TOKEN(_PyTokenizer_indenterror(tok));
                }
            }
            else if (col > tok->indstack[tok->indent]) {
                /* Indent -- always one */
                if (tok->indent+1 >= MAXINDENT) {
                    tok->done = E_TOODEEP;
                    tok->cur = tok->inp;
                    return MAKE_TOKEN(ERRORTOKEN);
                }
                if (altcol <= tok->altindstack[tok->indent]) {
                    return MAKE_TOKEN(_PyTokenizer_indenterror(tok));
                }
                tok->pendin++;
                tok->indstack[++tok->indent] = col;
                tok->altindstack[tok->indent] = altcol;
            }
            else /* col < tok->indstack[tok->indent] */ {
                /* Dedent -- any number, must be consistent */
                while (tok->indent > 0 &&
                    col < tok->indstack[tok->indent]) {
                    tok->pendin--;
                    tok->indent--;
                }
                if (col != tok->indstack[tok->indent]) {
                    tok->done = E_DEDENT;
                    tok->cur = tok->inp;
                    return MAKE_TOKEN(ERRORTOKEN);
                }
                if (altcol != tok->altindstack[tok->indent]) {
                    return MAKE_TOKEN(_PyTokenizer_indenterror(tok));
                }
            }
        }
    }

    tok->start = tok->cur;
    tok->starting_col_offset = tok->col_offset;

    /* Return pending indents/dedents */
    if (tok->pendin != 0) {
        if (tok->pendin < 0) {
            if (tok->tok_extra_tokens) {
                p_start = tok->cur;
                p_end = tok->cur;
            }
            tok->pendin++;
            return MAKE_TOKEN(DEDENT);
        }
        else {
            if (tok->tok_extra_tokens) {
                p_start = tok->buf;
                p_end = tok->cur;
            }
            tok->pendin--;
            return MAKE_TOKEN(INDENT);
        }
    }

    /* Peek ahead at the next character */
    c = tok_nextc(tok);
    tok_backup(tok, c);

 again:
    tok->start = NULL;
    /* Skip spaces */
    do {
        c = tok_nextc(tok);
    } while (c == ' ' || c == '\t' || c == '\014');

    /* Set start of current token */
    tok->start = tok->cur == NULL ? NULL : tok->cur - 1;
    tok->starting_col_offset = tok->col_offset - 1;

    /* Skip comment, unless it's a type comment */
    if (c == '#') {

        const char* p = NULL;
        const char *prefix, *type_start;
        int current_starting_col_offset;

        while (c != EOF && c != '\n' && c != '\r') {
            c = tok_nextc(tok);
        }

        if (tok->tok_extra_tokens) {
            p = tok->start;
        }

        if (tok->type_comments) {
            p = tok->start;
            current_starting_col_offset = tok->starting_col_offset;
            prefix = type_comment_prefix;
            while (*prefix && p < tok->cur) {
                if (*prefix == ' ') {
                    while (*p == ' ' || *p == '\t') {
                        p++;
                        current_starting_col_offset++;
                    }
                } else if (*prefix == *p) {
                    p++;
                    current_starting_col_offset++;
                } else {
                    break;
                }

                prefix++;
            }

            /* This is a type comment if we matched all of type_comment_prefix. */
            if (!*prefix) {
                int is_type_ignore = 1;
                // +6 in order to skip the word 'ignore'
                const char *ignore_end = p + 6;
                const int ignore_end_col_offset = current_starting_col_offset + 6;
                tok_backup(tok, c);  /* don't eat the newline or EOF */

                type_start = p;

                /* A TYPE_IGNORE is "type: ignore" followed by the end of the token
                 * or anything ASCII and non-alphanumeric. */
                is_type_ignore = (
                    tok->cur >= ignore_end && memcmp(p, "ignore", 6) == 0
                    && !(tok->cur > ignore_end
                         && ((unsigned char)ignore_end[0] >= 128 || Py_ISALNUM(ignore_end[0]))));

                if (is_type_ignore) {
                    p_start = ignore_end;
                    p_end = tok->cur;

                    /* If this type ignore is the only thing on the line, consume the newline also. */
                    if (blankline) {
                        tok_nextc(tok);
                        tok->atbol = 1;
                    }
                    return MAKE_TYPE_COMMENT_TOKEN(TYPE_IGNORE, ignore_end_col_offset, tok->col_offset);
                } else {
                    p_start = type_start;
                    p_end = tok->cur;
                    return MAKE_TYPE_COMMENT_TOKEN(TYPE_COMMENT, current_starting_col_offset, tok->col_offset);
                }
            }
        }
        if (tok->tok_extra_tokens) {
            tok_backup(tok, c);  /* don't eat the newline or EOF */
            p_start = p;
            p_end = tok->cur;
            tok->comment_newline = blankline;
            return MAKE_TOKEN(COMMENT);
        }
    }

    if (tok->done == E_INTERACT_STOP) {
        return MAKE_TOKEN(ENDMARKER);
    }

    /* Check for EOF and errors now */
    if (c == EOF) {
        if (tok->level) {
            return MAKE_TOKEN(ERRORTOKEN);
        }
        return MAKE_TOKEN(tok->done == E_EOF ? ENDMARKER : ERRORTOKEN);
    }

    /* Identifier (most frequent token!) */
    nonascii = 0;
    if (is_potential_identifier_start(c)) {
        /* Process the various legal combinations of b"", r"", u"", and f"". */
        int saw_b = 0, saw_r = 0, saw_u = 0, saw_f = 0, saw_t = 0;
        while (1) {
            if (!saw_b && (c == 'b' || c == 'B')) {
                saw_b = 1;
            }
            /* Since this is a backwards compatibility support literal we don't
               want to support it in arbitrary order like byte literals. */
            else if (!saw_u && (c == 'u'|| c == 'U')) {
                saw_u = 1;
            }
            /* ur"" and ru"" are not supported */
            else if (!saw_r && (c == 'r' || c == 'R')) {
                saw_r = 1;
            }
            else if (!saw_f && (c == 'f' || c == 'F')) {
                saw_f = 1;
            }
            else if (!saw_t && (c == 't' || c == 'T')) {
                saw_t = 1;
            }
            else {
                break;
            }
            c = tok_nextc(tok);
            if (c == '"' || c == '\'') {
                // Raise error on incompatible string prefixes:
                int status = _PyLexer_check_string_prefixes(
                    tok, saw_b, saw_r, saw_u, saw_f, saw_t);
                if (status < 0) {
                    return MAKE_TOKEN(ERRORTOKEN);
                }

                // Handle valid f or t string creation:
                if (saw_f || saw_t) {
                    return _PyLexer_scan_fstring_start(tok, token, c);
                }
                return _PyLexer_scan_string(tok, token, c);
            }
        }
        while (is_potential_identifier_char(c)) {
            if (c >= 128) {
                nonascii = 1;
            }
            c = tok_nextc(tok);
        }
        tok_backup(tok, c);
        if (nonascii && !verify_identifier(tok)) {
            return MAKE_TOKEN(ERRORTOKEN);
        }

        p_start = tok->start;
        p_end = tok->cur;

        return MAKE_TOKEN(NAME);
    }

    if (c == '\r') {
        c = tok_nextc(tok);
    }

    /* Newline */
    if (c == '\n') {
        tok->atbol = 1;
        if (blankline || tok->level > 0) {
            if (tok->tok_extra_tokens) {
                if (tok->comment_newline) {
                    tok->comment_newline = 0;
                }
                p_start = tok->start;
                p_end = tok->cur;
                return MAKE_TOKEN(NL);
            }
            goto nextline;
        }
        if (tok->comment_newline && tok->tok_extra_tokens) {
            tok->comment_newline = 0;
            p_start = tok->start;
            p_end = tok->cur;
            return MAKE_TOKEN(NL);
        }
        p_start = tok->start;
        p_end = tok->cur - 1; /* Leave '\n' out of the string */
        tok->cont_line = 0;
        return MAKE_TOKEN(NEWLINE);
    }

    /* Period or number starting with period? */
    if (c == '.') {
        c = tok_nextc(tok);
        if (Py_ISDIGIT(c)) {
            return _PyLexer_scan_number(tok, token, c, 1);
        } else if (c == '.') {
            c = tok_nextc(tok);
            if (c == '.') {
                p_start = tok->start;
                p_end = tok->cur;
                return MAKE_TOKEN(ELLIPSIS);
            }
            else {
                tok_backup(tok, c);
            }
            tok_backup(tok, '.');
        }
        else {
            tok_backup(tok, c);
        }
        p_start = tok->start;
        p_end = tok->cur;
        return MAKE_TOKEN(DOT);
    }


    /* Number */
    if (Py_ISDIGIT(c)) {
        return _PyLexer_scan_number(tok, token, c, 0);
    }

    /* String */
    if (c == '\'' || c == '"') {
        return _PyLexer_scan_string(tok, token, c);
    }

    /* Line continuation */
    if (c == '\\') {
        if ((c = tok_continuation_line(tok)) == -1) {
            return MAKE_TOKEN(ERRORTOKEN);
        }
        tok->cont_line = 1;
        goto again; /* Read next line */
    }

    /* Punctuation character */
    int is_punctuation = (c == ':' || c == '}' || c == '!' || c == '{');
    if (is_punctuation && INSIDE_FSTRING(tok) && INSIDE_FSTRING_EXPR(current_tok)) {
        /* This code block gets executed before the curly_bracket_depth is incremented
         * by the `{` case, so for ensuring that we are on the 0th level, we need
         * to adjust it manually */
        int cursor = current_tok->curly_bracket_depth - (c != '{');
        int in_format_spec = current_tok->in_format_spec;
         int cursor_in_format_with_debug =
             cursor == 1 && (current_tok->in_debug || in_format_spec);
         int cursor_valid = cursor == 0 || cursor_in_format_with_debug;
        if ((cursor_valid) && !_PyLexer_update_ftstring_expr(tok, c)) {
            return MAKE_TOKEN(ENDMARKER);
        }
        if ((cursor_valid) && c != '{' && _PyLexer_set_ftstring_expr(tok, token, c)) {
            return MAKE_TOKEN(ERRORTOKEN);
        }

        if (c == ':' && cursor == current_tok->curly_bracket_expr_start_depth) {
            current_tok->kind = TOK_FSTRING_MODE;
            current_tok->in_format_spec = 1;
            p_start = tok->start;
            p_end = tok->cur;
            return MAKE_TOKEN(_PyToken_OneChar(c));
        }
    }

    /* Check for two-character token */
    {
        int c2 = tok_nextc(tok);
        int current_token = _PyToken_TwoChars(c, c2);
        if (current_token != OP) {
            int c3 = tok_nextc(tok);
            int current_token3 = _PyToken_ThreeChars(c, c2, c3);
            if (current_token3 != OP) {
                current_token = current_token3;
            }
            else {
                tok_backup(tok, c3);
            }
            p_start = tok->start;
            p_end = tok->cur;
            return MAKE_TOKEN(current_token);
        }
        tok_backup(tok, c2);
    }

    /* Keep track of parentheses nesting level */
    switch (c) {
    case '(':
    case '[':
    case '{':
        if (tok->level >= MAXLEVEL) {
            return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "too many nested parentheses"));
        }
        tok->parenstack[tok->level] = c;
        tok->parenlinenostack[tok->level] = tok->lineno;
        tok->parencolstack[tok->level] = (int)(tok->start - tok->line_start);
        tok->level++;
        if (INSIDE_FSTRING(tok)) {
            current_tok->curly_bracket_depth++;
        }
        break;
    case ')':
    case ']':
    case '}':
        if (INSIDE_FSTRING(tok) && !current_tok->curly_bracket_depth && c == '}') {
            return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok,
                "%c-string: single '}' is not allowed", TOK_GET_STRING_PREFIX(tok)));
        }
        if (!tok->tok_extra_tokens && !tok->level) {
            return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "unmatched '%c'", c));
        }
        if (tok->level > 0) {
            tok->level--;
            int opening = tok->parenstack[tok->level];
            if (!tok->tok_extra_tokens && !((opening == '(' && c == ')') ||
                                            (opening == '[' && c == ']') ||
                                            (opening == '{' && c == '}'))) {
                /* If the opening bracket belongs to an f-string's expression
                part (e.g. f"{)}") and the closing bracket is an arbitrary
                nested expression, then instead of matching a different
                syntactical construct with it; we'll throw an unmatched
                parentheses error. */
                if (INSIDE_FSTRING(tok) && opening == '{') {
                    assert(current_tok->curly_bracket_depth >= 0);
                    int previous_bracket = current_tok->curly_bracket_depth - 1;
                    if (previous_bracket == current_tok->curly_bracket_expr_start_depth) {
                        return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok,
                            "%c-string: unmatched '%c'", TOK_GET_STRING_PREFIX(tok), c));
                    }
                }
                if (tok->parenlinenostack[tok->level] != tok->lineno) {
                    return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok,
                            "closing parenthesis '%c' does not match "
                            "opening parenthesis '%c' on line %d",
                            c, opening, tok->parenlinenostack[tok->level]));
                }
                else {
                    return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok,
                            "closing parenthesis '%c' does not match "
                            "opening parenthesis '%c'",
                            c, opening));
                }
            }
        }

        if (INSIDE_FSTRING(tok)) {
            current_tok->curly_bracket_depth--;
            if (current_tok->curly_bracket_depth < 0) {
                return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "%c-string: unmatched '%c'",
                    TOK_GET_STRING_PREFIX(tok), c));
            }
            if (c == '}' && current_tok->curly_bracket_depth == current_tok->curly_bracket_expr_start_depth) {
                current_tok->curly_bracket_expr_start_depth--;
                current_tok->kind = TOK_FSTRING_MODE;
                current_tok->in_format_spec = 0;
                current_tok->in_debug = 0;
            }
        }
        break;
    default:
        break;
    }

    if (!Py_UNICODE_ISPRINTABLE(c)) {
        return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid non-printable character U+%04X", c));
    }

    if( c == '=' && INSIDE_FSTRING_EXPR_AT_TOP(current_tok)) {
        current_tok->in_debug = 1;
    }

    /* Punctuation character */
    p_start = tok->start;
    p_end = tok->cur;
    return MAKE_TOKEN(_PyToken_OneChar(c));
}


static int
tok_get(struct tok_state *tok, struct token *token)
{
    tokenizer_mode *current_tok = TOK_GET_MODE(tok);
    if (current_tok->kind == TOK_REGULAR_MODE) {
        return _PyLexer_get_normal_mode(tok, current_tok, token);
    } else {
        return _PyLexer_get_fstring_mode(tok, current_tok, token);
    }
}

int
_PyTokenizer_Get(struct tok_state *tok, struct token *token)
{
    int result = tok_get(tok, token);
    if (tok->decoding_erred) {
        result = ERRORTOKEN;
        tok->done = E_DECODE;
    }
    return result;
}

Coverage Report

Created: 2026-07-14 06:16

Line	Count	Source
1		#include "Python.h"
2		#include "pycore_token.h"
3		#include "pycore_unicodeobject.h"
4		#include "errcode.h"
5
6		#include "lexer_internal.h"
7		#include "../tokenizer/helpers.h"
8
9		/* Alternate tab spacing */
10	1.19k	#define ALTTABSIZE 1
11
12
13	2.01M	#define MAKE_TOKEN(token_type) _PyLexer_token_setup(tok, token, token_type, p_start, p_end)
14	0	#define MAKE_TYPE_COMMENT_TOKEN(token_type, col_offset, end_col_offset) (\
15	0	_PyLexer_type_comment_token_setup(tok, token, token_type, col_offset, end_col_offset, p_start, p_end))
16
17		/* Spaces in this constant are treated as "zero or more spaces or tabs" when
18		tokenizing. */
19		static const char* type_comment_prefix = "# type: ";
20
21		static inline int
22		contains_null_bytes(const char* str, size_t size)
23	283k	{
24	283k	return memchr(str, 0, size) != NULL;
25	283k	}
26
27		/* Get next char, updating state; error code goes into tok->done */
28		int
29		_PyLexer_nextc(struct tok_state *tok)
30	11.6M	{
31	11.6M	int rc;
32	11.9M	for (;;) {
33	11.9M	if (tok->cur != tok->inp) {
34	11.3M	if ((unsigned int) tok->col_offset >= (unsigned int) INT_MAX) {
35	0	tok->done = E_COLUMNOVERFLOW;
36	0	return EOF;
37	0	}
38	11.3M	tok->col_offset++;
39	11.3M	return Py_CHARMASK(tok->cur++); / Fast path */
40	11.3M	}
41	576k	if (tok->done != E_OK) {
42	195k	return EOF;
43	195k	}
44	380k	rc = tok->underflow(tok);
45		#if defined(Py_DEBUG)
46		if (tok->debug) {
47		fprintf(stderr, "line[%d] = ", tok->lineno);
48		_PyTokenizer_print_escape(stderr, tok->cur, tok->inp - tok->cur);
49		fprintf(stderr, " tok->done = %d\n", tok->done);
50		}
51		#endif
52	380k	if (!rc) {
53	97.6k	tok->cur = tok->inp;
54	97.6k	return EOF;
55	97.6k	}
56	283k	tok->line_start = tok->cur;
57
58	283k	if (contains_null_bytes(tok->line_start, tok->inp - tok->line_start)) {
59	0	_PyTokenizer_syntaxerror(tok, "source code cannot contain null bytes");
60	0	tok->cur = tok->inp;
61	0	return EOF;
62	0	}
63	283k	}
64	11.6M	Py_UNREACHABLE();
65	11.6M	}
66
67		/* Back-up one character */
68		void
69		_PyLexer_backup(struct tok_state *tok, int c)
70	4.71M	{
71	4.71M	if (c != EOF) {
72	4.52M	if (--tok->cur < tok->buf) {
73	0	Py_FatalError("tokenizer beginning of buffer");
74	0	}
75	4.52M	if ((int)(unsigned char)*tok->cur != Py_CHARMASK(c)) {
76	0	Py_FatalError("tok_backup: wrong character");
77	0	}
78	4.52M	tok->col_offset--;
79	4.52M	}
80	4.71M	}
81
82
83
84		/* Verify that the identifier follows PEP 3131. */
85		static int
86		verify_identifier(struct tok_state *tok)
87	12.8k	{
88	12.8k	if (tok->tok_extra_tokens) {
89	0	return 1;
90	0	}
91	12.8k	PyObject *s;
92	12.8k	if (tok->decoding_erred)
93	0	return 0;
94	12.8k	s = PyUnicode_DecodeUTF8(tok->start, tok->cur - tok->start, NULL);
95	12.8k	if (s == NULL) {
96	0	if (PyErr_ExceptionMatches(PyExc_UnicodeDecodeError)) {
97	0	tok->done = E_DECODE;
98	0	}
99	0	else {
100	0	tok->done = E_ERROR;
101	0	}
102	0	return 0;
103	0	}
104	12.8k	Py_ssize_t invalid = _PyUnicode_ScanIdentifier(s);
105	12.8k	assert(invalid >= 0);
106	12.8k	assert(PyUnicode_GET_LENGTH(s) > 0);
107	12.8k	if (invalid < PyUnicode_GET_LENGTH(s)) {
108	539	Py_UCS4 ch = PyUnicode_READ_CHAR(s, invalid);
109	539	if (invalid + 1 < PyUnicode_GET_LENGTH(s)) {
110		/* Determine the offset in UTF-8 encoded input */
111	378	Py_SETREF(s, PyUnicode_Substring(s, 0, invalid + 1));
112	378	if (s != NULL) {
113	378	Py_SETREF(s, PyUnicode_AsUTF8String(s));
114	378	}
115	378	if (s == NULL) {
116	0	tok->done = E_ERROR;
117	0	return 0;
118	0	}
119	378	tok->cur = (char *)tok->start + PyBytes_GET_SIZE(s);
120	378	}
121	539	Py_DECREF(s);
122	539	if (Py_UNICODE_ISPRINTABLE(ch)) {
123	281	_PyTokenizer_syntaxerror(tok, "invalid character '%c' (U+%04X)", ch, ch);
124	281	}
125	258	else {
126	258	_PyTokenizer_syntaxerror(tok, "invalid non-printable character U+%04X", ch);
127	258	}
128	539	return 0;
129	539	}
130	12.2k	Py_DECREF(s);
131	12.2k	return 1;
132	12.8k	}
133
134
135
136		static inline int
137	1.11k	tok_continuation_line(struct tok_state *tok) {
138	1.11k	int c = tok_nextc(tok);
139	1.11k	if (c == '\r') {
140	0	c = tok_nextc(tok);
141	0	}
142	1.11k	if (c != '\n') {
143	77	tok->done = E_LINECONT;
144	77	return -1;
145	77	}
146	1.03k	c = tok_nextc(tok);
147	1.03k	if (c == EOF) {
148	47	tok->done = E_EOF;
149	47	tok->cur = tok->inp;
150	47	return -1;
151	987	} else {
152	987	tok_backup(tok, c);
153	987	}
154	987	return c;
155	1.03k	}
156
157
158
159		int
160		_PyLexer_get_normal_mode(struct tok_state tok, tokenizer_mode current_tok, struct token *token)
161	2.17M	{
162	2.17M	int c;
163	2.17M	int blankline, nonascii;
164
165	2.17M	const char *p_start = NULL;
166	2.17M	const char *p_end = NULL;
167	2.28M	nextline:
168	2.28M	tok->start = NULL;
169	2.28M	tok->starting_col_offset = -1;
170	2.28M	blankline = 0;
171
172
173		/* Get indentation level */
174	2.28M	if (tok->atbol) {
175	373k	int col = 0;
176	373k	int altcol = 0;
177	373k	tok->atbol = 0;
178	373k	int cont_line_col = 0;
179	637k	for (;;) {
180	637k	c = tok_nextc(tok);
181	637k	if (c == ' ') {
182	261k	col++, altcol++;
183	261k	}
184	375k	else if (c == '\t') {
185	598	col = (col / tok->tabsize + 1) * tok->tabsize;
186	598	altcol = (altcol / ALTTABSIZE + 1) * ALTTABSIZE;
187	598	}
188	375k	else if (c == '\014') {/* Control-L (formfeed) */
189	1.07k	col = altcol = 0; /* For Emacs users */
190	1.07k	}
191	374k	else if (c == '\\') {
192		// Indentation cannot be split over multiple physical lines
193		// using backslashes. This means that if we found a backslash
194		// preceded by whitespace, the first one we find determines
195		// the level of indentation of whatever comes next.
196	645	cont_line_col = cont_line_col ? cont_line_col : col;
197	645	if ((c = tok_continuation_line(tok)) == -1) {
198	26	return MAKE_TOKEN(ERRORTOKEN);
199	26	}
200	645	}
201	373k	else if (c == EOF && PyErr_Occurred()) {
202	0	return MAKE_TOKEN(ERRORTOKEN);
203	0	}
204	373k	else {
205	373k	break;
206	373k	}
207	637k	}
208	373k	tok_backup(tok, c);
209	373k	if (c == '#' \|\| c == '\n' \|\| c == '\r') {
210		/* Lines with only whitespace and/or comments
211		shouldn't affect the indentation and are
212		not passed to the parser as NEWLINE tokens,
213		except totally empty lines in interactive
214		mode, which signal the end of a command group. */
215	73.5k	if (col == 0 && c == '\n' && tok->prompt != NULL) {
216	0	blankline = 0; /* Let it through */
217	0	}
218	73.5k	else if (tok->prompt != NULL && tok->lineno == 1) {
219		/* In interactive mode, if the first line contains
220		only spaces and/or a comment, let it through. */
221	0	blankline = 0;
222	0	col = altcol = 0;
223	0	}
224	73.5k	else {
225	73.5k	blankline = 1; /* Ignore completely */
226	73.5k	}
227		/* We can't jump back right here since we still
228		may need to skip to the end of a comment */
229	73.5k	}
230	373k	if (!blankline && tok->level == 0) {
231	265k	col = cont_line_col ? cont_line_col : col;
232	265k	altcol = cont_line_col ? cont_line_col : altcol;
233	265k	if (col == tok->indstack[tok->indent]) {
234		/* No change */
235	247k	if (altcol != tok->altindstack[tok->indent]) {
236	1	return MAKE_TOKEN(_PyTokenizer_indenterror(tok));
237	1	}
238	247k	}
239	17.7k	else if (col > tok->indstack[tok->indent]) {
240		/* Indent -- always one */
241	9.84k	if (tok->indent+1 >= MAXINDENT) {
242	0	tok->done = E_TOODEEP;
243	0	tok->cur = tok->inp;
244	0	return MAKE_TOKEN(ERRORTOKEN);
245	0	}
246	9.84k	if (altcol <= tok->altindstack[tok->indent]) {
247	1	return MAKE_TOKEN(_PyTokenizer_indenterror(tok));
248	1	}
249	9.84k	tok->pendin++;
250	9.84k	tok->indstack[++tok->indent] = col;
251	9.84k	tok->altindstack[tok->indent] = altcol;
252	9.84k	}
253	7.89k	else /* col < tok->indstack[tok->indent] */ {
254		/* Dedent -- any number, must be consistent */
255	17.1k	while (tok->indent > 0 &&
256	13.3k	col < tok->indstack[tok->indent]) {
257	9.26k	tok->pendin--;
258	9.26k	tok->indent--;
259	9.26k	}
260	7.89k	if (col != tok->indstack[tok->indent]) {
261	14	tok->done = E_DEDENT;
262	14	tok->cur = tok->inp;
263	14	return MAKE_TOKEN(ERRORTOKEN);
264	14	}
265	7.88k	if (altcol != tok->altindstack[tok->indent]) {
266	1	return MAKE_TOKEN(_PyTokenizer_indenterror(tok));
267	1	}
268	7.88k	}
269	265k	}
270	373k	}
271
272	2.28M	tok->start = tok->cur;
273	2.28M	tok->starting_col_offset = tok->col_offset;
274
275		/* Return pending indents/dedents */
276	2.28M	if (tok->pendin != 0) {
277	19.0k	if (tok->pendin < 0) {
278	9.22k	if (tok->tok_extra_tokens) {
279	60	p_start = tok->cur;
280	60	p_end = tok->cur;
281	60	}
282	9.22k	tok->pendin++;
283	9.22k	return MAKE_TOKEN(DEDENT);
284	9.22k	}
285	9.84k	else {
286	9.84k	if (tok->tok_extra_tokens) {
287	64	p_start = tok->buf;
288	64	p_end = tok->cur;
289	64	}
290	9.84k	tok->pendin--;
291	9.84k	return MAKE_TOKEN(INDENT);
292	9.84k	}
293	19.0k	}
294
295		/* Peek ahead at the next character */
296	2.26M	c = tok_nextc(tok);
297	2.26M	tok_backup(tok, c);
298
299	2.26M	again:
300	2.26M	tok->start = NULL;
301		/* Skip spaces */
302	2.70M	do {
303	2.70M	c = tok_nextc(tok);
304	2.70M	} while (c == ' ' \|\| c == '\t' \|\| c == '\014');
305
306		/* Set start of current token */
307	2.26M	tok->start = tok->cur == NULL ? NULL : tok->cur - 1;
308	2.26M	tok->starting_col_offset = tok->col_offset - 1;
309
310		/* Skip comment, unless it's a type comment */
311	2.26M	if (c == '#') {
312
313	32.0k	const char* p = NULL;
314	32.0k	const char prefix, type_start;
315	32.0k	int current_starting_col_offset;
316
317	992k	while (c != EOF && c != '\n' && c != '\r') {
318	960k	c = tok_nextc(tok);
319	960k	}
320
321	32.0k	if (tok->tok_extra_tokens) {
322	44	p = tok->start;
323	44	}
324
325	32.0k	if (tok->type_comments) {
326	0	p = tok->start;
327	0	current_starting_col_offset = tok->starting_col_offset;
328	0	prefix = type_comment_prefix;
329	0	while (*prefix && p < tok->cur) {
330	0	if (*prefix == ' ') {
331	0	while (p == ' ' \|\| p == '\t') {
332	0	p++;
333	0	current_starting_col_offset++;
334	0	}
335	0	} else if (prefix == p) {
336	0	p++;
337	0	current_starting_col_offset++;
338	0	} else {
339	0	break;
340	0	}
341
342	0	prefix++;
343	0	}
344
345		/* This is a type comment if we matched all of type_comment_prefix. */
346	0	if (!*prefix) {
347	0	int is_type_ignore = 1;
348		// +6 in order to skip the word 'ignore'
349	0	const char *ignore_end = p + 6;
350	0	const int ignore_end_col_offset = current_starting_col_offset + 6;
351	0	tok_backup(tok, c); /* don't eat the newline or EOF */
352
353	0	type_start = p;
354
355		/* A TYPE_IGNORE is "type: ignore" followed by the end of the token
356		* or anything ASCII and non-alphanumeric. */
357	0	is_type_ignore = (
358	0	tok->cur >= ignore_end && memcmp(p, "ignore", 6) == 0
359	0	&& !(tok->cur > ignore_end
360	0	&& ((unsigned char)ignore_end[0] >= 128 \|\| Py_ISALNUM(ignore_end[0]))));
361
362	0	if (is_type_ignore) {
363	0	p_start = ignore_end;
364	0	p_end = tok->cur;
365
366		/* If this type ignore is the only thing on the line, consume the newline also. */
367	0	if (blankline) {
368	0	tok_nextc(tok);
369	0	tok->atbol = 1;
370	0	}
371	0	return MAKE_TYPE_COMMENT_TOKEN(TYPE_IGNORE, ignore_end_col_offset, tok->col_offset);
372	0	} else {
373	0	p_start = type_start;
374	0	p_end = tok->cur;
375	0	return MAKE_TYPE_COMMENT_TOKEN(TYPE_COMMENT, current_starting_col_offset, tok->col_offset);
376	0	}
377	0	}
378	0	}
379	32.0k	if (tok->tok_extra_tokens) {
380	44	tok_backup(tok, c); /* don't eat the newline or EOF */
381	44	p_start = p;
382	44	p_end = tok->cur;
383	44	tok->comment_newline = blankline;
384	44	return MAKE_TOKEN(COMMENT);
385	44	}
386	32.0k	}
387
388	2.26M	if (tok->done == E_INTERACT_STOP) {
389	0	return MAKE_TOKEN(ENDMARKER);
390	0	}
391
392		/* Check for EOF and errors now */
393	2.26M	if (c == EOF) {
394	97.5k	if (tok->level) {
395	4.31k	return MAKE_TOKEN(ERRORTOKEN);
396	4.31k	}
397	93.2k	return MAKE_TOKEN(tok->done == E_EOF ? ENDMARKER : ERRORTOKEN);
398	97.5k	}
399
400		/* Identifier (most frequent token!) */
401	2.17M	nonascii = 0;
402	2.17M	if (is_potential_identifier_start(c)) {
403		/* Process the various legal combinations of b"", r"", u"", and f"". */
404	715k	int saw_b = 0, saw_r = 0, saw_u = 0, saw_f = 0, saw_t = 0;
405	905k	while (1) {
406	905k	if (!saw_b && (c == 'b' \|\| c == 'B')) {
407	18.8k	saw_b = 1;
408	18.8k	}
409		/* Since this is a backwards compatibility support literal we don't
410		want to support it in arbitrary order like byte literals. */
411	886k	else if (!saw_u && (c == 'u'\|\| c == 'U')) {
412	86.5k	saw_u = 1;
413	86.5k	}
414		/* ur"" and ru"" are not supported */
415	800k	else if (!saw_r && (c == 'r' \|\| c == 'R')) {
416	26.3k	saw_r = 1;
417	26.3k	}
418	773k	else if (!saw_f && (c == 'f' \|\| c == 'F')) {
419	46.1k	saw_f = 1;
420	46.1k	}
421	727k	else if (!saw_t && (c == 't' \|\| c == 'T')) {
422	39.3k	saw_t = 1;
423	39.3k	}
424	688k	else {
425	688k	break;
426	688k	}
427	217k	c = tok_nextc(tok);
428	217k	if (c == '"' \|\| c == '\'') {
429		// Raise error on incompatible string prefixes:
430	26.6k	int status = _PyLexer_check_string_prefixes(
431	26.6k	tok, saw_b, saw_r, saw_u, saw_f, saw_t);
432	26.6k	if (status < 0) {
433	9	return MAKE_TOKEN(ERRORTOKEN);
434	9	}
435
436		// Handle valid f or t string creation:
437	26.6k	if (saw_f \|\| saw_t) {
438	22.4k	return _PyLexer_scan_fstring_start(tok, token, c);
439	22.4k	}
440	4.23k	return _PyLexer_scan_string(tok, token, c);
441	26.6k	}
442	217k	}
443	3.12M	while (is_potential_identifier_char(c)) {
444	2.43M	if (c >= 128) {
445	191k	nonascii = 1;
446	191k	}
447	2.43M	c = tok_nextc(tok);
448	2.43M	}
449	688k	tok_backup(tok, c);
450	688k	if (nonascii && !verify_identifier(tok)) {
451	539	return MAKE_TOKEN(ERRORTOKEN);
452	539	}
453
454	687k	p_start = tok->start;
455	687k	p_end = tok->cur;
456
457	687k	return MAKE_TOKEN(NAME);
458	688k	}
459
460	1.45M	if (c == '\r') {
461	0	c = tok_nextc(tok);
462	0	}
463
464		/* Newline */
465	1.45M	if (c == '\n') {
466	273k	tok->atbol = 1;
467	273k	if (blankline \|\| tok->level > 0) {
468	108k	if (tok->tok_extra_tokens) {
469	128	if (tok->comment_newline) {
470	24	tok->comment_newline = 0;
471	24	}
472	128	p_start = tok->start;
473	128	p_end = tok->cur;
474	128	return MAKE_TOKEN(NL);
475	128	}
476	107k	goto nextline;
477	108k	}
478	165k	if (tok->comment_newline && tok->tok_extra_tokens) {
479	12	tok->comment_newline = 0;
480	12	p_start = tok->start;
481	12	p_end = tok->cur;
482	12	return MAKE_TOKEN(NL);
483	12	}
484	165k	p_start = tok->start;
485	165k	p_end = tok->cur - 1; /* Leave '\n' out of the string */
486	165k	tok->cont_line = 0;
487	165k	return MAKE_TOKEN(NEWLINE);
488	165k	}
489
490		/* Period or number starting with period? */
491	1.18M	if (c == '.') {
492	25.3k	c = tok_nextc(tok);
493	25.3k	if (Py_ISDIGIT(c)) {
494	2.80k	return _PyLexer_scan_number(tok, token, c, 1);
495	22.5k	} else if (c == '.') {
496	1.84k	c = tok_nextc(tok);
497	1.84k	if (c == '.') {
498	1.04k	p_start = tok->start;
499	1.04k	p_end = tok->cur;
500	1.04k	return MAKE_TOKEN(ELLIPSIS);
501	1.04k	}
502	809	else {
503	809	tok_backup(tok, c);
504	809	}
505	809	tok_backup(tok, '.');
506	809	}
507	20.7k	else {
508	20.7k	tok_backup(tok, c);
509	20.7k	}
510	21.5k	p_start = tok->start;
511	21.5k	p_end = tok->cur;
512	21.5k	return MAKE_TOKEN(DOT);
513	25.3k	}
514
515
516		/* Number */
517	1.15M	if (Py_ISDIGIT(c)) {
518	96.2k	return _PyLexer_scan_number(tok, token, c, 0);
519	96.2k	}
520
521		/* String */
522	1.06M	if (c == '\'' \|\| c == '"') {
523	34.3k	return _PyLexer_scan_string(tok, token, c);
524	34.3k	}
525
526		/* Line continuation */
527	1.02M	if (c == '\\') {
528	466	if ((c = tok_continuation_line(tok)) == -1) {
529	98	return MAKE_TOKEN(ERRORTOKEN);
530	98	}
531	368	tok->cont_line = 1;
532	368	goto again; /* Read next line */
533	466	}
534
535		/* Punctuation character */
536	1.02M	int is_punctuation = (c == ':' \|\| c == '}' \|\| c == '!' \|\| c == '{');
537	1.02M	if (is_punctuation && INSIDE_FSTRING(tok) && INSIDE_FSTRING_EXPR(current_tok)) {
538		/* This code block gets executed before the curly_bracket_depth is incremented
539		* by the `{` case, so for ensuring that we are on the 0th level, we need
540		* to adjust it manually */
541	66.0k	int cursor = current_tok->curly_bracket_depth - (c != '{');
542	66.0k	int in_format_spec = current_tok->in_format_spec;
543	66.0k	int cursor_in_format_with_debug =
544	66.0k	cursor == 1 && (current_tok->in_debug \|\| in_format_spec);
545	66.0k	int cursor_valid = cursor == 0 \|\| cursor_in_format_with_debug;
546	66.0k	if ((cursor_valid) && !_PyLexer_update_ftstring_expr(tok, c)) {
547	0	return MAKE_TOKEN(ENDMARKER);
548	0	}
549	66.0k	if ((cursor_valid) && c != '{' && _PyLexer_set_ftstring_expr(tok, token, c)) {
550	0	return MAKE_TOKEN(ERRORTOKEN);
551	0	}
552
553	66.0k	if (c == ':' && cursor == current_tok->curly_bracket_expr_start_depth) {
554	5.09k	current_tok->kind = TOK_FSTRING_MODE;
555	5.09k	current_tok->in_format_spec = 1;
556	5.09k	p_start = tok->start;
557	5.09k	p_end = tok->cur;
558	5.09k	return MAKE_TOKEN(_PyToken_OneChar(c));
559	5.09k	}
560	66.0k	}
561
562		/* Check for two-character token */
563	1.02M	{
564	1.02M	int c2 = tok_nextc(tok);
565	1.02M	int current_token = _PyToken_TwoChars(c, c2);
566	1.02M	if (current_token != OP) {
567	25.5k	int c3 = tok_nextc(tok);
568	25.5k	int current_token3 = _PyToken_ThreeChars(c, c2, c3);
569	25.5k	if (current_token3 != OP) {
570	1.67k	current_token = current_token3;
571	1.67k	}
572	23.8k	else {
573	23.8k	tok_backup(tok, c3);
574	23.8k	}
575	25.5k	p_start = tok->start;
576	25.5k	p_end = tok->cur;
577	25.5k	return MAKE_TOKEN(current_token);
578	25.5k	}
579	995k	tok_backup(tok, c2);
580	995k	}
581
582		/* Keep track of parentheses nesting level */
583	0	switch (c) {
584	72.0k	case '(':
585	109k	case '[':
586	160k	case '{':
587	160k	if (tok->level >= MAXLEVEL) {
588	16	return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "too many nested parentheses"));
589	16	}
590	160k	tok->parenstack[tok->level] = c;
591	160k	tok->parenlinenostack[tok->level] = tok->lineno;
592	160k	tok->parencolstack[tok->level] = (int)(tok->start - tok->line_start);
593	160k	tok->level++;
594	160k	if (INSIDE_FSTRING(tok)) {
595	39.7k	current_tok->curly_bracket_depth++;
596	39.7k	}
597	160k	break;
598	40.9k	case ')':
599	47.9k	case ']':
600	75.5k	case '}':
601	75.5k	if (INSIDE_FSTRING(tok) && !current_tok->curly_bracket_depth && c == '}') {
602	47	return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok,
603	47	"%c-string: single '}' is not allowed", TOK_GET_STRING_PREFIX(tok)));
604	47	}
605	75.5k	if (!tok->tok_extra_tokens && !tok->level) {
606	234	return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "unmatched '%c'", c));
607	234	}
608	75.2k	if (tok->level > 0) {
609	75.2k	tok->level--;
610	75.2k	int opening = tok->parenstack[tok->level];
611	75.2k	if (!tok->tok_extra_tokens && !((opening == '(' && c == ')') \|\|
612	34.5k	(opening == '[' && c == ']') \|\|
613	27.5k	(opening == '{' && c == '}'))) {
614		/* If the opening bracket belongs to an f-string's expression
615		part (e.g. f"{)}") and the closing bracket is an arbitrary
616		nested expression, then instead of matching a different
617		syntactical construct with it; we'll throw an unmatched
618		parentheses error. */
619	60	if (INSIDE_FSTRING(tok) && opening == '{') {
620	6	assert(current_tok->curly_bracket_depth >= 0);
621	6	int previous_bracket = current_tok->curly_bracket_depth - 1;
622	6	if (previous_bracket == current_tok->curly_bracket_expr_start_depth) {
623	3	return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok,
624	3	"%c-string: unmatched '%c'", TOK_GET_STRING_PREFIX(tok), c));
625	3	}
626	6	}
627	57	if (tok->parenlinenostack[tok->level] != tok->lineno) {
628	7	return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok,
629	7	"closing parenthesis '%c' does not match "
630	7	"opening parenthesis '%c' on line %d",
631	7	c, opening, tok->parenlinenostack[tok->level]));
632	7	}
633	50	else {
634	50	return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok,
635	50	"closing parenthesis '%c' does not match "
636	50	"opening parenthesis '%c'",
637	50	c, opening));
638	50	}
639	57	}
640	75.2k	}
641
642	75.2k	if (INSIDE_FSTRING(tok)) {
643	24.4k	current_tok->curly_bracket_depth--;
644	24.4k	if (current_tok->curly_bracket_depth < 0) {
645	1	return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "%c-string: unmatched '%c'",
646	1	TOK_GET_STRING_PREFIX(tok), c));
647	1	}
648	24.4k	if (c == '}' && current_tok->curly_bracket_depth == current_tok->curly_bracket_expr_start_depth) {
649	23.0k	current_tok->curly_bracket_expr_start_depth--;
650	23.0k	current_tok->kind = TOK_FSTRING_MODE;
651	23.0k	current_tok->in_format_spec = 0;
652	23.0k	current_tok->in_debug = 0;
653	23.0k	}
654	24.4k	}
655	75.2k	break;
656	758k	default:
657	758k	break;
658	995k	}
659
660	994k	if (!Py_UNICODE_ISPRINTABLE(c)) {
661	432	return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid non-printable character U+%04X", c));
662	432	}
663
664	994k	if( c == '=' && INSIDE_FSTRING_EXPR_AT_TOP(current_tok)) {
665	6.89k	current_tok->in_debug = 1;
666	6.89k	}
667
668		/* Punctuation character */
669	994k	p_start = tok->start;
670	994k	p_end = tok->cur;
671	994k	return MAKE_TOKEN(_PyToken_OneChar(c));
672	994k	}
673
674
675		static int
676		tok_get(struct tok_state tok, struct token token)
677	2.22M	{
678	2.22M	tokenizer_mode *current_tok = TOK_GET_MODE(tok);
679	2.22M	if (current_tok->kind == TOK_REGULAR_MODE) {
680	2.16M	return _PyLexer_get_normal_mode(tok, current_tok, token);
681	2.16M	} else {
682	60.9k	return _PyLexer_get_fstring_mode(tok, current_tok, token);
683	60.9k	}
684	2.22M	}
685
686		int
687		_PyTokenizer_Get(struct tok_state tok, struct token token)
688	2.22M	{
689	2.22M	int result = tok_get(tok, token);
690	2.22M	if (tok->decoding_erred) {
691	0	result = ERRORTOKEN;
692	0	tok->done = E_DECODE;
693	0	}
694	2.22M	return result;
695	2.22M	}