/src/cpython/Parser/lexer/string.c

Source
#include "Python.h"
#include "pycore_token.h"
#include "errcode.h"

#include "lexer_internal.h"
#include "../tokenizer/helpers.h"

#define MAKE_TOKEN(token_type) _PyLexer_token_setup(tok, token, token_type, p_start, p_end)

static int
string_error_token(struct tok_state *tok, struct token *token,
                   _PyTok_Off start, _PyTok_Loc location)
{
    tok->diagnostic = (_PyTokenizer_Diagnostic){
        .location = {location.lineno, location.byte_col + 1},
        .text_span = {start - location.byte_col, tok->inp},
    };
    int type = _PyLexer_token_setup(tok, token, ERRORTOKEN, -1, -1);
    token->start_loc = location;
    token->end_loc = (_PyTok_Loc){location.lineno, -1};
    return type;
}

static int
incomplete_string_token(struct tok_state *tok, struct token *token,
                        int type, _PyTok_Off start, _PyTok_Loc location)
{
    string_error_token(tok, token, start, location);
    tok->done = E_INCOMPLETE;
    token->is_incomplete = 1;
    return type;
}

int
_PyLexer_record_ftstring_comment(struct tok_state *tok, ftstring_state *state,
                                 _PyTok_Off start, _PyTok_Off end)
{
    assert(state == _PyLexer_CurrentFTString(tok) && state->mode == FTSTRING_MODE_EXPRESSION);
    if (state->expr_span.end >= 0) {
        return 0;
    }
    assert(state->expr_span.start >= 0);
    tokenizer_comments *comments = state->comments;
    if (comments == NULL || comments->count == comments->capacity) {
        Py_ssize_t max_capacity = (PY_SSIZE_T_MAX -
            (Py_ssize_t)sizeof(*comments)) /
            (Py_ssize_t)sizeof(*comments->spans);
        if (comments != NULL && comments->capacity > max_capacity / 2) {
            PyErr_NoMemory();
            return -1;
        }
        Py_ssize_t capacity = comments == NULL ? 4 : comments->capacity * 2;
        size_t size = sizeof(*comments) +
            (size_t)capacity * sizeof(*comments->spans);
        tokenizer_comments *resized = PyMem_Realloc(comments, size);
        if (resized == NULL) {
            PyErr_NoMemory();
            return -1;
        }
        comments = resized;
        if (state->comments == NULL) {
            comments->count = 0;
        }
        comments->capacity = capacity;
        state->comments = comments;
    }
    comments->spans[comments->count++] =
        (_PyTok_Span){start, end};
    return 0;
}

static int
finish_ftstring_expr(struct tok_state *tok, ftstring_state *state,
                     struct token *token)
{
    assert(token != NULL && state == _PyLexer_CurrentFTString(tok));
    assert(state->mode == FTSTRING_MODE_EXPRESSION && tok->start >= 0);

    if (state->expr_span.end >= 0) {
        return 0;
    }
    assert(state->expr_span.start >= 0);
    state->expr_span.end = tok->start;
    int tstring_interpolation = _PyLexer_IsTString(state->kind) &&
        state->replacement_depth == 1;
    if (!(state->debug_expr || tstring_interpolation) || token->metadata) {
        return 0;
    }
    tokenizer_comments *comments = state->comments;
    PyObject *res;
    if (comments != NULL && comments->count > 0) {
        Py_ssize_t stripped_size =
            state->expr_span.end - state->expr_span.start;
        Py_ssize_t comment_count = 0;
        for (Py_ssize_t i = 0; i < comments->count; i++) {
            _PyTok_Span comment = comments->spans[i];
            assert(_PyTok_SpanIsValid(comment));
            assert(comment.start >= (i == 0 ? state->expr_span.start
                                           : comments->spans[i - 1].end));
            if (comment.start >= state->expr_span.end) {
                break;
            }
            assert(comment.end <= state->expr_span.end);
            stripped_size -= comment.end - comment.start;
            comment_count++;
        }
        char *stripped = PyMem_Malloc((size_t)stripped_size);
        if (stripped == NULL) {
            PyErr_NoMemory();
            return -1;
        }
        _PyTok_Span kept = {state->expr_span.start, state->expr_span.start};
        Py_ssize_t stripped_len = 0;
        for (Py_ssize_t i = 0; i < comment_count; i++) {
            kept.end = comments->spans[i].start;
            Py_ssize_t length;
            const char *text = _PyTok_SourceSpanView(
                &tok->source, kept, &length);
            memcpy(stripped + stripped_len, text, (size_t)length);
            stripped_len += length;
            kept.start = comments->spans[i].end;
        }
        kept.end = state->expr_span.end;
        Py_ssize_t length;
        const char *text = _PyTok_SourceSpanView(&tok->source, kept, &length);
        memcpy(stripped + stripped_len, text, (size_t)length);
        res = PyUnicode_DecodeUTF8(stripped, stripped_size, NULL);
        PyMem_Free(stripped);
    }
    else {
        Py_ssize_t expr_len;
        const char *expr = _PyTok_SourceSpanView(
            &tok->source, state->expr_span, &expr_len);
        res = PyUnicode_DecodeUTF8(expr, expr_len, NULL);
    }

    if (!res) {
        return -1;
    }
    token->metadata = res;
    return 0;
}

static int
begin_ftstring_expr(struct tok_state *tok, ftstring_state *state,
                    _PyTok_Off expr_start)
{
    assert(state->mode != FTSTRING_MODE_EXPRESSION);
    state->expr_span = (_PyTok_Span){expr_start, -1};
    if (state->comments != NULL) {
        state->comments->count = 0;
    }
    if (state->replacement_depth >= MAX_EXPR_NESTING) {
        _PyTokenizer_syntaxerror(
            tok, "%c-string: expressions nested too deeply",
            _PyLexer_StringPrefix(state->kind));
        return -1;
    }
    state->replacement_depth++;
    state->mode = FTSTRING_MODE_EXPRESSION;
    state->debug_expr = 0;
    return 0;
}

int
_PyLexer_ftstring_punctuation(struct tok_state *tok, ftstring_state *state,
                              struct token *token, int c)
{
    assert(state->mode == FTSTRING_MODE_EXPRESSION);
    assert(c == ':' || c == '}' || c == '!');
    if (_PyLexer_FTStringBracketDepth(tok, state) != state->replacement_depth) {
        return 0;
    }
    if (c == '!') {
        int next = tok_nextc(tok);
        tok_backup(tok, next);
        if (next == '=') {
            return 0;
        }
    }
    if (finish_ftstring_expr(tok, state, token) < 0) {
        return -1;
    }
    if (c == ':') {
        state->mode = FTSTRING_MODE_FORMAT_SPEC;
        return COLON;
    }
    return 0;
}

int
_PyLexer_close_ftstring_expr(struct tok_state *tok, ftstring_state *state,
                             int c)
{
    assert(state->mode == FTSTRING_MODE_EXPRESSION);
    assert(c == ')' || c == ']' || c == '}');
    int depth = _PyLexer_FTStringBracketDepth(tok, state);
    if (depth < 0) {
        _PyTokenizer_syntaxerror(tok, "%c-string: unmatched '%c'",
                                _PyLexer_StringPrefix(state->kind), c);
        return -1;
    }
    if (c == '}' && depth == state->replacement_depth - 1) {
        state->replacement_depth--;
        state->mode = FTSTRING_MODE_MIDDLE;
        state->debug_expr = 0;
    }
    return 0;
}

void
_PyLexer_mark_ftstring_debug(struct tok_state *tok, ftstring_state *state)
{
    assert(state->mode == FTSTRING_MODE_EXPRESSION);
    if (_PyLexer_FTStringBracketDepth(tok, state) == state->replacement_depth) {
        state->debug_expr = 1;
    }
}

int
_PyLexer_check_string_prefixes(struct tok_state *tok,
                                             int saw_b, int saw_r, int saw_u,
                                             int saw_f, int saw_t) {
    // Supported: rb, rf, rt (in any order)
    // Unsupported: ub, ur, uf, ut, bf, bt, ft (in any order)

#define RETURN_SYNTAX_ERROR(PREFIX1, PREFIX2)                             \
    do {                                                                  \
        (void)_PyTokenizer_syntaxerror_known_range(                       \
            tok, (int)(tok->start + 1 - tok->line_start),                 \
            (int)(tok->cur - tok->line_start),                            \
            "'" PREFIX1 "' and '" PREFIX2 "' prefixes are incompatible"); \
        return -1;                                                        \
    } while (0)

    if (saw_u && saw_b) {
        RETURN_SYNTAX_ERROR("u", "b");
    }
    if (saw_u && saw_r) {
        RETURN_SYNTAX_ERROR("u", "r");
    }
    if (saw_u && saw_f) {
        RETURN_SYNTAX_ERROR("u", "f");
    }
    if (saw_u && saw_t) {
        RETURN_SYNTAX_ERROR("u", "t");
    }

    if (saw_b && saw_f) {
        RETURN_SYNTAX_ERROR("b", "f");
    }
    if (saw_b && saw_t) {
        RETURN_SYNTAX_ERROR("b", "t");
    }

    if (saw_f && saw_t) {
        RETURN_SYNTAX_ERROR("f", "t");
    }

#undef RETURN_SYNTAX_ERROR

    return 0;
}

int
_PyLexer_scan_fstring_start(struct tok_state *tok, struct token *token,
                            int c, ftstring_kind kind)
{
    _PyTok_Off p_start = -1;
    _PyTok_Off p_end = -1;

    int quote = c;
    int quote_size = 1;             /* 1 or 3 */

    /* Find the quote size and start of string */
    int after_quote = tok_nextc(tok);
    if (after_quote == quote) {
        int after_after_quote = tok_nextc(tok);
        if (after_after_quote == quote) {
            quote_size = 3;
        }
        else {
            // TODO: Check this
            tok_backup(tok, after_after_quote);
            tok_backup(tok, after_quote);
        }
    }
    if (after_quote != quote) {
        tok_backup(tok, after_quote);
    }

    p_start = tok->start;
    p_end = tok->cur;
    ftstring_state *state = _PyLexer_PushFTString(tok);
    if (state == NULL) {
        return MAKE_TOKEN(ERRORTOKEN);
    }
    state->mode = FTSTRING_MODE_MIDDLE;
    state->quote = quote;
    state->quote_size = quote_size;
    state->paren_level = tok->level;
    state->start = tok->start;
    state->start_loc = tok->start_loc;
    state->expr_span = (_PyTok_Span){-1, -1};

    state->kind = kind;
    return _PyLexer_IsTString(kind)
        ? MAKE_TOKEN(TSTRING_START) : MAKE_TOKEN(FSTRING_START);
}

int
_PyLexer_scan_string(struct tok_state *tok, struct token *token, int c)
{
    _PyTok_Off p_start = -1;
    _PyTok_Off p_end = -1;

    int quote = c;
    int quote_size = 1;             /* 1 or 3 */
    int end_quote_size = 0;
    int has_escaped_quote = 0;

    /* Find the quote size and start of string */
    c = tok_nextc(tok);
    if (c == quote) {
        c = tok_nextc(tok);
        if (c == quote) {
            quote_size = 3;
        }
        else {
            end_quote_size = 1;     /* empty string found */
        }
    }
    if (c != quote) {
        tok_backup(tok, c);
    }

    /* Get rest of string */
    while (end_quote_size != quote_size) {
        c = tok_nextc(tok);
        if (tok->done == E_ERROR) {
            return MAKE_TOKEN(ERRORTOKEN);
        }
        if (tok->done == E_DECODE) {
            break;
        }
        if (c == EOF || (quote_size == 1 && c == '\n')) {
            if (tok_failed(tok)) {
                return MAKE_TOKEN(ERRORTOKEN);
            }
            if (c == EOF && tok->allow_incomplete_input) {
                return incomplete_string_token(tok, token, STRING,
                                               tok->start, tok->start_loc);
            }
            int end_lineno = tok->lineno;
            _PyTok_Loc location = tok->start_loc;
            const char *line = _PyTok_SourcePointer(
                &tok->source, tok->start - location.byte_col);
            Py_ssize_t cursor_offset = (Py_ssize_t)location.byte_col + 1;

            const ftstring_state *state = _PyLexer_CurrentFTString(tok);
            if (state != NULL) {
                /* A matching quote may have been intended to close the
                 * surrounding formatted string instead of opening a string
                 * inside a replacement field. */
                if (state->quote == quote && state->quote_size == quote_size) {
                    int level = state->paren_level + state->replacement_depth - 1;
                    assert(level >= 0 && level < tok->level);
                    assert(tok->parenstack[level] == '{');
                    int lineno = tok->parenlinenostack[level];
                    if (lineno != location.lineno) {
                        _PyTokenizer_syntaxerror_at(
                            tok, line, cursor_offset, location.lineno, -1, -1,
                            "%c-string: expecting '}' to close '{' on line %d",
                            _PyLexer_StringPrefix(state->kind), lineno);
                    }
                    else {
                        _PyTokenizer_syntaxerror_at(
                            tok, line, cursor_offset, location.lineno, -1, -1,
                            "%c-string: expecting '}'",
                            _PyLexer_StringPrefix(state->kind));
                    }
                    return string_error_token(tok, token, tok->start, location);
                }
            }

            if (quote_size == 3) {
                _PyTokenizer_syntaxerror_at(
                    tok, line, cursor_offset, location.lineno, -1, -1,
                    "unterminated triple-quoted string literal"
                    " (detected at line %d)", end_lineno);
                if (c != '\n') {
                    tok->done = E_EOFS;
                }
                return string_error_token(tok, token, tok->start, location);
            }
            else {
                if (has_escaped_quote) {
                    _PyTokenizer_syntaxerror_at(
                        tok, line, cursor_offset, location.lineno, -1, -1,
                        "unterminated string literal (detected at line %d); "
                        "perhaps you escaped the end quote?",
                        end_lineno
                    );
                } else {
                    _PyTokenizer_syntaxerror_at(
                        tok, line, cursor_offset, location.lineno, -1, -1,
                        "unterminated string literal (detected at line %d)", end_lineno
                    );
                }
                if (c != '\n') {
                    tok->done = E_EOLS;
                }
                return string_error_token(tok, token, tok->start, location);
            }
        }
        if (c == quote) {
            end_quote_size += 1;
        }
        else {
            end_quote_size = 0;
            if (c == '\\') {
                c = tok_nextc(tok);  /* skip escaped char */
                if (c == quote) {  /* but record whether the escaped char was a quote */
                    has_escaped_quote = 1;
                }
                if (c == '\r') {
                    c = tok_nextc(tok);
                }
            }
        }
    }

    p_start = tok->start;
    p_end = tok->cur;
    return MAKE_TOKEN(STRING);
}

int
_PyLexer_get_ftstring(struct tok_state *tok, ftstring_state *current, struct token *token)
{
    assert(current == _PyLexer_CurrentFTString(tok) && current->mode != FTSTRING_MODE_EXPRESSION);
    assert((current->quote_size == 1 || current->quote_size == 3) &&
           current->replacement_depth <= MAX_EXPR_NESTING);
    _PyTok_Off p_start = -1;
    _PyTok_Off p_end = -1;
    int token_type;
    int end_quote_size = 0;
    int unicode_escape = 0;
    int quote = current->quote;
    int quote_size = current->quote_size;
    int in_format_spec = current->mode == FTSTRING_MODE_FORMAT_SPEC;
    int raw = _PyLexer_IsRawString(current->kind);

    tok->start = tok->cur;
    tok->start_loc = (_PyTok_Loc){tok->lineno, _PyLexer_ByteColumn(tok)};

    while (end_quote_size != quote_size) {
        int c = tok_nextc(tok);
        if (tok->done == E_ERROR || tok->done == E_DECODE) {
            return MAKE_TOKEN(ERRORTOKEN);
        }

        if (c == EOF || (quote_size == 1 && c == '\n')) {
            if (tok_failed(tok)) {
                return MAKE_TOKEN(ERRORTOKEN);
            }
            if (c == EOF && tok->allow_incomplete_input) {
                int type = incomplete_string_token(tok, token,
                    FTSTRING_MIDDLE(current), current->start, current->start_loc);
                // Preserve the actual prefix for incremental escape decoding.
                _PyTok_Off end = tok->cur;
                if (end > tok->start && tok->implicit_newline) {
                    end--;
                }
                token->span = (_PyTok_Span){tok->start, end};
                token->start_loc = tok->start_loc;
                token->end_loc = (_PyTok_Loc){tok->lineno, (int)(end - tok->line_start)};
                token->is_raw = raw;
                return type;
            }

            if (in_format_spec && c == '\n') {
                return MAKE_TOKEN(_PyTokenizer_syntaxerror(
                    tok,
                    "%c-string: newlines are not allowed in format specifiers for single quoted %c-strings",
                    _PyLexer_StringPrefix(current->kind),
                    _PyLexer_StringPrefix(current->kind)));
            }

            int end_lineno = tok->lineno;
            _PyTok_Loc location = current->start_loc;
            const char *line = _PyTok_SourcePointer(
                &tok->source, current->start - location.byte_col);
            Py_ssize_t cursor_offset = (Py_ssize_t)location.byte_col + 1;

            if (quote_size == 3) {
                _PyTokenizer_syntaxerror_at(
                    tok, line, cursor_offset, location.lineno, -1, -1,
                    "unterminated triple-quoted %c-string literal"
                    " (detected at line %d)",
                    _PyLexer_StringPrefix(current->kind), end_lineno);
                if (c != '\n') {
                    tok->done = E_EOFS;
                }
                return string_error_token(tok, token,
                    current->start, location);
            }
            else {
                _PyTokenizer_syntaxerror_at(
                    tok, line, cursor_offset, location.lineno, -1, -1,
                    "unterminated %c-string literal (detected at line %d)",
                    _PyLexer_StringPrefix(current->kind), end_lineno);
                return string_error_token(tok, token,
                    current->start, location);
            }
        }

        if (c == quote) {
            end_quote_size += 1;
            continue;
        } else {
            end_quote_size = 0;
        }

        if (c == '{') {
            int peek = tok_nextc(tok);
            if (peek != '{' || in_format_spec) {
                tok_backup(tok, peek);
                _PyTok_Off expr_start = tok->cur;
                tok_backup(tok, c);
                if (begin_ftstring_expr(tok, current, expr_start) < 0) {
                    return MAKE_TOKEN(ERRORTOKEN);
                }
                p_start = tok->start;
                p_end = tok->cur;
                if (p_start == p_end) {
                    return _PyLexer_get_normal(tok, current, token);
                }
            } else {
                p_start = tok->start;
                p_end = tok->cur - 1;
            }
            goto emit_middle;
        } else if (c == '}') {
            if (unicode_escape) {
                p_start = tok->start;
                p_end = tok->cur;
                goto emit_middle;
            }
            int peek = tok_nextc(tok);

            int bracket_depth = _PyLexer_FTStringBracketDepth(tok, current);
            if (peek == '}' && !in_format_spec && bracket_depth == 0) {
                p_start = tok->start;
                p_end = tok->cur - 1;
            }
            else {
                tok_backup(tok, peek);
                if (!in_format_spec && bracket_depth == 0) {
                    if (tok->start == tok->cur - 1) {
                        return MAKE_TOKEN(_PyTokenizer_syntaxerror(
                            tok, "%c-string: single '}' is not allowed",
                            _PyLexer_StringPrefix(current->kind)));
                    }
                    tok_backup(tok, c);
                    p_start = tok->start;
                    p_end = tok->cur;
                    goto emit_middle;
                }
                tok_backup(tok, c);
                current->mode = FTSTRING_MODE_EXPRESSION;
                p_start = tok->start;
                p_end = tok->cur;
            }
            goto emit_middle;
        } else if (c == '\\') {
            int peek = tok_nextc(tok);
            if (peek == '\r') {
                peek = tok_nextc(tok);
            }
            if (peek == '{' || peek == '}') {
                if (!raw) {
                    if (_PyTokenizer_warn_invalid_escape_sequence(tok, peek)) {
                        return MAKE_TOKEN(ERRORTOKEN);
                    }
                }
                tok_backup(tok, peek);
                continue;
            }

            if (!raw) {
                if (peek == 'N') {
                    /* Handle named unicode escapes (\N{BULLET}) */
                    peek = tok_nextc(tok);
                    if (peek == '{') {
                        unicode_escape = 1;
                    } else {
                        tok_backup(tok, peek);
                    }
                }
            }
        }
    }

    p_start = tok->start;
    p_end = tok->cur;
    if (p_end - quote_size == p_start) {
        int end_token = FTSTRING_END(current);
        _PyLexer_PopFTString(tok);
        return MAKE_TOKEN(end_token);
    }
    for (int i = 0; i < quote_size; i++) {
        tok_backup(tok, quote);
    }
    p_end = tok->cur;
emit_middle:
    token_type = MAKE_TOKEN(FTSTRING_MIDDLE(current));
    token->is_raw = raw;
    return token_type;
}

Coverage Report

Created: 2026-10-31 06:21

Line	Count	Source
1		#include "Python.h"
2		#include "pycore_token.h"
3		#include "errcode.h"
4
5		#include "lexer_internal.h"
6		#include "../tokenizer/helpers.h"
7
8	105k	#define MAKE_TOKEN(token_type) _PyLexer_token_setup(tok, token, token_type, p_start, p_end)
9
10		static int
11		string_error_token(struct tok_state tok, struct token token,
12		_PyTok_Off start, _PyTok_Loc location)
13	686	{
14	686	tok->diagnostic = (_PyTokenizer_Diagnostic){
15	686	.location = {location.lineno, location.byte_col + 1},
16	686	.text_span = {start - location.byte_col, tok->inp},
17	686	};
18	686	int type = _PyLexer_token_setup(tok, token, ERRORTOKEN, -1, -1);
19	686	token->start_loc = location;
20	686	token->end_loc = (_PyTok_Loc){location.lineno, -1};
21	686	return type;
22	686	}
23
24		static int
25		incomplete_string_token(struct tok_state tok, struct token token,
26		int type, _PyTok_Off start, _PyTok_Loc location)
27	0	{
28	0	string_error_token(tok, token, start, location);
29	0	tok->done = E_INCOMPLETE;
30	0	token->is_incomplete = 1;
31	0	return type;
32	0	}
33
34		int
35		_PyLexer_record_ftstring_comment(struct tok_state tok, ftstring_state state,
36		_PyTok_Off start, _PyTok_Off end)
37	1.53k	{
38	1.53k	assert(state == _PyLexer_CurrentFTString(tok) && state->mode == FTSTRING_MODE_EXPRESSION);
39	1.53k	if (state->expr_span.end >= 0) {
40	215	return 0;
41	215	}
42	1.53k	assert(state->expr_span.start >= 0);
43	1.31k	tokenizer_comments *comments = state->comments;
44	1.31k	if (comments == NULL \|\| comments->count == comments->capacity) {
45	375	Py_ssize_t max_capacity = (PY_SSIZE_T_MAX -
46	375	(Py_ssize_t)sizeof(*comments)) /
47	375	(Py_ssize_t)sizeof(*comments->spans);
48	375	if (comments != NULL && comments->capacity > max_capacity / 2) {
49	0	PyErr_NoMemory();
50	0	return -1;
51	0	}
52	375	Py_ssize_t capacity = comments == NULL ? 4 : comments->capacity * 2;
53	375	size_t size = sizeof(*comments) +
54	375	(size_t)capacity * sizeof(*comments->spans);
55	375	tokenizer_comments *resized = PyMem_Realloc(comments, size);
56	375	if (resized == NULL) {
57	0	PyErr_NoMemory();
58	0	return -1;
59	0	}
60	375	comments = resized;
61	375	if (state->comments == NULL) {
62	278	comments->count = 0;
63	278	}
64	375	comments->capacity = capacity;
65	375	state->comments = comments;
66	375	}
67	1.31k	comments->spans[comments->count++] =
68	1.31k	(_PyTok_Span){start, end};
69	1.31k	return 0;
70	1.31k	}
71
72		static int
73		finish_ftstring_expr(struct tok_state tok, ftstring_state state,
74		struct token *token)
75	32.5k	{
76	32.5k	assert(token != NULL && state == _PyLexer_CurrentFTString(tok));
77	32.5k	assert(state->mode == FTSTRING_MODE_EXPRESSION && tok->start >= 0);
78
79	32.5k	if (state->expr_span.end >= 0) {
80	5.58k	return 0;
81	5.58k	}
82	32.5k	assert(state->expr_span.start >= 0);
83	26.9k	state->expr_span.end = tok->start;
84	26.9k	int tstring_interpolation = _PyLexer_IsTString(state->kind) &&
85	7.89k	state->replacement_depth == 1;
86	26.9k	if (!(state->debug_expr \|\| tstring_interpolation) \|\| token->metadata) {
87	13.0k	return 0;
88	13.0k	}
89	13.9k	tokenizer_comments *comments = state->comments;
90	13.9k	PyObject *res;
91	13.9k	if (comments != NULL && comments->count > 0) {
92	121	Py_ssize_t stripped_size =
93	121	state->expr_span.end - state->expr_span.start;
94	121	Py_ssize_t comment_count = 0;
95	469	for (Py_ssize_t i = 0; i < comments->count; i++) {
96	348	_PyTok_Span comment = comments->spans[i];
97	348	assert(_PyTok_SpanIsValid(comment));
98	348	assert(comment.start >= (i == 0 ? state->expr_span.start
99	348	: comments->spans[i - 1].end));
100	348	if (comment.start >= state->expr_span.end) {
101	0	break;
102	0	}
103	348	assert(comment.end <= state->expr_span.end);
104	348	stripped_size -= comment.end - comment.start;
105	348	comment_count++;
106	348	}
107	121	char *stripped = PyMem_Malloc((size_t)stripped_size);
108	121	if (stripped == NULL) {
109	0	PyErr_NoMemory();
110	0	return -1;
111	0	}
112	121	_PyTok_Span kept = {state->expr_span.start, state->expr_span.start};
113	121	Py_ssize_t stripped_len = 0;
114	469	for (Py_ssize_t i = 0; i < comment_count; i++) {
115	348	kept.end = comments->spans[i].start;
116	348	Py_ssize_t length;
117	348	const char *text = _PyTok_SourceSpanView(
118	348	&tok->source, kept, &length);
119	348	memcpy(stripped + stripped_len, text, (size_t)length);
120	348	stripped_len += length;
121	348	kept.start = comments->spans[i].end;
122	348	}
123	121	kept.end = state->expr_span.end;
124	121	Py_ssize_t length;
125	121	const char *text = _PyTok_SourceSpanView(&tok->source, kept, &length);
126	121	memcpy(stripped + stripped_len, text, (size_t)length);
127	121	res = PyUnicode_DecodeUTF8(stripped, stripped_size, NULL);
128	121	PyMem_Free(stripped);
129	121	}
130	13.7k	else {
131	13.7k	Py_ssize_t expr_len;
132	13.7k	const char *expr = _PyTok_SourceSpanView(
133	13.7k	&tok->source, state->expr_span, &expr_len);
134	13.7k	res = PyUnicode_DecodeUTF8(expr, expr_len, NULL);
135	13.7k	}
136
137	13.9k	if (!res) {
138	0	return -1;
139	0	}
140	13.9k	token->metadata = res;
141	13.9k	return 0;
142	13.9k	}
143
144		static int
145		begin_ftstring_expr(struct tok_state tok, ftstring_state state,
146		_PyTok_Off expr_start)
147	33.7k	{
148	33.7k	assert(state->mode != FTSTRING_MODE_EXPRESSION);
149	33.7k	state->expr_span = (_PyTok_Span){expr_start, -1};
150	33.7k	if (state->comments != NULL) {
151	599	state->comments->count = 0;
152	599	}
153	33.7k	if (state->replacement_depth >= MAX_EXPR_NESTING) {
154	4	_PyTokenizer_syntaxerror(
155	4	tok, "%c-string: expressions nested too deeply",
156	4	_PyLexer_StringPrefix(state->kind));
157	4	return -1;
158	4	}
159	33.7k	state->replacement_depth++;
160	33.7k	state->mode = FTSTRING_MODE_EXPRESSION;
161	33.7k	state->debug_expr = 0;
162	33.7k	return 0;
163	33.7k	}
164
165		int
166		_PyLexer_ftstring_punctuation(struct tok_state tok, ftstring_state state,
167		struct token *token, int c)
168	33.5k	{
169	33.5k	assert(state->mode == FTSTRING_MODE_EXPRESSION);
170	33.5k	assert(c == ':' \|\| c == '}' \|\| c == '!');
171	33.5k	if (_PyLexer_FTStringBracketDepth(tok, state) != state->replacement_depth) {
172	826	return 0;
173	826	}
174	32.7k	if (c == '!') {
175	2.01k	int next = tok_nextc(tok);
176	2.01k	tok_backup(tok, next);
177	2.01k	if (next == '=') {
178	196	return 0;
179	196	}
180	2.01k	}
181	32.5k	if (finish_ftstring_expr(tok, state, token) < 0) {
182	0	return -1;
183	0	}
184	32.5k	if (c == ':') {
185	5.26k	state->mode = FTSTRING_MODE_FORMAT_SPEC;
186	5.26k	return COLON;
187	5.26k	}
188	27.2k	return 0;
189	32.5k	}
190
191		int
192		_PyLexer_close_ftstring_expr(struct tok_state tok, ftstring_state state,
193		int c)
194	26.8k	{
195	26.8k	assert(state->mode == FTSTRING_MODE_EXPRESSION);
196	26.8k	assert(c == ')' \|\| c == ']' \|\| c == '}');
197	26.8k	int depth = _PyLexer_FTStringBracketDepth(tok, state);
198	26.8k	if (depth < 0) {
199	0	_PyTokenizer_syntaxerror(tok, "%c-string: unmatched '%c'",
200	0	_PyLexer_StringPrefix(state->kind), c);
201	0	return -1;
202	0	}
203	26.8k	if (c == '}' && depth == state->replacement_depth - 1) {
204	25.4k	state->replacement_depth--;
205	25.4k	state->mode = FTSTRING_MODE_MIDDLE;
206	25.4k	state->debug_expr = 0;
207	25.4k	}
208	26.8k	return 0;
209	26.8k	}
210
211		void
212		_PyLexer_mark_ftstring_debug(struct tok_state tok, ftstring_state state)
213	8.47k	{
214	8.47k	assert(state->mode == FTSTRING_MODE_EXPRESSION);
215	8.47k	if (_PyLexer_FTStringBracketDepth(tok, state) == state->replacement_depth) {
216	8.12k	state->debug_expr = 1;
217	8.12k	}
218	8.47k	}
219
220		int
221		_PyLexer_check_string_prefixes(struct tok_state *tok,
222		int saw_b, int saw_r, int saw_u,
223	26.6k	int saw_f, int saw_t) {
224		// Supported: rb, rf, rt (in any order)
225		// Unsupported: ub, ur, uf, ut, bf, bt, ft (in any order)
226
227	26.6k	#define RETURN_SYNTAX_ERROR(PREFIX1, PREFIX2) \
228	26.6k	do { \
229	8	(void)_PyTokenizer_syntaxerror_known_range( \
230	8	tok, (int)(tok->start + 1 - tok->line_start), \
231	8	(int)(tok->cur - tok->line_start), \
232	8	"'" PREFIX1 "' and '" PREFIX2 "' prefixes are incompatible"); \
233	8	return -1; \
234	8	} while (0)
235
236	26.6k	if (saw_u && saw_b) {
237	1	RETURN_SYNTAX_ERROR("u", "b");
238	1	}
239	26.6k	if (saw_u && saw_r) {
240	1	RETURN_SYNTAX_ERROR("u", "r");
241	1	}
242	26.6k	if (saw_u && saw_f) {
243	1	RETURN_SYNTAX_ERROR("u", "f");
244	1	}
245	26.6k	if (saw_u && saw_t) {
246	1	RETURN_SYNTAX_ERROR("u", "t");
247	1	}
248
249	26.6k	if (saw_b && saw_f) {
250	2	RETURN_SYNTAX_ERROR("b", "f");
251	2	}
252	26.6k	if (saw_b && saw_t) {
253	1	RETURN_SYNTAX_ERROR("b", "t");
254	1	}
255
256	26.6k	if (saw_f && saw_t) {
257	1	RETURN_SYNTAX_ERROR("f", "t");
258	1	}
259
260	26.6k	#undef RETURN_SYNTAX_ERROR
261
262	26.6k	return 0;
263	26.6k	}
264
265		int
266		_PyLexer_scan_fstring_start(struct tok_state tok, struct token token,
267		int c, ftstring_kind kind)
268	21.7k	{
269	21.7k	_PyTok_Off p_start = -1;
270	21.7k	_PyTok_Off p_end = -1;
271
272	21.7k	int quote = c;
273	21.7k	int quote_size = 1; /* 1 or 3 */
274
275		/* Find the quote size and start of string */
276	21.7k	int after_quote = tok_nextc(tok);
277	21.7k	if (after_quote == quote) {
278	2.87k	int after_after_quote = tok_nextc(tok);
279	2.87k	if (after_after_quote == quote) {
280	791	quote_size = 3;
281	791	}
282	2.08k	else {
283		// TODO: Check this
284	2.08k	tok_backup(tok, after_after_quote);
285	2.08k	tok_backup(tok, after_quote);
286	2.08k	}
287	2.87k	}
288	21.7k	if (after_quote != quote) {
289	18.8k	tok_backup(tok, after_quote);
290	18.8k	}
291
292	21.7k	p_start = tok->start;
293	21.7k	p_end = tok->cur;
294	21.7k	ftstring_state *state = _PyLexer_PushFTString(tok);
295	21.7k	if (state == NULL) {
296	1	return MAKE_TOKEN(ERRORTOKEN);
297	1	}
298	21.7k	state->mode = FTSTRING_MODE_MIDDLE;
299	21.7k	state->quote = quote;
300	21.7k	state->quote_size = quote_size;
301	21.7k	state->paren_level = tok->level;
302	21.7k	state->start = tok->start;
303	21.7k	state->start_loc = tok->start_loc;
304	21.7k	state->expr_span = (_PyTok_Span){-1, -1};
305
306	21.7k	state->kind = kind;
307	21.7k	return _PyLexer_IsTString(kind)
308	21.7k	? MAKE_TOKEN(TSTRING_START) : MAKE_TOKEN(FSTRING_START);
309	21.7k	}
310
311		int
312		_PyLexer_scan_string(struct tok_state tok, struct token token, int c)
313	39.2k	{
314	39.2k	_PyTok_Off p_start = -1;
315	39.2k	_PyTok_Off p_end = -1;
316
317	39.2k	int quote = c;
318	39.2k	int quote_size = 1; /* 1 or 3 */
319	39.2k	int end_quote_size = 0;
320	39.2k	int has_escaped_quote = 0;
321
322		/* Find the quote size and start of string */
323	39.2k	c = tok_nextc(tok);
324	39.2k	if (c == quote) {
325	7.74k	c = tok_nextc(tok);
326	7.74k	if (c == quote) {
327	726	quote_size = 3;
328	726	}
329	7.01k	else {
330	7.01k	end_quote_size = 1; /* empty string found */
331	7.01k	}
332	7.74k	}
333	39.2k	if (c != quote) {
334	38.5k	tok_backup(tok, c);
335	38.5k	}
336
337		/* Get rest of string */
338	421k	while (end_quote_size != quote_size) {
339	382k	c = tok_nextc(tok);
340	382k	if (tok->done == E_ERROR) {
341	0	return MAKE_TOKEN(ERRORTOKEN);
342	0	}
343	382k	if (tok->done == E_DECODE) {
344	0	break;
345	0	}
346	382k	if (c == EOF \|\| (quote_size == 1 && c == '\n')) {
347	414	if (tok_failed(tok)) {
348	0	return MAKE_TOKEN(ERRORTOKEN);
349	0	}
350	414	if (c == EOF && tok->allow_incomplete_input) {
351	0	return incomplete_string_token(tok, token, STRING,
352	0	tok->start, tok->start_loc);
353	0	}
354	414	int end_lineno = tok->lineno;
355	414	_PyTok_Loc location = tok->start_loc;
356	414	const char *line = _PyTok_SourcePointer(
357	414	&tok->source, tok->start - location.byte_col);
358	414	Py_ssize_t cursor_offset = (Py_ssize_t)location.byte_col + 1;
359
360	414	const ftstring_state *state = _PyLexer_CurrentFTString(tok);
361	414	if (state != NULL) {
362		/* A matching quote may have been intended to close the
363		* surrounding formatted string instead of opening a string
364		* inside a replacement field. */
365	47	if (state->quote == quote && state->quote_size == quote_size) {
366	29	int level = state->paren_level + state->replacement_depth - 1;
367	29	assert(level >= 0 && level < tok->level);
368	29	assert(tok->parenstack[level] == '{');
369	29	int lineno = tok->parenlinenostack[level];
370	29	if (lineno != location.lineno) {
371	5	_PyTokenizer_syntaxerror_at(
372	5	tok, line, cursor_offset, location.lineno, -1, -1,
373	5	"%c-string: expecting '}' to close '{' on line %d",
374	5	_PyLexer_StringPrefix(state->kind), lineno);
375	5	}
376	24	else {
377	24	_PyTokenizer_syntaxerror_at(
378	24	tok, line, cursor_offset, location.lineno, -1, -1,
379	24	"%c-string: expecting '}'",
380	24	_PyLexer_StringPrefix(state->kind));
381	24	}
382	29	return string_error_token(tok, token, tok->start, location);
383	29	}
384	47	}
385
386	385	if (quote_size == 3) {
387	29	_PyTokenizer_syntaxerror_at(
388	29	tok, line, cursor_offset, location.lineno, -1, -1,
389	29	"unterminated triple-quoted string literal"
390	29	" (detected at line %d)", end_lineno);
391	29	if (c != '\n') {
392	29	tok->done = E_EOFS;
393	29	}
394	29	return string_error_token(tok, token, tok->start, location);
395	29	}
396	356	else {
397	356	if (has_escaped_quote) {
398	9	_PyTokenizer_syntaxerror_at(
399	9	tok, line, cursor_offset, location.lineno, -1, -1,
400	9	"unterminated string literal (detected at line %d); "
401	9	"perhaps you escaped the end quote?",
402	9	end_lineno
403	9	);
404	347	} else {
405	347	_PyTokenizer_syntaxerror_at(
406	347	tok, line, cursor_offset, location.lineno, -1, -1,
407	347	"unterminated string literal (detected at line %d)", end_lineno
408	347	);
409	347	}
410	356	if (c != '\n') {
411	9	tok->done = E_EOLS;
412	9	}
413	356	return string_error_token(tok, token, tok->start, location);
414	356	}
415	385	}
416	382k	if (c == quote) {
417	33.8k	end_quote_size += 1;
418	33.8k	}
419	348k	else {
420	348k	end_quote_size = 0;
421	348k	if (c == '\\') {
422	30.6k	c = tok_nextc(tok); /* skip escaped char */
423	30.6k	if (c == quote) { /* but record whether the escaped char was a quote */
424	996	has_escaped_quote = 1;
425	996	}
426	30.6k	if (c == '\r') {
427	0	c = tok_nextc(tok);
428	0	}
429	30.6k	}
430	348k	}
431	382k	}
432
433	38.8k	p_start = tok->start;
434	38.8k	p_end = tok->cur;
435	38.8k	return MAKE_TOKEN(STRING);
436	39.2k	}
437
438		int
439		_PyLexer_get_ftstring(struct tok_state tok, ftstring_state current, struct token *token)
440	62.2k	{
441	62.2k	assert(current == _PyLexer_CurrentFTString(tok) && current->mode != FTSTRING_MODE_EXPRESSION);
442	62.2k	assert((current->quote_size == 1 \|\| current->quote_size == 3) &&
443	62.2k	current->replacement_depth <= MAX_EXPR_NESTING);
444	62.2k	_PyTok_Off p_start = -1;
445	62.2k	_PyTok_Off p_end = -1;
446	62.2k	int token_type;
447	62.2k	int end_quote_size = 0;
448	62.2k	int unicode_escape = 0;
449	62.2k	int quote = current->quote;
450	62.2k	int quote_size = current->quote_size;
451	62.2k	int in_format_spec = current->mode == FTSTRING_MODE_FORMAT_SPEC;
452	62.2k	int raw = _PyLexer_IsRawString(current->kind);
453
454	62.2k	tok->start = tok->cur;
455	62.2k	tok->start_loc = (_PyTok_Loc){tok->lineno, _PyLexer_ByteColumn(tok)};
456
457	265k	while (end_quote_size != quote_size) {
458	245k	int c = tok_nextc(tok);
459	245k	if (tok->done == E_ERROR \|\| tok->done == E_DECODE) {
460	0	return MAKE_TOKEN(ERRORTOKEN);
461	0	}
462
463	245k	if (c == EOF \|\| (quote_size == 1 && c == '\n')) {
464	320	if (tok_failed(tok)) {
465	0	return MAKE_TOKEN(ERRORTOKEN);
466	0	}
467	320	if (c == EOF && tok->allow_incomplete_input) {
468	0	int type = incomplete_string_token(tok, token,
469	0	FTSTRING_MIDDLE(current), current->start, current->start_loc);
470		// Preserve the actual prefix for incremental escape decoding.
471	0	_PyTok_Off end = tok->cur;
472	0	if (end > tok->start && tok->implicit_newline) {
473	0	end--;
474	0	}
475	0	token->span = (_PyTok_Span){tok->start, end};
476	0	token->start_loc = tok->start_loc;
477	0	token->end_loc = (_PyTok_Loc){tok->lineno, (int)(end - tok->line_start)};
478	0	token->is_raw = raw;
479	0	return type;
480	0	}
481
482	320	if (in_format_spec && c == '\n') {
483	48	return MAKE_TOKEN(_PyTokenizer_syntaxerror(
484	48	tok,
485	48	"%c-string: newlines are not allowed in format specifiers for single quoted %c-strings",
486	48	_PyLexer_StringPrefix(current->kind),
487	48	_PyLexer_StringPrefix(current->kind)));
488	48	}
489
490	272	int end_lineno = tok->lineno;
491	272	_PyTok_Loc location = current->start_loc;
492	272	const char *line = _PyTok_SourcePointer(
493	272	&tok->source, current->start - location.byte_col);
494	272	Py_ssize_t cursor_offset = (Py_ssize_t)location.byte_col + 1;
495
496	272	if (quote_size == 3) {
497	19	_PyTokenizer_syntaxerror_at(
498	19	tok, line, cursor_offset, location.lineno, -1, -1,
499	19	"unterminated triple-quoted %c-string literal"
500	19	" (detected at line %d)",
501	19	_PyLexer_StringPrefix(current->kind), end_lineno);
502	19	if (c != '\n') {
503	19	tok->done = E_EOFS;
504	19	}
505	19	return string_error_token(tok, token,
506	19	current->start, location);
507	19	}
508	253	else {
509	253	_PyTokenizer_syntaxerror_at(
510	253	tok, line, cursor_offset, location.lineno, -1, -1,
511	253	"unterminated %c-string literal (detected at line %d)",
512	253	_PyLexer_StringPrefix(current->kind), end_lineno);
513	253	return string_error_token(tok, token,
514	253	current->start, location);
515	253	}
516	272	}
517
518	245k	if (c == quote) {
519	28.4k	end_quote_size += 1;
520	28.4k	continue;
521	217k	} else {
522	217k	end_quote_size = 0;
523	217k	}
524
525	217k	if (c == '{') {
526	36.9k	int peek = tok_nextc(tok);
527	36.9k	if (peek != '{' \|\| in_format_spec) {
528	33.7k	tok_backup(tok, peek);
529	33.7k	_PyTok_Off expr_start = tok->cur;
530	33.7k	tok_backup(tok, c);
531	33.7k	if (begin_ftstring_expr(tok, current, expr_start) < 0) {
532	4	return MAKE_TOKEN(ERRORTOKEN);
533	4	}
534	33.7k	p_start = tok->start;
535	33.7k	p_end = tok->cur;
536	33.7k	if (p_start == p_end) {
537	17.2k	return _PyLexer_get_normal(tok, current, token);
538	17.2k	}
539	33.7k	} else {
540	3.14k	p_start = tok->start;
541	3.14k	p_end = tok->cur - 1;
542	3.14k	}
543	19.6k	goto emit_middle;
544	180k	} else if (c == '}') {
545	5.12k	if (unicode_escape) {
546	239	p_start = tok->start;
547	239	p_end = tok->cur;
548	239	goto emit_middle;
549	239	}
550	4.88k	int peek = tok_nextc(tok);
551
552	4.88k	int bracket_depth = _PyLexer_FTStringBracketDepth(tok, current);
553	4.88k	if (peek == '}' && !in_format_spec && bracket_depth == 0) {
554	963	p_start = tok->start;
555	963	p_end = tok->cur - 1;
556	963	}
557	3.92k	else {
558	3.92k	tok_backup(tok, peek);
559	3.92k	if (!in_format_spec && bracket_depth == 0) {
560	64	if (tok->start == tok->cur - 1) {
561	43	return MAKE_TOKEN(_PyTokenizer_syntaxerror(
562	43	tok, "%c-string: single '}' is not allowed",
563	43	_PyLexer_StringPrefix(current->kind)));
564	43	}
565	21	tok_backup(tok, c);
566	21	p_start = tok->start;
567	21	p_end = tok->cur;
568	21	goto emit_middle;
569	64	}
570	3.85k	tok_backup(tok, c);
571	3.85k	current->mode = FTSTRING_MODE_EXPRESSION;
572	3.85k	p_start = tok->start;
573	3.85k	p_end = tok->cur;
574	3.85k	}
575	4.82k	goto emit_middle;
576	175k	} else if (c == '\\') {
577	5.76k	int peek = tok_nextc(tok);
578	5.76k	if (peek == '\r') {
579	0	peek = tok_nextc(tok);
580	0	}
581	5.76k	if (peek == '{' \|\| peek == '}') {
582	1.37k	if (!raw) {
583	1.12k	if (_PyTokenizer_warn_invalid_escape_sequence(tok, peek)) {
584	0	return MAKE_TOKEN(ERRORTOKEN);
585	0	}
586	1.12k	}
587	1.37k	tok_backup(tok, peek);
588	1.37k	continue;
589	1.37k	}
590
591	4.38k	if (!raw) {
592	4.14k	if (peek == 'N') {
593		/* Handle named unicode escapes (\N{BULLET}) */
594	490	peek = tok_nextc(tok);
595	490	if (peek == '{') {
596	281	unicode_escape = 1;
597	281	} else {
598	209	tok_backup(tok, peek);
599	209	}
600	490	}
601	4.14k	}
602	4.38k	}
603	217k	}
604
605	19.9k	p_start = tok->start;
606	19.9k	p_end = tok->cur;
607	19.9k	if (p_end - quote_size == p_start) {
608	14.4k	int end_token = FTSTRING_END(current);
609	14.4k	_PyLexer_PopFTString(tok);
610	14.4k	return MAKE_TOKEN(end_token);
611	14.4k	}
612	12.0k	for (int i = 0; i < quote_size; i++) {
613	6.56k	tok_backup(tok, quote);
614	6.56k	}
615	5.47k	p_end = tok->cur;
616	30.1k	emit_middle:
617	30.1k	token_type = MAKE_TOKEN(FTSTRING_MIDDLE(current));
618	30.1k	token->is_raw = raw;
619	30.1k	return token_type;
620	5.47k	}