/src/cpython/Parser/lexer/string.c

Source
#include "Python.h"
#include "pycore_token.h"
#include "errcode.h"

#include "lexer_internal.h"
#include "../tokenizer/helpers.h"

#define MAKE_TOKEN(token_type) _PyLexer_token_setup(tok, token, token_type, p_start, p_end)

static int
string_error_token(struct tok_state *tok, struct token *token,
                   _PyTok_Off start, _PyTok_Loc location)
{
    tok->diagnostic = (_PyTokenizer_Diagnostic){
        .location = {location.lineno, location.byte_col + 1},
        .text_span = {start - location.byte_col, tok->inp},
    };
    int type = _PyLexer_token_setup(tok, token, ERRORTOKEN, -1, -1);
    token->start_loc = location;
    token->end_loc = (_PyTok_Loc){location.lineno, -1};
    return type;
}

int
_PyLexer_record_ftstring_comment(struct tok_state *tok, ftstring_state *state,
                                 _PyTok_Off start, _PyTok_Off end)
{
    assert(state == _PyLexer_CurrentFTString(tok) && state->mode == FTSTRING_MODE_EXPRESSION);
    if (state->expr_span.end >= 0) {
        return 0;
    }
    assert(state->expr_span.start >= 0);
    tokenizer_comments *comments = state->comments;
    if (comments == NULL || comments->count == comments->capacity) {
        Py_ssize_t max_capacity = (PY_SSIZE_T_MAX -
            (Py_ssize_t)sizeof(*comments)) /
            (Py_ssize_t)sizeof(*comments->spans);
        if (comments != NULL && comments->capacity > max_capacity / 2) {
            PyErr_NoMemory();
            return -1;
        }
        Py_ssize_t capacity = comments == NULL ? 4 : comments->capacity * 2;
        size_t size = sizeof(*comments) +
            (size_t)capacity * sizeof(*comments->spans);
        tokenizer_comments *resized = PyMem_Realloc(comments, size);
        if (resized == NULL) {
            PyErr_NoMemory();
            return -1;
        }
        comments = resized;
        if (state->comments == NULL) {
            comments->count = 0;
        }
        comments->capacity = capacity;
        state->comments = comments;
    }
    comments->spans[comments->count++] =
        (_PyTok_Span){start, end};
    return 0;
}

static int
finish_ftstring_expr(struct tok_state *tok, ftstring_state *state,
                     struct token *token)
{
    assert(token != NULL && state == _PyLexer_CurrentFTString(tok));
    assert(state->mode == FTSTRING_MODE_EXPRESSION && tok->start >= 0);

    if (state->expr_span.end >= 0) {
        return 0;
    }
    assert(state->expr_span.start >= 0);
    state->expr_span.end = tok->start;
    int tstring_interpolation = _PyLexer_IsTString(state->kind) &&
        state->replacement_depth == 1;
    if (!(state->debug_expr || tstring_interpolation) || token->metadata) {
        return 0;
    }
    tokenizer_comments *comments = state->comments;
    PyObject *res;
    if (comments != NULL && comments->count > 0) {
        Py_ssize_t stripped_size =
            state->expr_span.end - state->expr_span.start;
        Py_ssize_t comment_count = 0;
        for (Py_ssize_t i = 0; i < comments->count; i++) {
            _PyTok_Span comment = comments->spans[i];
            assert(_PyTok_SpanIsValid(comment));
            assert(comment.start >= (i == 0 ? state->expr_span.start
                                           : comments->spans[i - 1].end));
            if (comment.start >= state->expr_span.end) {
                break;
            }
            assert(comment.end <= state->expr_span.end);
            stripped_size -= comment.end - comment.start;
            comment_count++;
        }
        char *stripped = PyMem_Malloc((size_t)stripped_size);
        if (stripped == NULL) {
            PyErr_NoMemory();
            return -1;
        }
        _PyTok_Span kept = {state->expr_span.start, state->expr_span.start};
        Py_ssize_t stripped_len = 0;
        for (Py_ssize_t i = 0; i < comment_count; i++) {
            kept.end = comments->spans[i].start;
            Py_ssize_t length;
            const char *text = _PyTok_SourceSpanView(
                &tok->source, kept, &length);
            memcpy(stripped + stripped_len, text, (size_t)length);
            stripped_len += length;
            kept.start = comments->spans[i].end;
        }
        kept.end = state->expr_span.end;
        Py_ssize_t length;
        const char *text = _PyTok_SourceSpanView(&tok->source, kept, &length);
        memcpy(stripped + stripped_len, text, (size_t)length);
        res = PyUnicode_DecodeUTF8(stripped, stripped_size, NULL);
        PyMem_Free(stripped);
    }
    else {
        Py_ssize_t expr_len;
        const char *expr = _PyTok_SourceSpanView(
            &tok->source, state->expr_span, &expr_len);
        res = PyUnicode_DecodeUTF8(expr, expr_len, NULL);
    }

    if (!res) {
        return -1;
    }
    token->metadata = res;
    return 0;
}

static int
begin_ftstring_expr(struct tok_state *tok, ftstring_state *state,
                    _PyTok_Off expr_start)
{
    assert(state->mode != FTSTRING_MODE_EXPRESSION);
    state->expr_span = (_PyTok_Span){expr_start, -1};
    if (state->comments != NULL) {
        state->comments->count = 0;
    }
    if (state->replacement_depth >= MAX_EXPR_NESTING) {
        _PyTokenizer_syntaxerror(
            tok, "%c-string: expressions nested too deeply",
            _PyLexer_StringPrefix(state->kind));
        return -1;
    }
    state->replacement_depth++;
    state->mode = FTSTRING_MODE_EXPRESSION;
    state->debug_expr = 0;
    return 0;
}

int
_PyLexer_ftstring_punctuation(struct tok_state *tok, ftstring_state *state,
                              struct token *token, int c)
{
    assert(state->mode == FTSTRING_MODE_EXPRESSION);
    assert(c == ':' || c == '}' || c == '!');
    if (_PyLexer_FTStringBracketDepth(tok, state) != state->replacement_depth) {
        return 0;
    }
    if (c == '!') {
        int next = tok_nextc(tok);
        tok_backup(tok, next);
        if (next == '=') {
            return 0;
        }
    }
    if (finish_ftstring_expr(tok, state, token) < 0) {
        return -1;
    }
    if (c == ':') {
        state->mode = FTSTRING_MODE_FORMAT_SPEC;
        return COLON;
    }
    return 0;
}

int
_PyLexer_close_ftstring_expr(struct tok_state *tok, ftstring_state *state,
                             int c)
{
    assert(state->mode == FTSTRING_MODE_EXPRESSION);
    assert(c == ')' || c == ']' || c == '}');
    int depth = _PyLexer_FTStringBracketDepth(tok, state);
    if (depth < 0) {
        _PyTokenizer_syntaxerror(tok, "%c-string: unmatched '%c'",
                                _PyLexer_StringPrefix(state->kind), c);
        return -1;
    }
    if (c == '}' && depth == state->replacement_depth - 1) {
        state->replacement_depth--;
        state->mode = FTSTRING_MODE_MIDDLE;
        state->debug_expr = 0;
    }
    return 0;
}

void
_PyLexer_mark_ftstring_debug(struct tok_state *tok, ftstring_state *state)
{
    assert(state->mode == FTSTRING_MODE_EXPRESSION);
    if (_PyLexer_FTStringBracketDepth(tok, state) == state->replacement_depth) {
        state->debug_expr = 1;
    }
}

int
_PyLexer_check_string_prefixes(struct tok_state *tok,
                                             int saw_b, int saw_r, int saw_u,
                                             int saw_f, int saw_t) {
    // Supported: rb, rf, rt (in any order)
    // Unsupported: ub, ur, uf, ut, bf, bt, ft (in any order)

#define RETURN_SYNTAX_ERROR(PREFIX1, PREFIX2)                             \
    do {                                                                  \
        (void)_PyTokenizer_syntaxerror_known_range(                       \
            tok, (int)(tok->start + 1 - tok->line_start),                 \
            (int)(tok->cur - tok->line_start),                            \
            "'" PREFIX1 "' and '" PREFIX2 "' prefixes are incompatible"); \
        return -1;                                                        \
    } while (0)

    if (saw_u && saw_b) {
        RETURN_SYNTAX_ERROR("u", "b");
    }
    if (saw_u && saw_r) {
        RETURN_SYNTAX_ERROR("u", "r");
    }
    if (saw_u && saw_f) {
        RETURN_SYNTAX_ERROR("u", "f");
    }
    if (saw_u && saw_t) {
        RETURN_SYNTAX_ERROR("u", "t");
    }

    if (saw_b && saw_f) {
        RETURN_SYNTAX_ERROR("b", "f");
    }
    if (saw_b && saw_t) {
        RETURN_SYNTAX_ERROR("b", "t");
    }

    if (saw_f && saw_t) {
        RETURN_SYNTAX_ERROR("f", "t");
    }

#undef RETURN_SYNTAX_ERROR

    return 0;
}

int
_PyLexer_scan_fstring_start(struct tok_state *tok, struct token *token,
                            int c, ftstring_kind kind)
{
    _PyTok_Off p_start = -1;
    _PyTok_Off p_end = -1;

    int quote = c;
    int quote_size = 1;             /* 1 or 3 */

    /* Find the quote size and start of string */
    int after_quote = tok_nextc(tok);
    if (after_quote == quote) {
        int after_after_quote = tok_nextc(tok);
        if (after_after_quote == quote) {
            quote_size = 3;
        }
        else {
            // TODO: Check this
            tok_backup(tok, after_after_quote);
            tok_backup(tok, after_quote);
        }
    }
    if (after_quote != quote) {
        tok_backup(tok, after_quote);
    }

    p_start = tok->start;
    p_end = tok->cur;
    ftstring_state *state = _PyLexer_PushFTString(tok);
    if (state == NULL) {
        return MAKE_TOKEN(ERRORTOKEN);
    }
    state->mode = FTSTRING_MODE_MIDDLE;
    state->quote = quote;
    state->quote_size = quote_size;
    state->paren_level = tok->level;
    state->start = tok->start;
    state->start_loc = tok->start_loc;
    state->expr_span = (_PyTok_Span){-1, -1};

    state->kind = kind;
    return _PyLexer_IsTString(kind)
        ? MAKE_TOKEN(TSTRING_START) : MAKE_TOKEN(FSTRING_START);
}

int
_PyLexer_scan_string(struct tok_state *tok, struct token *token, int c)
{
    _PyTok_Off p_start = -1;
    _PyTok_Off p_end = -1;

    int quote = c;
    int quote_size = 1;             /* 1 or 3 */
    int end_quote_size = 0;
    int has_escaped_quote = 0;

    /* Find the quote size and start of string */
    c = tok_nextc(tok);
    if (c == quote) {
        c = tok_nextc(tok);
        if (c == quote) {
            quote_size = 3;
        }
        else {
            end_quote_size = 1;     /* empty string found */
        }
    }
    if (c != quote) {
        tok_backup(tok, c);
    }

    /* Get rest of string */
    while (end_quote_size != quote_size) {
        c = tok_nextc(tok);
        if (tok->done == E_ERROR) {
            return MAKE_TOKEN(ERRORTOKEN);
        }
        if (tok->done == E_DECODE) {
            break;
        }
        if (c == EOF || (quote_size == 1 && c == '\n')) {
            if (tok_failed(tok)) {
                return MAKE_TOKEN(ERRORTOKEN);
            }
            int end_lineno = tok->lineno;
            _PyTok_Loc location = tok->start_loc;
            const char *line = _PyTok_SourcePointer(
                &tok->source, tok->start - location.byte_col);
            Py_ssize_t cursor_offset = (Py_ssize_t)location.byte_col + 1;

            const ftstring_state *state = _PyLexer_CurrentFTString(tok);
            if (state != NULL) {
                /* A matching quote may have been intended to close the
                 * surrounding formatted string instead of opening a string
                 * inside a replacement field. */
                if (state->quote == quote && state->quote_size == quote_size) {
                    int level = state->paren_level + state->replacement_depth - 1;
                    assert(level >= 0 && level < tok->level);
                    assert(tok->parenstack[level] == '{');
                    int lineno = tok->parenlinenostack[level];
                    if (lineno != location.lineno) {
                        _PyTokenizer_syntaxerror_at(
                            tok, line, cursor_offset, location.lineno, -1, -1,
                            "%c-string: expecting '}' to close '{' on line %d",
                            _PyLexer_StringPrefix(state->kind), lineno);
                    }
                    else {
                        _PyTokenizer_syntaxerror_at(
                            tok, line, cursor_offset, location.lineno, -1, -1,
                            "%c-string: expecting '}'",
                            _PyLexer_StringPrefix(state->kind));
                    }
                    return string_error_token(tok, token, tok->start, location);
                }
            }

            if (quote_size == 3) {
                _PyTokenizer_syntaxerror_at(
                    tok, line, cursor_offset, location.lineno, -1, -1,
                    "unterminated triple-quoted string literal"
                    " (detected at line %d)", end_lineno);
                if (c != '\n') {
                    tok->done = E_EOFS;
                }
                return string_error_token(tok, token, tok->start, location);
            }
            else {
                if (has_escaped_quote) {
                    _PyTokenizer_syntaxerror_at(
                        tok, line, cursor_offset, location.lineno, -1, -1,
                        "unterminated string literal (detected at line %d); "
                        "perhaps you escaped the end quote?",
                        end_lineno
                    );
                } else {
                    _PyTokenizer_syntaxerror_at(
                        tok, line, cursor_offset, location.lineno, -1, -1,
                        "unterminated string literal (detected at line %d)", end_lineno
                    );
                }
                if (c != '\n') {
                    tok->done = E_EOLS;
                }
                return string_error_token(tok, token, tok->start, location);
            }
        }
        if (c == quote) {
            end_quote_size += 1;
        }
        else {
            end_quote_size = 0;
            if (c == '\\') {
                c = tok_nextc(tok);  /* skip escaped char */
                if (c == quote) {  /* but record whether the escaped char was a quote */
                    has_escaped_quote = 1;
                }
                if (c == '\r') {
                    c = tok_nextc(tok);
                }
            }
        }
    }

    p_start = tok->start;
    p_end = tok->cur;
    return MAKE_TOKEN(STRING);
}

int
_PyLexer_get_ftstring(struct tok_state *tok, ftstring_state *current, struct token *token)
{
    assert(current == _PyLexer_CurrentFTString(tok) && current->mode != FTSTRING_MODE_EXPRESSION);
    assert((current->quote_size == 1 || current->quote_size == 3) &&
           current->replacement_depth <= MAX_EXPR_NESTING);
    _PyTok_Off p_start = -1;
    _PyTok_Off p_end = -1;
    int token_type;
    int end_quote_size = 0;
    int unicode_escape = 0;
    int quote = current->quote;
    int quote_size = current->quote_size;
    int in_format_spec = current->mode == FTSTRING_MODE_FORMAT_SPEC;
    int raw = _PyLexer_IsRawString(current->kind);

    tok->start = tok->cur;
    tok->start_loc = (_PyTok_Loc){tok->lineno, _PyLexer_ByteColumn(tok)};

    while (end_quote_size != quote_size) {
        int c = tok_nextc(tok);
        if (tok->done == E_ERROR || tok->done == E_DECODE) {
            return MAKE_TOKEN(ERRORTOKEN);
        }

        if (c == EOF || (quote_size == 1 && c == '\n')) {
            if (tok_failed(tok)) {
                return MAKE_TOKEN(ERRORTOKEN);
            }

            if (in_format_spec && c == '\n') {
                return MAKE_TOKEN(_PyTokenizer_syntaxerror(
                    tok,
                    "%c-string: newlines are not allowed in format specifiers for single quoted %c-strings",
                    _PyLexer_StringPrefix(current->kind),
                    _PyLexer_StringPrefix(current->kind)));
            }

            int end_lineno = tok->lineno;
            _PyTok_Loc location = current->start_loc;
            const char *line = _PyTok_SourcePointer(
                &tok->source, current->start - location.byte_col);
            Py_ssize_t cursor_offset = (Py_ssize_t)location.byte_col + 1;

            if (quote_size == 3) {
                _PyTokenizer_syntaxerror_at(
                    tok, line, cursor_offset, location.lineno, -1, -1,
                    "unterminated triple-quoted %c-string literal"
                    " (detected at line %d)",
                    _PyLexer_StringPrefix(current->kind), end_lineno);
                if (c != '\n') {
                    tok->done = E_EOFS;
                }
                return string_error_token(tok, token,
                    current->start, location);
            }
            else {
                _PyTokenizer_syntaxerror_at(
                    tok, line, cursor_offset, location.lineno, -1, -1,
                    "unterminated %c-string literal (detected at line %d)",
                    _PyLexer_StringPrefix(current->kind), end_lineno);
                return string_error_token(tok, token,
                    current->start, location);
            }
        }

        if (c == quote) {
            end_quote_size += 1;
            continue;
        } else {
            end_quote_size = 0;
        }

        if (c == '{') {
            int peek = tok_nextc(tok);
            if (peek != '{' || in_format_spec) {
                tok_backup(tok, peek);
                _PyTok_Off expr_start = tok->cur;
                tok_backup(tok, c);
                if (begin_ftstring_expr(tok, current, expr_start) < 0) {
                    return MAKE_TOKEN(ERRORTOKEN);
                }
                p_start = tok->start;
                p_end = tok->cur;
                if (p_start == p_end) {
                    return _PyLexer_get_normal(tok, current, token);
                }
            } else {
                p_start = tok->start;
                p_end = tok->cur - 1;
            }
            goto emit_middle;
        } else if (c == '}') {
            if (unicode_escape) {
                p_start = tok->start;
                p_end = tok->cur;
                goto emit_middle;
            }
            int peek = tok_nextc(tok);

            int bracket_depth = _PyLexer_FTStringBracketDepth(tok, current);
            if (peek == '}' && !in_format_spec && bracket_depth == 0) {
                p_start = tok->start;
                p_end = tok->cur - 1;
            }
            else {
                tok_backup(tok, peek);
                if (!in_format_spec && bracket_depth == 0) {
                    if (tok->start == tok->cur - 1) {
                        return MAKE_TOKEN(_PyTokenizer_syntaxerror(
                            tok, "%c-string: single '}' is not allowed",
                            _PyLexer_StringPrefix(current->kind)));
                    }
                    tok_backup(tok, c);
                    p_start = tok->start;
                    p_end = tok->cur;
                    goto emit_middle;
                }
                tok_backup(tok, c);
                current->mode = FTSTRING_MODE_EXPRESSION;
                p_start = tok->start;
                p_end = tok->cur;
            }
            goto emit_middle;
        } else if (c == '\\') {
            int peek = tok_nextc(tok);
            if (peek == '\r') {
                peek = tok_nextc(tok);
            }
            if (peek == '{' || peek == '}') {
                if (!raw) {
                    if (_PyTokenizer_warn_invalid_escape_sequence(tok, peek)) {
                        return MAKE_TOKEN(ERRORTOKEN);
                    }
                }
                tok_backup(tok, peek);
                continue;
            }

            if (!raw) {
                if (peek == 'N') {
                    /* Handle named unicode escapes (\N{BULLET}) */
                    peek = tok_nextc(tok);
                    if (peek == '{') {
                        unicode_escape = 1;
                    } else {
                        tok_backup(tok, peek);
                    }
                }
            }
        }
    }

    p_start = tok->start;
    p_end = tok->cur;
    if (p_end - quote_size == p_start) {
        int end_token = FTSTRING_END(current);
        _PyLexer_PopFTString(tok);
        return MAKE_TOKEN(end_token);
    }
    for (int i = 0; i < quote_size; i++) {
        tok_backup(tok, quote);
    }
    p_end = tok->cur;
emit_middle:
    token_type = MAKE_TOKEN(FTSTRING_MIDDLE(current));
    token->is_raw = raw;
    return token_type;
}

Coverage Report

Created: 2026-10-17 06:15

Line	Count	Source
1		#include "Python.h"
2		#include "pycore_token.h"
3		#include "errcode.h"
4
5		#include "lexer_internal.h"
6		#include "../tokenizer/helpers.h"
7
8	99.9k	#define MAKE_TOKEN(token_type) _PyLexer_token_setup(tok, token, token_type, p_start, p_end)
9
10		static int
11		string_error_token(struct tok_state tok, struct token token,
12		_PyTok_Off start, _PyTok_Loc location)
13	663	{
14	663	tok->diagnostic = (_PyTokenizer_Diagnostic){
15	663	.location = {location.lineno, location.byte_col + 1},
16	663	.text_span = {start - location.byte_col, tok->inp},
17	663	};
18	663	int type = _PyLexer_token_setup(tok, token, ERRORTOKEN, -1, -1);
19	663	token->start_loc = location;
20	663	token->end_loc = (_PyTok_Loc){location.lineno, -1};
21	663	return type;
22	663	}
23
24		int
25		_PyLexer_record_ftstring_comment(struct tok_state tok, ftstring_state state,
26		_PyTok_Off start, _PyTok_Off end)
27	1.50k	{
28	1.50k	assert(state == _PyLexer_CurrentFTString(tok) && state->mode == FTSTRING_MODE_EXPRESSION);
29	1.50k	if (state->expr_span.end >= 0) {
30	214	return 0;
31	214	}
32	1.50k	assert(state->expr_span.start >= 0);
33	1.29k	tokenizer_comments *comments = state->comments;
34	1.29k	if (comments == NULL \|\| comments->count == comments->capacity) {
35	370	Py_ssize_t max_capacity = (PY_SSIZE_T_MAX -
36	370	(Py_ssize_t)sizeof(*comments)) /
37	370	(Py_ssize_t)sizeof(*comments->spans);
38	370	if (comments != NULL && comments->capacity > max_capacity / 2) {
39	0	PyErr_NoMemory();
40	0	return -1;
41	0	}
42	370	Py_ssize_t capacity = comments == NULL ? 4 : comments->capacity * 2;
43	370	size_t size = sizeof(*comments) +
44	370	(size_t)capacity * sizeof(*comments->spans);
45	370	tokenizer_comments *resized = PyMem_Realloc(comments, size);
46	370	if (resized == NULL) {
47	0	PyErr_NoMemory();
48	0	return -1;
49	0	}
50	370	comments = resized;
51	370	if (state->comments == NULL) {
52	274	comments->count = 0;
53	274	}
54	370	comments->capacity = capacity;
55	370	state->comments = comments;
56	370	}
57	1.29k	comments->spans[comments->count++] =
58	1.29k	(_PyTok_Span){start, end};
59	1.29k	return 0;
60	1.29k	}
61
62		static int
63		finish_ftstring_expr(struct tok_state tok, ftstring_state state,
64		struct token *token)
65	30.2k	{
66	30.2k	assert(token != NULL && state == _PyLexer_CurrentFTString(tok));
67	30.2k	assert(state->mode == FTSTRING_MODE_EXPRESSION && tok->start >= 0);
68
69	30.2k	if (state->expr_span.end >= 0) {
70	5.67k	return 0;
71	5.67k	}
72	30.2k	assert(state->expr_span.start >= 0);
73	24.5k	state->expr_span.end = tok->start;
74	24.5k	int tstring_interpolation = _PyLexer_IsTString(state->kind) &&
75	6.79k	state->replacement_depth == 1;
76	24.5k	if (!(state->debug_expr \|\| tstring_interpolation) \|\| token->metadata) {
77	12.3k	return 0;
78	12.3k	}
79	12.1k	tokenizer_comments *comments = state->comments;
80	12.1k	PyObject *res;
81	12.1k	if (comments != NULL && comments->count > 0) {
82	100	Py_ssize_t stripped_size =
83	100	state->expr_span.end - state->expr_span.start;
84	100	Py_ssize_t comment_count = 0;
85	428	for (Py_ssize_t i = 0; i < comments->count; i++) {
86	328	_PyTok_Span comment = comments->spans[i];
87	328	assert(_PyTok_SpanIsValid(comment));
88	328	assert(comment.start >= (i == 0 ? state->expr_span.start
89	328	: comments->spans[i - 1].end));
90	328	if (comment.start >= state->expr_span.end) {
91	0	break;
92	0	}
93	328	assert(comment.end <= state->expr_span.end);
94	328	stripped_size -= comment.end - comment.start;
95	328	comment_count++;
96	328	}
97	100	char *stripped = PyMem_Malloc((size_t)stripped_size);
98	100	if (stripped == NULL) {
99	0	PyErr_NoMemory();
100	0	return -1;
101	0	}
102	100	_PyTok_Span kept = {state->expr_span.start, state->expr_span.start};
103	100	Py_ssize_t stripped_len = 0;
104	428	for (Py_ssize_t i = 0; i < comment_count; i++) {
105	328	kept.end = comments->spans[i].start;
106	328	Py_ssize_t length;
107	328	const char *text = _PyTok_SourceSpanView(
108	328	&tok->source, kept, &length);
109	328	memcpy(stripped + stripped_len, text, (size_t)length);
110	328	stripped_len += length;
111	328	kept.start = comments->spans[i].end;
112	328	}
113	100	kept.end = state->expr_span.end;
114	100	Py_ssize_t length;
115	100	const char *text = _PyTok_SourceSpanView(&tok->source, kept, &length);
116	100	memcpy(stripped + stripped_len, text, (size_t)length);
117	100	res = PyUnicode_DecodeUTF8(stripped, stripped_size, NULL);
118	100	PyMem_Free(stripped);
119	100	}
120	12.0k	else {
121	12.0k	Py_ssize_t expr_len;
122	12.0k	const char *expr = _PyTok_SourceSpanView(
123	12.0k	&tok->source, state->expr_span, &expr_len);
124	12.0k	res = PyUnicode_DecodeUTF8(expr, expr_len, NULL);
125	12.0k	}
126
127	12.1k	if (!res) {
128	0	return -1;
129	0	}
130	12.1k	token->metadata = res;
131	12.1k	return 0;
132	12.1k	}
133
134		static int
135		begin_ftstring_expr(struct tok_state tok, ftstring_state state,
136		_PyTok_Off expr_start)
137	30.8k	{
138	30.8k	assert(state->mode != FTSTRING_MODE_EXPRESSION);
139	30.8k	state->expr_span = (_PyTok_Span){expr_start, -1};
140	30.8k	if (state->comments != NULL) {
141	517	state->comments->count = 0;
142	517	}
143	30.8k	if (state->replacement_depth >= MAX_EXPR_NESTING) {
144	5	_PyTokenizer_syntaxerror(
145	5	tok, "%c-string: expressions nested too deeply",
146	5	_PyLexer_StringPrefix(state->kind));
147	5	return -1;
148	5	}
149	30.8k	state->replacement_depth++;
150	30.8k	state->mode = FTSTRING_MODE_EXPRESSION;
151	30.8k	state->debug_expr = 0;
152	30.8k	return 0;
153	30.8k	}
154
155		int
156		_PyLexer_ftstring_punctuation(struct tok_state tok, ftstring_state state,
157		struct token *token, int c)
158	31.2k	{
159	31.2k	assert(state->mode == FTSTRING_MODE_EXPRESSION);
160	31.2k	assert(c == ':' \|\| c == '}' \|\| c == '!');
161	31.2k	if (_PyLexer_FTStringBracketDepth(tok, state) != state->replacement_depth) {
162	816	return 0;
163	816	}
164	30.4k	if (c == '!') {
165	2.05k	int next = tok_nextc(tok);
166	2.05k	tok_backup(tok, next);
167	2.05k	if (next == '=') {
168	196	return 0;
169	196	}
170	2.05k	}
171	30.2k	if (finish_ftstring_expr(tok, state, token) < 0) {
172	0	return -1;
173	0	}
174	30.2k	if (c == ':') {
175	5.26k	state->mode = FTSTRING_MODE_FORMAT_SPEC;
176	5.26k	return COLON;
177	5.26k	}
178	24.9k	return 0;
179	30.2k	}
180
181		int
182		_PyLexer_close_ftstring_expr(struct tok_state tok, ftstring_state state,
183		int c)
184	24.2k	{
185	24.2k	assert(state->mode == FTSTRING_MODE_EXPRESSION);
186	24.2k	assert(c == ')' \|\| c == ']' \|\| c == '}');
187	24.2k	int depth = _PyLexer_FTStringBracketDepth(tok, state);
188	24.2k	if (depth < 0) {
189	0	_PyTokenizer_syntaxerror(tok, "%c-string: unmatched '%c'",
190	0	_PyLexer_StringPrefix(state->kind), c);
191	0	return -1;
192	0	}
193	24.2k	if (c == '}' && depth == state->replacement_depth - 1) {
194	23.1k	state->replacement_depth--;
195	23.1k	state->mode = FTSTRING_MODE_MIDDLE;
196	23.1k	state->debug_expr = 0;
197	23.1k	}
198	24.2k	return 0;
199	24.2k	}
200
201		void
202		_PyLexer_mark_ftstring_debug(struct tok_state tok, ftstring_state state)
203	8.05k	{
204	8.05k	assert(state->mode == FTSTRING_MODE_EXPRESSION);
205	8.05k	if (_PyLexer_FTStringBracketDepth(tok, state) == state->replacement_depth) {
206	7.67k	state->debug_expr = 1;
207	7.67k	}
208	8.05k	}
209
210		int
211		_PyLexer_check_string_prefixes(struct tok_state *tok,
212		int saw_b, int saw_r, int saw_u,
213	24.6k	int saw_f, int saw_t) {
214		// Supported: rb, rf, rt (in any order)
215		// Unsupported: ub, ur, uf, ut, bf, bt, ft (in any order)
216
217	24.6k	#define RETURN_SYNTAX_ERROR(PREFIX1, PREFIX2) \
218	24.6k	do { \
219	11	(void)_PyTokenizer_syntaxerror_known_range( \
220	11	tok, (int)(tok->start + 1 - tok->line_start), \
221	11	(int)(tok->cur - tok->line_start), \
222	11	"'" PREFIX1 "' and '" PREFIX2 "' prefixes are incompatible"); \
223	11	return -1; \
224	11	} while (0)
225
226	24.6k	if (saw_u && saw_b) {
227	1	RETURN_SYNTAX_ERROR("u", "b");
228	1	}
229	24.6k	if (saw_u && saw_r) {
230	2	RETURN_SYNTAX_ERROR("u", "r");
231	2	}
232	24.6k	if (saw_u && saw_f) {
233	1	RETURN_SYNTAX_ERROR("u", "f");
234	1	}
235	24.6k	if (saw_u && saw_t) {
236	1	RETURN_SYNTAX_ERROR("u", "t");
237	1	}
238
239	24.6k	if (saw_b && saw_f) {
240	4	RETURN_SYNTAX_ERROR("b", "f");
241	4	}
242	24.6k	if (saw_b && saw_t) {
243	1	RETURN_SYNTAX_ERROR("b", "t");
244	1	}
245
246	24.6k	if (saw_f && saw_t) {
247	1	RETURN_SYNTAX_ERROR("f", "t");
248	1	}
249
250	24.6k	#undef RETURN_SYNTAX_ERROR
251
252	24.6k	return 0;
253	24.6k	}
254
255		int
256		_PyLexer_scan_fstring_start(struct tok_state tok, struct token token,
257		int c, ftstring_kind kind)
258	20.2k	{
259	20.2k	_PyTok_Off p_start = -1;
260	20.2k	_PyTok_Off p_end = -1;
261
262	20.2k	int quote = c;
263	20.2k	int quote_size = 1; /* 1 or 3 */
264
265		/* Find the quote size and start of string */
266	20.2k	int after_quote = tok_nextc(tok);
267	20.2k	if (after_quote == quote) {
268	2.63k	int after_after_quote = tok_nextc(tok);
269	2.63k	if (after_after_quote == quote) {
270	768	quote_size = 3;
271	768	}
272	1.87k	else {
273		// TODO: Check this
274	1.87k	tok_backup(tok, after_after_quote);
275	1.87k	tok_backup(tok, after_quote);
276	1.87k	}
277	2.63k	}
278	20.2k	if (after_quote != quote) {
279	17.6k	tok_backup(tok, after_quote);
280	17.6k	}
281
282	20.2k	p_start = tok->start;
283	20.2k	p_end = tok->cur;
284	20.2k	ftstring_state *state = _PyLexer_PushFTString(tok);
285	20.2k	if (state == NULL) {
286	1	return MAKE_TOKEN(ERRORTOKEN);
287	1	}
288	20.2k	state->mode = FTSTRING_MODE_MIDDLE;
289	20.2k	state->quote = quote;
290	20.2k	state->quote_size = quote_size;
291	20.2k	state->paren_level = tok->level;
292	20.2k	state->start = tok->start;
293	20.2k	state->start_loc = tok->start_loc;
294	20.2k	state->expr_span = (_PyTok_Span){-1, -1};
295
296	20.2k	state->kind = kind;
297	20.2k	return _PyLexer_IsTString(kind)
298	20.2k	? MAKE_TOKEN(TSTRING_START) : MAKE_TOKEN(FSTRING_START);
299	20.2k	}
300
301		int
302		_PyLexer_scan_string(struct tok_state tok, struct token token, int c)
303	38.4k	{
304	38.4k	_PyTok_Off p_start = -1;
305	38.4k	_PyTok_Off p_end = -1;
306
307	38.4k	int quote = c;
308	38.4k	int quote_size = 1; /* 1 or 3 */
309	38.4k	int end_quote_size = 0;
310	38.4k	int has_escaped_quote = 0;
311
312		/* Find the quote size and start of string */
313	38.4k	c = tok_nextc(tok);
314	38.4k	if (c == quote) {
315	7.20k	c = tok_nextc(tok);
316	7.20k	if (c == quote) {
317	736	quote_size = 3;
318	736	}
319	6.47k	else {
320	6.47k	end_quote_size = 1; /* empty string found */
321	6.47k	}
322	7.20k	}
323	38.4k	if (c != quote) {
324	37.7k	tok_backup(tok, c);
325	37.7k	}
326
327		/* Get rest of string */
328	441k	while (end_quote_size != quote_size) {
329	403k	c = tok_nextc(tok);
330	403k	if (tok->done == E_ERROR) {
331	0	return MAKE_TOKEN(ERRORTOKEN);
332	0	}
333	403k	if (tok->done == E_DECODE) {
334	0	break;
335	0	}
336	403k	if (c == EOF \|\| (quote_size == 1 && c == '\n')) {
337	406	if (tok_failed(tok)) {
338	0	return MAKE_TOKEN(ERRORTOKEN);
339	0	}
340	406	int end_lineno = tok->lineno;
341	406	_PyTok_Loc location = tok->start_loc;
342	406	const char *line = _PyTok_SourcePointer(
343	406	&tok->source, tok->start - location.byte_col);
344	406	Py_ssize_t cursor_offset = (Py_ssize_t)location.byte_col + 1;
345
346	406	const ftstring_state *state = _PyLexer_CurrentFTString(tok);
347	406	if (state != NULL) {
348		/* A matching quote may have been intended to close the
349		* surrounding formatted string instead of opening a string
350		* inside a replacement field. */
351	45	if (state->quote == quote && state->quote_size == quote_size) {
352	29	int level = state->paren_level + state->replacement_depth - 1;
353	29	assert(level >= 0 && level < tok->level);
354	29	assert(tok->parenstack[level] == '{');
355	29	int lineno = tok->parenlinenostack[level];
356	29	if (lineno != location.lineno) {
357	6	_PyTokenizer_syntaxerror_at(
358	6	tok, line, cursor_offset, location.lineno, -1, -1,
359	6	"%c-string: expecting '}' to close '{' on line %d",
360	6	_PyLexer_StringPrefix(state->kind), lineno);
361	6	}
362	23	else {
363	23	_PyTokenizer_syntaxerror_at(
364	23	tok, line, cursor_offset, location.lineno, -1, -1,
365	23	"%c-string: expecting '}'",
366	23	_PyLexer_StringPrefix(state->kind));
367	23	}
368	29	return string_error_token(tok, token, tok->start, location);
369	29	}
370	45	}
371
372	377	if (quote_size == 3) {
373	31	_PyTokenizer_syntaxerror_at(
374	31	tok, line, cursor_offset, location.lineno, -1, -1,
375	31	"unterminated triple-quoted string literal"
376	31	" (detected at line %d)", end_lineno);
377	31	if (c != '\n') {
378	31	tok->done = E_EOFS;
379	31	}
380	31	return string_error_token(tok, token, tok->start, location);
381	31	}
382	346	else {
383	346	if (has_escaped_quote) {
384	8	_PyTokenizer_syntaxerror_at(
385	8	tok, line, cursor_offset, location.lineno, -1, -1,
386	8	"unterminated string literal (detected at line %d); "
387	8	"perhaps you escaped the end quote?",
388	8	end_lineno
389	8	);
390	338	} else {
391	338	_PyTokenizer_syntaxerror_at(
392	338	tok, line, cursor_offset, location.lineno, -1, -1,
393	338	"unterminated string literal (detected at line %d)", end_lineno
394	338	);
395	338	}
396	346	if (c != '\n') {
397	7	tok->done = E_EOLS;
398	7	}
399	346	return string_error_token(tok, token, tok->start, location);
400	346	}
401	377	}
402	402k	if (c == quote) {
403	33.5k	end_quote_size += 1;
404	33.5k	}
405	369k	else {
406	369k	end_quote_size = 0;
407	369k	if (c == '\\') {
408	29.6k	c = tok_nextc(tok); /* skip escaped char */
409	29.6k	if (c == quote) { /* but record whether the escaped char was a quote */
410	952	has_escaped_quote = 1;
411	952	}
412	29.6k	if (c == '\r') {
413	0	c = tok_nextc(tok);
414	0	}
415	29.6k	}
416	369k	}
417	402k	}
418
419	38.0k	p_start = tok->start;
420	38.0k	p_end = tok->cur;
421	38.0k	return MAKE_TOKEN(STRING);
422	38.4k	}
423
424		int
425		_PyLexer_get_ftstring(struct tok_state tok, ftstring_state current, struct token *token)
426	58.1k	{
427	58.1k	assert(current == _PyLexer_CurrentFTString(tok) && current->mode != FTSTRING_MODE_EXPRESSION);
428	58.1k	assert((current->quote_size == 1 \|\| current->quote_size == 3) &&
429	58.1k	current->replacement_depth <= MAX_EXPR_NESTING);
430	58.1k	_PyTok_Off p_start = -1;
431	58.1k	_PyTok_Off p_end = -1;
432	58.1k	int token_type;
433	58.1k	int end_quote_size = 0;
434	58.1k	int unicode_escape = 0;
435	58.1k	int quote = current->quote;
436	58.1k	int quote_size = current->quote_size;
437	58.1k	int in_format_spec = current->mode == FTSTRING_MODE_FORMAT_SPEC;
438	58.1k	int raw = _PyLexer_IsRawString(current->kind);
439
440	58.1k	tok->start = tok->cur;
441	58.1k	tok->start_loc = (_PyTok_Loc){tok->lineno, _PyLexer_ByteColumn(tok)};
442
443	228k	while (end_quote_size != quote_size) {
444	209k	int c = tok_nextc(tok);
445	209k	if (tok->done == E_ERROR \|\| tok->done == E_DECODE) {
446	0	return MAKE_TOKEN(ERRORTOKEN);
447	0	}
448
449	209k	if (c == EOF \|\| (quote_size == 1 && c == '\n')) {
450	304	if (tok_failed(tok)) {
451	0	return MAKE_TOKEN(ERRORTOKEN);
452	0	}
453
454	304	if (in_format_spec && c == '\n') {
455	47	return MAKE_TOKEN(_PyTokenizer_syntaxerror(
456	47	tok,
457	47	"%c-string: newlines are not allowed in format specifiers for single quoted %c-strings",
458	47	_PyLexer_StringPrefix(current->kind),
459	47	_PyLexer_StringPrefix(current->kind)));
460	47	}
461
462	257	int end_lineno = tok->lineno;
463	257	_PyTok_Loc location = current->start_loc;
464	257	const char *line = _PyTok_SourcePointer(
465	257	&tok->source, current->start - location.byte_col);
466	257	Py_ssize_t cursor_offset = (Py_ssize_t)location.byte_col + 1;
467
468	257	if (quote_size == 3) {
469	18	_PyTokenizer_syntaxerror_at(
470	18	tok, line, cursor_offset, location.lineno, -1, -1,
471	18	"unterminated triple-quoted %c-string literal"
472	18	" (detected at line %d)",
473	18	_PyLexer_StringPrefix(current->kind), end_lineno);
474	18	if (c != '\n') {
475	18	tok->done = E_EOFS;
476	18	}
477	18	return string_error_token(tok, token,
478	18	current->start, location);
479	18	}
480	239	else {
481	239	_PyTokenizer_syntaxerror_at(
482	239	tok, line, cursor_offset, location.lineno, -1, -1,
483	239	"unterminated %c-string literal (detected at line %d)",
484	239	_PyLexer_StringPrefix(current->kind), end_lineno);
485	239	return string_error_token(tok, token,
486	239	current->start, location);
487	239	}
488	257	}
489
490	209k	if (c == quote) {
491	26.3k	end_quote_size += 1;
492	26.3k	continue;
493	183k	} else {
494	183k	end_quote_size = 0;
495	183k	}
496
497	183k	if (c == '{') {
498	34.0k	int peek = tok_nextc(tok);
499	34.0k	if (peek != '{' \|\| in_format_spec) {
500	30.8k	tok_backup(tok, peek);
501	30.8k	_PyTok_Off expr_start = tok->cur;
502	30.8k	tok_backup(tok, c);
503	30.8k	if (begin_ftstring_expr(tok, current, expr_start) < 0) {
504	5	return MAKE_TOKEN(ERRORTOKEN);
505	5	}
506	30.8k	p_start = tok->start;
507	30.8k	p_end = tok->cur;
508	30.8k	if (p_start == p_end) {
509	16.2k	return _PyLexer_get_normal(tok, current, token);
510	16.2k	}
511	30.8k	} else {
512	3.19k	p_start = tok->start;
513	3.19k	p_end = tok->cur - 1;
514	3.19k	}
515	17.7k	goto emit_middle;
516	148k	} else if (c == '}') {
517	5.18k	if (unicode_escape) {
518	246	p_start = tok->start;
519	246	p_end = tok->cur;
520	246	goto emit_middle;
521	246	}
522	4.93k	int peek = tok_nextc(tok);
523
524	4.93k	int bracket_depth = _PyLexer_FTStringBracketDepth(tok, current);
525	4.93k	if (peek == '}' && !in_format_spec && bracket_depth == 0) {
526	954	p_start = tok->start;
527	954	p_end = tok->cur - 1;
528	954	}
529	3.98k	else {
530	3.98k	tok_backup(tok, peek);
531	3.98k	if (!in_format_spec && bracket_depth == 0) {
532	57	if (tok->start == tok->cur - 1) {
533	39	return MAKE_TOKEN(_PyTokenizer_syntaxerror(
534	39	tok, "%c-string: single '}' is not allowed",
535	39	_PyLexer_StringPrefix(current->kind)));
536	39	}
537	18	tok_backup(tok, c);
538	18	p_start = tok->start;
539	18	p_end = tok->cur;
540	18	goto emit_middle;
541	57	}
542	3.92k	tok_backup(tok, c);
543	3.92k	current->mode = FTSTRING_MODE_EXPRESSION;
544	3.92k	p_start = tok->start;
545	3.92k	p_end = tok->cur;
546	3.92k	}
547	4.88k	goto emit_middle;
548	143k	} else if (c == '\\') {
549	5.34k	int peek = tok_nextc(tok);
550	5.34k	if (peek == '\r') {
551	0	peek = tok_nextc(tok);
552	0	}
553	5.34k	if (peek == '{' \|\| peek == '}') {
554	1.19k	if (!raw) {
555	1.12k	if (_PyTokenizer_warn_invalid_escape_sequence(tok, peek)) {
556	0	return MAKE_TOKEN(ERRORTOKEN);
557	0	}
558	1.12k	}
559	1.19k	tok_backup(tok, peek);
560	1.19k	continue;
561	1.19k	}
562
563	4.15k	if (!raw) {
564	3.94k	if (peek == 'N') {
565		/* Handle named unicode escapes (\N{BULLET}) */
566	495	peek = tok_nextc(tok);
567	495	if (peek == '{') {
568	288	unicode_escape = 1;
569	288	} else {
570	207	tok_backup(tok, peek);
571	207	}
572	495	}
573	3.94k	}
574	4.15k	}
575	183k	}
576
577	18.5k	p_start = tok->start;
578	18.5k	p_end = tok->cur;
579	18.5k	if (p_end - quote_size == p_start) {
580	13.5k	int end_token = FTSTRING_END(current);
581	13.5k	_PyLexer_PopFTString(tok);
582	13.5k	return MAKE_TOKEN(end_token);
583	13.5k	}
584	11.2k	for (int i = 0; i < quote_size; i++) {
585	6.18k	tok_backup(tok, quote);
586	6.18k	}
587	5.08k	p_end = tok->cur;
588	28.0k	emit_middle:
589	28.0k	token_type = MAKE_TOKEN(FTSTRING_MIDDLE(current));
590	28.0k	token->is_raw = raw;
591	28.0k	return token_type;
592	5.08k	}