/src/cpython/Parser/tokenizer/decoder.c

Source
#include "Python.h"
#include "pycore_codecs.h"
#include "pycore_global_strings.h"
#include "pycore_runtime.h"
#include "errcode.h"

#include "reader_internal.h"
#include "helpers.h"
#include "../lexer/state.h"

#define UNICODE_BOM 0xFEFF
#define UTF16_LE_BOM "\xff\xfe"
#define UTF16_BE_BOM "\xfe\xff"
#define UTF16_BOM_LENGTH ((Py_ssize_t)(sizeof(UTF16_LE_BOM) - 1))
#define UTF8_BOM "\xef\xbb\xbf"
#define UTF8_BOM_LENGTH ((Py_ssize_t)(sizeof(UTF8_BOM) - 1))
#define NORMALIZED_LINE_INITIAL_CAPACITY 256

char *
_PyTok_CopyBytes(const char *data, Py_ssize_t len)
{
    if (len < 0 || len == PY_SSIZE_T_MAX) {
        PyErr_NoMemory();
        return NULL;
    }
    char *copy = PyMem_Malloc((size_t)len + 1);
    if (copy == NULL) {
        PyErr_NoMemory();
        return NULL;
    }
    memcpy(copy, data, (size_t)len);
    copy[len] = '\0';
    return copy;
}

static void
chunk_release_data(_PyTok_Chunk *chunk)
{
    switch (chunk->ownership) {
        case _PYTOK_CHUNK_BORROWED:
            break;
        case _PYTOK_CHUNK_PYMEM:
            PyMem_Free(chunk->data);
            break;
        case _PYTOK_CHUNK_PYOBJECT:
            Py_DECREF(chunk->owner);
            break;
    }
}

void
_PyTok_ChunkClear(_PyTok_Chunk *chunk)
{
    chunk_release_data(chunk);
    *chunk = (_PyTok_Chunk){0};
}

static int
chunk_set_unicode(struct tok_state *tok, _PyTok_Chunk *chunk,
                  PyObject *unicode, int strip_bom)
{
    Py_ssize_t utf8_len;
    const char *utf8 = PyUnicode_AsUTF8AndSize(unicode, &utf8_len);
    if (utf8 == NULL) {
        Py_DECREF(unicode);
        tok->done = PyErr_ExceptionMatches(PyExc_MemoryError)
            ? E_NOMEM : E_DECODE;
        return -1;
    }
    if (strip_bom && PyUnicode_GET_LENGTH(unicode) > 0 &&
            PyUnicode_ReadChar(unicode, 0) == UNICODE_BOM) {
        utf8 += UTF8_BOM_LENGTH;
        utf8_len -= UTF8_BOM_LENGTH;
    }
    chunk_release_data(chunk);
    chunk->owner = unicode;
    chunk->data = (char *)utf8;
    chunk->len = utf8_len;
    chunk->ownership = _PYTOK_CHUNK_PYOBJECT;
    return 0;
}

char *
_PyTok_NormalizeNewlines(const char *data, Py_ssize_t len,
                         Py_ssize_t *out_len)
{
    if (len < 0 || len == PY_SSIZE_T_MAX) {
        PyErr_NoMemory();
        return NULL;
    }
    char *result = PyMem_Malloc((size_t)len + 1);
    if (result == NULL) {
        PyErr_NoMemory();
        return NULL;
    }
    Py_ssize_t write = 0;
    for (Py_ssize_t read = 0; read < len; read++) {
        char c = data[read];
        if (c == '\r') {
            if (read + 1 < len && data[read + 1] == '\n') {
                read++;
            }
            c = '\n';
        }
        result[write++] = c;
    }
    result[write] = '\0';
    *out_len = write;
    return result;
}

int
_PyTok_SetEncoding(struct tok_state *tok, const char *encoding)
{
    char *copy = _PyTok_CopyBytes(encoding, strlen(encoding));
    if (copy == NULL) {
        tok->done = E_NOMEM;
        return -1;
    }
    PyMem_Free(tok->reader->encoding);
    tok->reader->encoding = copy;
    return 0;
}

static int
find_cookie(const char *line, Py_ssize_t len, char **encoding, int *scan_next)
{
    Py_ssize_t i = 0;
    *encoding = NULL;
    *scan_next = 1;
    for (; i < len; i++) {
        if (line[i] == '#') {
            break;
        }
        if (line[i] == '\n' || line[i] == '\r') {
            return 0;
        }
        if (line[i] != ' ' && line[i] != '\t' && line[i] != '\f') {
            *scan_next = 0;
            return 0;
        }
    }
    const Py_ssize_t marker_len = sizeof("coding") - 1;
    for (; i < len - marker_len; i++) {
        if (memcmp(line + i, "coding", marker_len) != 0) {
            continue;
        }
        const char *cursor = line + i + marker_len;
        if (*cursor != ':' && *cursor != '=') {
            continue;
        }
        do {
            cursor++;
        } while (cursor < line + len &&
                 (*cursor == ' ' || *cursor == '\t'));
        const char *start = cursor;
        while (cursor < line + len &&
                (Py_ISALNUM(*cursor) || *cursor == '-' ||
                 *cursor == '_' || *cursor == '.')) {
            cursor++;
        }
        if (cursor == start) {
            continue;
        }
        Py_ssize_t encoding_len = cursor - start;
        // Include the longest recognized prefix and its NUL terminator.
        char normalized[sizeof("iso-latin-1-")];
        const Py_ssize_t prefix_len = Py_MIN(
            encoding_len, (Py_ssize_t)sizeof(normalized) - 1);
        Py_ssize_t n;
        for (n = 0; n < prefix_len; n++) {
            normalized[n] = start[n] == '_' ? '-' : Py_TOLOWER(start[n]);
        }
        normalized[n] = '\0';
        const char *canonical = start;
        if (strcmp(normalized, "utf-8") == 0 ||
                strncmp(normalized, "utf-8-", sizeof("utf-8-") - 1) == 0) {
            canonical = "utf-8";
        }
        else if (strcmp(normalized, "latin-1") == 0 ||
                 strcmp(normalized, "iso-8859-1") == 0 ||
                 strcmp(normalized, "iso-latin-1") == 0 ||
                 strncmp(normalized, "latin-1-", sizeof("latin-1-") - 1) == 0 ||
                 strncmp(normalized, "iso-8859-1-", sizeof("iso-8859-1-") - 1) == 0 ||
                 strncmp(normalized, "iso-latin-1-", sizeof("iso-latin-1-") - 1) == 0) {
            canonical = "iso-8859-1";
        }
        if (canonical != start) {
            encoding_len = strlen(canonical);
        }
        *encoding = _PyTok_CopyBytes(canonical, encoding_len);
        if (*encoding == NULL) {
            return -1;
        }
        *scan_next = 0;
        return 0;
    }
    return 0;
}

_PyTok_EncodingResult
_PyTok_DetectEncoding(struct tok_state *tok, const _PyTok_Chunk *first,
                      const _PyTok_Chunk *second, int final,
                      Py_ssize_t *bom_len)
{
    int bom = first->len >= UTF8_BOM_LENGTH &&
        memcmp(first->data, UTF8_BOM, UTF8_BOM_LENGTH) == 0;
    *bom_len = bom ? UTF8_BOM_LENGTH : 0;

    char *cookie = NULL;
    int scan_next = 0;
    int cookie_line = 1;
    const char *first_data = first->data + *bom_len;
    Py_ssize_t first_len = first->len - *bom_len;
    if (find_cookie(first_data, first_len, &cookie, &scan_next) < 0) {
        return _PYTOK_ENCODING_ERROR;
    }
    if (cookie == NULL && scan_next && second != NULL) {
        if (find_cookie(second->data, second->len, &cookie, &scan_next) < 0) {
            return _PYTOK_ENCODING_ERROR;
        }
        cookie_line = 2;
    }
    else if (cookie == NULL && scan_next && !final) {
        return _PYTOK_ENCODING_NEED_SECOND_LINE;
    }

    if (cookie == NULL) {
        if (bom && _PyTok_SetEncoding(tok, "utf-8") < 0) {
            return _PYTOK_ENCODING_ERROR;
        }
        return _PYTOK_ENCODING_DONE;
    }
    if (bom && strcmp(cookie, "utf-8") != 0) {
        const _PyTok_Chunk *line = cookie_line == 2 ? second : first;
        const char *line_data = line->data + (cookie_line == 1 ? *bom_len : 0);
        Py_ssize_t line_len = line->len - (cookie_line == 1 ? *bom_len : 0);
        int end_col = (int)Py_MIN(line_len, INT_MAX);
        if (end_col > 0 && (line_data[end_col - 1] == '\n' ||
                            line_data[end_col - 1] == '\r')) {
            end_col--;
        }
        _PyTokenizer_syntaxerror_at(
            tok, line_data, 0, cookie_line, 0, end_col,
            "encoding problem: %s with BOM", cookie);
        PyMem_Free(cookie);
        return _PYTOK_ENCODING_ERROR;
    }
    PyMem_Free(tok->reader->encoding);
    tok->reader->encoding = cookie;
    return _PYTOK_ENCODING_DONE;
}

int
_PyTok_DecodeOnce(struct tok_state *tok, _PyTok_Chunk *chunk,
                  const char *encoding, const char *errors)
{
    PyObject *unicode = PyUnicode_Decode(
        chunk->data, chunk->len, encoding, errors);
    if (unicode == NULL) {
        tok->done = PyErr_ExceptionMatches(PyExc_MemoryError)
            ? E_NOMEM : E_DECODE;
        return -1;
    }
    return chunk_set_unicode(tok, chunk, unicode, 0);
}

static Py_ssize_t
raw_line_length(const char *data, Py_ssize_t len)
{
    for (Py_ssize_t i = 0; i < len; i++) {
        if (data[i] == '\n') {
            return i + 1;
        }
        if (data[i] == '\r') {
            return i + 1 < len && data[i + 1] == '\n' ? i + 2 : i + 1;
        }
    }
    return len;
}

static int
store_prepared_source(struct tok_state *tok, const char *data, Py_ssize_t len,
                      int preserve_crlf, int add_final_newline)
{
    Py_ssize_t pos = 0;
    // SourceAppendLine copies the bytes, so reuse this buffer for each line.
    char *normalized = NULL;
    Py_ssize_t capacity = 0;
    while (pos < len) {
        Py_ssize_t raw_line_len;
        if (preserve_crlf) {
            const char *newline = memchr(data + pos, '\n', len - pos);
            raw_line_len = newline == NULL
                ? len - pos : newline - data - pos + 1;
        }
        else {
            raw_line_len = raw_line_length(data + pos, len - pos);
        }
        int terminated = preserve_crlf
            ? data[pos + raw_line_len - 1] == '\n'
            : data[pos + raw_line_len - 1] == '\n' ||
              data[pos + raw_line_len - 1] == '\r';
        int add_newline = add_final_newline &&
            pos + raw_line_len == len && !terminated;
        const char *line = data + pos;
        Py_ssize_t line_len = raw_line_len;
        // raw_line_length stops at the first CR or LF, so any CR is in
        // the line terminator.
        Py_ssize_t terminator_len = 0;
        if (!preserve_crlf) {
            if (line[line_len - 1] == '\r') {
                terminator_len = 1;
            }
            else if (line_len > 1 && line[line_len - 2] == '\r') {
                terminator_len = 2;
            }
        }
        if (terminator_len || add_newline) {
            Py_ssize_t payload_len = line_len - terminator_len;
            if (payload_len > PY_SSIZE_T_MAX - 2) {
                PyErr_NoMemory();
                tok->done = E_NOMEM;
                goto error;
            }
            Py_ssize_t needed = payload_len + 2;
            if (_PyTok_ReserveBuffer(&normalized, &capacity, needed,
                                    NORMALIZED_LINE_INITIAL_CAPACITY) < 0) {
                tok->done = E_NOMEM;
                goto error;
            }
            memcpy(normalized, line, payload_len);
            normalized[payload_len] = '\n';
            line_len = payload_len + 1;
            normalized[line_len] = '\0';
            line = normalized;
        }
        _PyTok_Off appended = _PyTok_SourceAppendLine(
            &tok->source, line, line_len);
        if (appended < 0) {
            tok->done = PyErr_ExceptionMatches(PyExc_MemoryError)
                ? E_NOMEM : E_ERROR;
            goto error;
        }
        tok->reader->prepared_final_newline_is_implicit = add_newline;
        pos += raw_line_len;
    }
    PyMem_Free(normalized);
    return 0;
error:
    PyMem_Free(normalized);
    return -1;
}

int
_PyTok_PrepareString(struct tok_state *tok, const char *input, int utf8_only,
                     int exec_input, int preserve_crlf)
{
    Py_ssize_t raw_len = strlen(input);
    char *raw = (char *)input;

    if (utf8_only) {
        if (_PyTok_SetEncoding(tok, "utf-8") < 0) {
            return -1;
        }
    }
    else {
        Py_ssize_t first_original_len = raw_line_length(raw, raw_len);
        _PyTok_Chunk first = {
            .data = raw,
            .len = first_original_len,
            .ownership = _PYTOK_CHUNK_BORROWED,
        };
        _PyTok_Chunk second = {0};
        int have_second = first_original_len < raw_len;
        if (have_second) {
            second.data = raw + first_original_len;
            second.len = raw_line_length(second.data,
                                         raw_len - first_original_len);
        }
        Py_ssize_t bom_len;
        _PyTok_EncodingResult detection = _PyTok_DetectEncoding(
            tok, &first, have_second ? &second : NULL, 1, &bom_len);
        if (detection == _PYTOK_ENCODING_ERROR) {
            return -1;
        }
        raw += bom_len;
        raw_len -= bom_len;
    }

    _PyTok_Chunk decoded = {
        .data = raw,
        .len = raw_len,
        .ownership = _PYTOK_CHUNK_BORROWED,
    };
    const char *encoding = tok->reader->encoding;
    if (encoding != NULL && strcmp(encoding, "utf-8") != 0) {
        if (_PyTok_DecodeOnce(
                tok, &decoded, encoding, NULL) < 0) {
            return -1;
        }
    }

    int stored = store_prepared_source(
        tok, decoded.data, decoded.len, preserve_crlf, exec_input);
    _PyTok_ChunkClear(&decoded);
    if (stored < 0) {
        return -1;
    }
    if (!utf8_only &&
            (encoding == NULL || strcmp(encoding, "utf-8") == 0) &&
            !_PyTokenizer_ensure_utf8(_PyTok_SourceData(&tok->source), tok, 1)) {
        return -1;
    }
    return 0;
}

int
_PyTok_StartDecoder(struct tok_state *tok, const char *errors)
{
    _PyTok_Reader *reader = tok->reader;
    if (reader->encoding == NULL || reader->decoder != NULL) {
        return 0;
    }
    if (reader->kind == _PYTOK_READER_FILE &&
            strcmp(reader->encoding, "utf-8") == 0) {
        return 0;
    }

    PyObject *codec = _PyCodec_LookupTextEncoding(reader->encoding, NULL);
    if (codec != NULL) {
        reader->decoder = _PyCodecInfo_GetIncrementalDecoder(codec, errors);
        Py_DECREF(codec);
    }
    if (reader->decoder == NULL) {
        tok->done = PyErr_ExceptionMatches(PyExc_MemoryError)
            ? E_NOMEM : E_DECODE;
        if (reader->kind == _PYTOK_READER_FILE) {
            _PyTokenizer_raise_init_error(
                tok->filename != NULL ? tok->filename : Py_None);
        }
        return -1;
    }
    return 0;
}

int
_PyTok_DecodeChunk(struct tok_state *tok, _PyTok_Chunk *chunk, int final)
{
    _PyTok_Reader *reader = tok->reader;
    if (reader->decoder == NULL) {
        return 0;
    }
    int strip_bom = reader->kind == _PYTOK_READER_READLINE &&
        chunk->len >= UTF16_BOM_LENGTH &&
        (memcmp(chunk->data, UTF16_LE_BOM, UTF16_BOM_LENGTH) == 0 ||
         memcmp(chunk->data, UTF16_BE_BOM, UTF16_BOM_LENGTH) == 0);
    PyObject *input;
    if (chunk->ownership == _PYTOK_CHUNK_PYOBJECT &&
            PyBytes_Check(chunk->owner) &&
            chunk->data == PyBytes_AS_STRING(chunk->owner)) {
        input = Py_NewRef(chunk->owner);
    }
    else {
        input = PyBytes_FromStringAndSize(chunk->data, chunk->len);
    }
    if (input == NULL) {
        tok->done = E_NOMEM;
        return -1;
    }
    PyObject *unicode = PyObject_CallMethodObjArgs(
        reader->decoder, &_Py_ID(decode), input,
        final ? Py_True : Py_False, NULL);
    Py_DECREF(input);
    if (unicode == NULL) {
        tok->done = PyErr_ExceptionMatches(PyExc_MemoryError)
            ? E_NOMEM : E_DECODE;
        if (reader->kind == _PYTOK_READER_FILE) {
            _PyTokenizer_raise_init_error(
                tok->filename != NULL ? tok->filename : Py_None);
        }
        return -1;
    }
    if (!PyUnicode_Check(unicode)) {
        PyErr_Format(PyExc_TypeError,
                     "decoder should return a string result, not '%.200s'",
                     Py_TYPE(unicode)->tp_name);
        Py_DECREF(unicode);
        tok->done = E_DECODE;
        return -1;
    }
    return chunk_set_unicode(tok, chunk, unicode, strip_bom);
}

int
_PyTok_DecoderHasBufferedInput(struct tok_state *tok)
{
    if (tok->reader->decoder == NULL) {
        return 0;
    }
    PyObject *state = PyObject_CallMethodNoArgs(
        tok->reader->decoder, &_Py_ID(getstate));
    if (state == NULL) {
        tok->done = PyErr_ExceptionMatches(PyExc_MemoryError)
            ? E_NOMEM : E_DECODE;
        return -1;
    }
    if (!PyTuple_Check(state) || PyTuple_GET_SIZE(state) != 2 ||
            !PyBytes_Check(PyTuple_GET_ITEM(state, 0)) ||
            !PyLong_Check(PyTuple_GET_ITEM(state, 1))) {
        Py_DECREF(state);
        PyErr_SetString(PyExc_TypeError,
                        "incremental decoder getstate() must return (bytes, int)");
        tok->done = E_DECODE;
        return -1;
    }
    int pending = PyBytes_GET_SIZE(PyTuple_GET_ITEM(state, 0)) != 0;
    Py_DECREF(state);
    return pending;
}

Coverage Report

Created: 2026-10-17 06:15

Line	Count	Source
1		#include "Python.h"
2		#include "pycore_codecs.h"
3		#include "pycore_global_strings.h"
4		#include "pycore_runtime.h"
5		#include "errcode.h"
6
7		#include "reader_internal.h"
8		#include "helpers.h"
9		#include "../lexer/state.h"
10
11	0	#define UNICODE_BOM 0xFEFF
12	20	#define UTF16_LE_BOM "\xff\xfe"
13	4	#define UTF16_BE_BOM "\xfe\xff"
14	24	#define UTF16_BOM_LENGTH ((Py_ssize_t)(sizeof(UTF16_LE_BOM) - 1))
15	65.1k	#define UTF8_BOM "\xef\xbb\xbf"
16	65.9k	#define UTF8_BOM_LENGTH ((Py_ssize_t)(sizeof(UTF8_BOM) - 1))
17	76.1k	#define NORMALIZED_LINE_INITIAL_CAPACITY 256
18
19		char *
20		_PyTok_CopyBytes(const char *data, Py_ssize_t len)
21	47.8k	{
22	47.8k	if (len < 0 \|\| len == PY_SSIZE_T_MAX) {
23	0	PyErr_NoMemory();
24	0	return NULL;
25	0	}
26	47.8k	char *copy = PyMem_Malloc((size_t)len + 1);
27	47.8k	if (copy == NULL) {
28	0	PyErr_NoMemory();
29	0	return NULL;
30	0	}
31	47.8k	memcpy(copy, data, (size_t)len);
32	47.8k	copy[len] = '\0';
33	47.8k	return copy;
34	47.8k	}
35
36		static void
37		chunk_release_data(_PyTok_Chunk *chunk)
38	452k	{
39	452k	switch (chunk->ownership) {
40	449k	case _PYTOK_CHUNK_BORROWED:
41	449k	break;
42	0	case _PYTOK_CHUNK_PYMEM:
43	0	PyMem_Free(chunk->data);
44	0	break;
45	3.41k	case _PYTOK_CHUNK_PYOBJECT:
46	3.41k	Py_DECREF(chunk->owner);
47	3.41k	break;
48	452k	}
49	452k	}
50
51		void
52		_PyTok_ChunkClear(_PyTok_Chunk *chunk)
53	449k	{
54	449k	chunk_release_data(chunk);
55	449k	*chunk = (_PyTok_Chunk){0};
56	449k	}
57
58		static int
59		chunk_set_unicode(struct tok_state tok, _PyTok_Chunk chunk,
60		PyObject *unicode, int strip_bom)
61	3.12k	{
62	3.12k	Py_ssize_t utf8_len;
63	3.12k	const char *utf8 = PyUnicode_AsUTF8AndSize(unicode, &utf8_len);
64	3.12k	if (utf8 == NULL) {
65	55	Py_DECREF(unicode);
66	55	tok->done = PyErr_ExceptionMatches(PyExc_MemoryError)
67	55	? E_NOMEM : E_DECODE;
68	55	return -1;
69	55	}
70	3.06k	if (strip_bom && PyUnicode_GET_LENGTH(unicode) > 0 &&
71	0	PyUnicode_ReadChar(unicode, 0) == UNICODE_BOM) {
72	0	utf8 += UTF8_BOM_LENGTH;
73	0	utf8_len -= UTF8_BOM_LENGTH;
74	0	}
75	3.06k	chunk_release_data(chunk);
76	3.06k	chunk->owner = unicode;
77	3.06k	chunk->data = (char *)utf8;
78	3.06k	chunk->len = utf8_len;
79	3.06k	chunk->ownership = _PYTOK_CHUNK_PYOBJECT;
80	3.06k	return 0;
81	3.12k	}
82
83		char *
84		_PyTok_NormalizeNewlines(const char *data, Py_ssize_t len,
85		Py_ssize_t *out_len)
86	0	{
87	0	if (len < 0 \|\| len == PY_SSIZE_T_MAX) {
88	0	PyErr_NoMemory();
89	0	return NULL;
90	0	}
91	0	char *result = PyMem_Malloc((size_t)len + 1);
92	0	if (result == NULL) {
93	0	PyErr_NoMemory();
94	0	return NULL;
95	0	}
96	0	Py_ssize_t write = 0;
97	0	for (Py_ssize_t read = 0; read < len; read++) {
98	0	char c = data[read];
99	0	if (c == '\r') {
100	0	if (read + 1 < len && data[read + 1] == '\n') {
101	0	read++;
102	0	}
103	0	c = '\n';
104	0	}
105	0	result[write++] = c;
106	0	}
107	0	result[write] = '\0';
108	0	*out_len = write;
109	0	return result;
110	0	}
111
112		int
113		_PyTok_SetEncoding(struct tok_state tok, const char encoding)
114	42.8k	{
115	42.8k	char *copy = _PyTok_CopyBytes(encoding, strlen(encoding));
116	42.8k	if (copy == NULL) {
117	0	tok->done = E_NOMEM;
118	0	return -1;
119	0	}
120	42.8k	PyMem_Free(tok->reader->encoding);
121	42.8k	tok->reader->encoding = copy;
122	42.8k	return 0;
123	42.8k	}
124
125		static int
126		find_cookie(const char line, Py_ssize_t len, char encoding, int scan_next)
127	22.8k	{
128	22.8k	Py_ssize_t i = 0;
129	22.8k	*encoding = NULL;
130	22.8k	*scan_next = 1;
131	23.7k	for (; i < len; i++) {
132	23.6k	if (line[i] == '#') {
133	5.45k	break;
134	5.45k	}
135	18.2k	if (line[i] == '\n' \|\| line[i] == '\r') {
136	375	return 0;
137	375	}
138	17.8k	if (line[i] != ' ' && line[i] != '\t' && line[i] != '\f') {
139	16.9k	*scan_next = 0;
140	16.9k	return 0;
141	16.9k	}
142	17.8k	}
143	5.48k	const Py_ssize_t marker_len = sizeof("coding") - 1;
144	258k	for (; i < len - marker_len; i++) {
145	257k	if (memcmp(line + i, "coding", marker_len) != 0) {
146	251k	continue;
147	251k	}
148	5.73k	const char *cursor = line + i + marker_len;
149	5.73k	if (cursor != ':' && cursor != '=') {
150	277	continue;
151	277	}
152	6.31k	do {
153	6.31k	cursor++;
154	6.31k	} while (cursor < line + len &&
155	6.29k	(cursor == ' ' \|\| cursor == '\t'));
156	5.45k	const char *start = cursor;
157	41.5k	while (cursor < line + len &&
158	39.9k	(Py_ISALNUM(cursor) \|\| cursor == '-' \|\|
159	36.0k	cursor == '_' \|\| cursor == '.')) {
160	36.0k	cursor++;
161	36.0k	}
162	5.45k	if (cursor == start) {
163	408	continue;
164	408	}
165	5.04k	Py_ssize_t encoding_len = cursor - start;
166		// Include the longest recognized prefix and its NUL terminator.
167	5.04k	char normalized[sizeof("iso-latin-1-")];
168	5.04k	const Py_ssize_t prefix_len = Py_MIN(
169	5.04k	encoding_len, (Py_ssize_t)sizeof(normalized) - 1);
170	5.04k	Py_ssize_t n;
171	33.8k	for (n = 0; n < prefix_len; n++) {
172	28.7k	normalized[n] = start[n] == '_' ? '-' : Py_TOLOWER(start[n]);
173	28.7k	}
174	5.04k	normalized[n] = '\0';
175	5.04k	const char *canonical = start;
176	5.04k	if (strcmp(normalized, "utf-8") == 0 \|\|
177	5.04k	strncmp(normalized, "utf-8-", sizeof("utf-8-") - 1) == 0) {
178	3	canonical = "utf-8";
179	3	}
180	5.04k	else if (strcmp(normalized, "latin-1") == 0 \|\|
181	5.03k	strcmp(normalized, "iso-8859-1") == 0 \|\|
182	5.03k	strcmp(normalized, "iso-latin-1") == 0 \|\|
183	5.02k	strncmp(normalized, "latin-1-", sizeof("latin-1-") - 1) == 0 \|\|
184	5.02k	strncmp(normalized, "iso-8859-1-", sizeof("iso-8859-1-") - 1) == 0 \|\|
185	5.02k	strncmp(normalized, "iso-latin-1-", sizeof("iso-latin-1-") - 1) == 0) {
186	20	canonical = "iso-8859-1";
187	20	}
188	5.04k	if (canonical != start) {
189	23	encoding_len = strlen(canonical);
190	23	}
191	5.04k	*encoding = _PyTok_CopyBytes(canonical, encoding_len);
192	5.04k	if (*encoding == NULL) {
193	0	return -1;
194	0	}
195	5.04k	*scan_next = 0;
196	5.04k	return 0;
197	5.04k	}
198	433	return 0;
199	5.48k	}
200
201		_PyTok_EncodingResult
202		_PyTok_DetectEncoding(struct tok_state tok, const _PyTok_Chunk first,
203		const _PyTok_Chunk *second, int final,
204		Py_ssize_t *bom_len)
205	22.2k	{
206	22.2k	int bom = first->len >= UTF8_BOM_LENGTH &&
207	21.4k	memcmp(first->data, UTF8_BOM, UTF8_BOM_LENGTH) == 0;
208	22.2k	*bom_len = bom ? UTF8_BOM_LENGTH : 0;
209
210	22.2k	char *cookie = NULL;
211	22.2k	int scan_next = 0;
212	22.2k	int cookie_line = 1;
213	22.2k	const char first_data = first->data + bom_len;
214	22.2k	Py_ssize_t first_len = first->len - *bom_len;
215	22.2k	if (find_cookie(first_data, first_len, &cookie, &scan_next) < 0) {
216	0	return _PYTOK_ENCODING_ERROR;
217	0	}
218	22.2k	if (cookie == NULL && scan_next && second != NULL) {
219	585	if (find_cookie(second->data, second->len, &cookie, &scan_next) < 0) {
220	0	return _PYTOK_ENCODING_ERROR;
221	0	}
222	585	cookie_line = 2;
223	585	}
224	21.6k	else if (cookie == NULL && scan_next && !final) {
225	0	return _PYTOK_ENCODING_NEED_SECOND_LINE;
226	0	}
227
228	22.2k	if (cookie == NULL) {
229	17.2k	if (bom && _PyTok_SetEncoding(tok, "utf-8") < 0) {
230	0	return _PYTOK_ENCODING_ERROR;
231	0	}
232	17.2k	return _PYTOK_ENCODING_DONE;
233	17.2k	}
234	5.04k	if (bom && strcmp(cookie, "utf-8") != 0) {
235	6	const _PyTok_Chunk *line = cookie_line == 2 ? second : first;
236	6	const char line_data = line->data + (cookie_line == 1 ? bom_len : 0);
237	6	Py_ssize_t line_len = line->len - (cookie_line == 1 ? *bom_len : 0);
238	6	int end_col = (int)Py_MIN(line_len, INT_MAX);
239	6	if (end_col > 0 && (line_data[end_col - 1] == '\n' \|\|
240	5	line_data[end_col - 1] == '\r')) {
241	2	end_col--;
242	2	}
243	6	_PyTokenizer_syntaxerror_at(
244	6	tok, line_data, 0, cookie_line, 0, end_col,
245	6	"encoding problem: %s with BOM", cookie);
246	6	PyMem_Free(cookie);
247	6	return _PYTOK_ENCODING_ERROR;
248	6	}
249	5.04k	PyMem_Free(tok->reader->encoding);
250	5.04k	tok->reader->encoding = cookie;
251	5.04k	return _PYTOK_ENCODING_DONE;
252	5.04k	}
253
254		int
255		_PyTok_DecodeOnce(struct tok_state tok, _PyTok_Chunk chunk,
256		const char encoding, const char errors)
257	5.03k	{
258	5.03k	PyObject *unicode = PyUnicode_Decode(
259	5.03k	chunk->data, chunk->len, encoding, errors);
260	5.03k	if (unicode == NULL) {
261	1.92k	tok->done = PyErr_ExceptionMatches(PyExc_MemoryError)
262	1.92k	? E_NOMEM : E_DECODE;
263	1.92k	return -1;
264	1.92k	}
265	3.11k	return chunk_set_unicode(tok, chunk, unicode, 0);
266	5.03k	}
267
268		static Py_ssize_t
269		raw_line_length(const char *data, Py_ssize_t len)
270	285k	{
271	8.84M	for (Py_ssize_t i = 0; i < len; i++) {
272	8.76M	if (data[i] == '\n') {
273	189k	return i + 1;
274	189k	}
275	8.57M	if (data[i] == '\r') {
276	15.2k	return i + 1 < len && data[i + 1] == '\n' ? i + 2 : i + 1;
277	15.2k	}
278	8.57M	}
279	81.2k	return len;
280	285k	}
281
282		static int
283		store_prepared_source(struct tok_state tok, const char data, Py_ssize_t len,
284		int preserve_crlf, int add_final_newline)
285	63.0k	{
286	63.0k	Py_ssize_t pos = 0;
287		// SourceAppendLine copies the bytes, so reuse this buffer for each line.
288	63.0k	char *normalized = NULL;
289	63.0k	Py_ssize_t capacity = 0;
290	321k	while (pos < len) {
291	258k	Py_ssize_t raw_line_len;
292	258k	if (preserve_crlf) {
293	0	const char *newline = memchr(data + pos, '\n', len - pos);
294	0	raw_line_len = newline == NULL
295	0	? len - pos : newline - data - pos + 1;
296	0	}
297	258k	else {
298	258k	raw_line_len = raw_line_length(data + pos, len - pos);
299	258k	}
300	258k	int terminated = preserve_crlf
301	258k	? data[pos + raw_line_len - 1] == '\n'
302	258k	: data[pos + raw_line_len - 1] == '\n' \|\|
303	75.5k	data[pos + raw_line_len - 1] == '\r';
304	258k	int add_newline = add_final_newline &&
305	257k	pos + raw_line_len == len && !terminated;
306	258k	const char *line = data + pos;
307	258k	Py_ssize_t line_len = raw_line_len;
308		// raw_line_length stops at the first CR or LF, so any CR is in
309		// the line terminator.
310	258k	Py_ssize_t terminator_len = 0;
311	258k	if (!preserve_crlf) {
312	258k	if (line[line_len - 1] == '\r') {
313	13.3k	terminator_len = 1;
314	13.3k	}
315	244k	else if (line_len > 1 && line[line_len - 2] == '\r') {
316	788	terminator_len = 2;
317	788	}
318	258k	}
319	258k	if (terminator_len \|\| add_newline) {
320	76.1k	Py_ssize_t payload_len = line_len - terminator_len;
321	76.1k	if (payload_len > PY_SSIZE_T_MAX - 2) {
322	0	PyErr_NoMemory();
323	0	tok->done = E_NOMEM;
324	0	goto error;
325	0	}
326	76.1k	Py_ssize_t needed = payload_len + 2;
327	76.1k	if (_PyTok_ReserveBuffer(&normalized, &capacity, needed,
328	76.1k	NORMALIZED_LINE_INITIAL_CAPACITY) < 0) {
329	0	tok->done = E_NOMEM;
330	0	goto error;
331	0	}
332	76.1k	memcpy(normalized, line, payload_len);
333	76.1k	normalized[payload_len] = '\n';
334	76.1k	line_len = payload_len + 1;
335	76.1k	normalized[line_len] = '\0';
336	76.1k	line = normalized;
337	76.1k	}
338	258k	_PyTok_Off appended = _PyTok_SourceAppendLine(
339	258k	&tok->source, line, line_len);
340	258k	if (appended < 0) {
341	0	tok->done = PyErr_ExceptionMatches(PyExc_MemoryError)
342	0	? E_NOMEM : E_ERROR;
343	0	goto error;
344	0	}
345	258k	tok->reader->prepared_final_newline_is_implicit = add_newline;
346	258k	pos += raw_line_len;
347	258k	}
348	63.0k	PyMem_Free(normalized);
349	63.0k	return 0;
350	0	error:
351	0	PyMem_Free(normalized);
352	0	return -1;
353	63.0k	}
354
355		int
356		_PyTok_PrepareString(struct tok_state tok, const char input, int utf8_only,
357		int exec_input, int preserve_crlf)
358	65.0k	{
359	65.0k	Py_ssize_t raw_len = strlen(input);
360	65.0k	char raw = (char )input;
361
362	65.0k	if (utf8_only) {
363	42.7k	if (_PyTok_SetEncoding(tok, "utf-8") < 0) {
364	0	return -1;
365	0	}
366	42.7k	}
367	22.2k	else {
368	22.2k	Py_ssize_t first_original_len = raw_line_length(raw, raw_len);
369	22.2k	_PyTok_Chunk first = {
370	22.2k	.data = raw,
371	22.2k	.len = first_original_len,
372	22.2k	.ownership = _PYTOK_CHUNK_BORROWED,
373	22.2k	};
374	22.2k	_PyTok_Chunk second = {0};
375	22.2k	int have_second = first_original_len < raw_len;
376	22.2k	if (have_second) {
377	5.60k	second.data = raw + first_original_len;
378	5.60k	second.len = raw_line_length(second.data,
379	5.60k	raw_len - first_original_len);
380	5.60k	}
381	22.2k	Py_ssize_t bom_len;
382	22.2k	_PyTok_EncodingResult detection = _PyTok_DetectEncoding(
383	22.2k	tok, &first, have_second ? &second : NULL, 1, &bom_len);
384	22.2k	if (detection == _PYTOK_ENCODING_ERROR) {
385	6	return -1;
386	6	}
387	22.2k	raw += bom_len;
388	22.2k	raw_len -= bom_len;
389	22.2k	}
390
391	65.0k	_PyTok_Chunk decoded = {
392	65.0k	.data = raw,
393	65.0k	.len = raw_len,
394	65.0k	.ownership = _PYTOK_CHUNK_BORROWED,
395	65.0k	};
396	65.0k	const char *encoding = tok->reader->encoding;
397	65.0k	if (encoding != NULL && strcmp(encoding, "utf-8") != 0) {
398	5.03k	if (_PyTok_DecodeOnce(
399	5.03k	tok, &decoded, encoding, NULL) < 0) {
400	1.98k	return -1;
401	1.98k	}
402	5.03k	}
403
404	63.0k	int stored = store_prepared_source(
405	63.0k	tok, decoded.data, decoded.len, preserve_crlf, exec_input);
406	63.0k	_PyTok_ChunkClear(&decoded);
407	63.0k	if (stored < 0) {
408	0	return -1;
409	0	}
410	63.0k	if (!utf8_only &&
411	20.2k	(encoding == NULL \|\| strcmp(encoding, "utf-8") == 0) &&
412	17.2k	!_PyTokenizer_ensure_utf8(_PyTok_SourceData(&tok->source), tok, 1)) {
413	541	return -1;
414	541	}
415	62.5k	return 0;
416	63.0k	}
417
418		int
419		_PyTok_StartDecoder(struct tok_state tok, const char errors)
420	4	{
421	4	_PyTok_Reader *reader = tok->reader;
422	4	if (reader->encoding == NULL \|\| reader->decoder != NULL) {
423	0	return 0;
424	0	}
425	4	if (reader->kind == _PYTOK_READER_FILE &&
426	0	strcmp(reader->encoding, "utf-8") == 0) {
427	0	return 0;
428	0	}
429
430	4	PyObject *codec = _PyCodec_LookupTextEncoding(reader->encoding, NULL);
431	4	if (codec != NULL) {
432	4	reader->decoder = _PyCodecInfo_GetIncrementalDecoder(codec, errors);
433	4	Py_DECREF(codec);
434	4	}
435	4	if (reader->decoder == NULL) {
436	0	tok->done = PyErr_ExceptionMatches(PyExc_MemoryError)
437	0	? E_NOMEM : E_DECODE;
438	0	if (reader->kind == _PYTOK_READER_FILE) {
439	0	_PyTokenizer_raise_init_error(
440	0	tok->filename != NULL ? tok->filename : Py_None);
441	0	}
442	0	return -1;
443	0	}
444	4	return 0;
445	4	}
446
447		int
448		_PyTok_DecodeChunk(struct tok_state tok, _PyTok_Chunk chunk, int final)
449	8	{
450	8	_PyTok_Reader *reader = tok->reader;
451	8	if (reader->decoder == NULL) {
452	0	return 0;
453	0	}
454	8	int strip_bom = reader->kind == _PYTOK_READER_READLINE &&
455	8	chunk->len >= UTF16_BOM_LENGTH &&
456	4	(memcmp(chunk->data, UTF16_LE_BOM, UTF16_BOM_LENGTH) == 0 \|\|
457	4	memcmp(chunk->data, UTF16_BE_BOM, UTF16_BOM_LENGTH) == 0);
458	8	PyObject *input;
459	8	if (chunk->ownership == _PYTOK_CHUNK_PYOBJECT &&
460	8	PyBytes_Check(chunk->owner) &&
461	4	chunk->data == PyBytes_AS_STRING(chunk->owner)) {
462	4	input = Py_NewRef(chunk->owner);
463	4	}
464	4	else {
465	4	input = PyBytes_FromStringAndSize(chunk->data, chunk->len);
466	4	}
467	8	if (input == NULL) {
468	0	tok->done = E_NOMEM;
469	0	return -1;
470	0	}
471	8	PyObject *unicode = PyObject_CallMethodObjArgs(
472	8	reader->decoder, &_Py_ID(decode), input,
473	8	final ? Py_True : Py_False, NULL);
474	8	Py_DECREF(input);
475	8	if (unicode == NULL) {
476	0	tok->done = PyErr_ExceptionMatches(PyExc_MemoryError)
477	0	? E_NOMEM : E_DECODE;
478	0	if (reader->kind == _PYTOK_READER_FILE) {
479	0	_PyTokenizer_raise_init_error(
480	0	tok->filename != NULL ? tok->filename : Py_None);
481	0	}
482	0	return -1;
483	0	}
484	8	if (!PyUnicode_Check(unicode)) {
485	0	PyErr_Format(PyExc_TypeError,
486	0	"decoder should return a string result, not '%.200s'",
487	0	Py_TYPE(unicode)->tp_name);
488	0	Py_DECREF(unicode);
489	0	tok->done = E_DECODE;
490	0	return -1;
491	0	}
492	8	return chunk_set_unicode(tok, chunk, unicode, strip_bom);
493	8	}
494
495		int
496		_PyTok_DecoderHasBufferedInput(struct tok_state *tok)
497	4	{
498	4	if (tok->reader->decoder == NULL) {
499	0	return 0;
500	0	}
501	4	PyObject *state = PyObject_CallMethodNoArgs(
502	4	tok->reader->decoder, &_Py_ID(getstate));
503	4	if (state == NULL) {
504	0	tok->done = PyErr_ExceptionMatches(PyExc_MemoryError)
505	0	? E_NOMEM : E_DECODE;
506	0	return -1;
507	0	}
508	4	if (!PyTuple_Check(state) \|\| PyTuple_GET_SIZE(state) != 2 \|\|
509	4	!PyBytes_Check(PyTuple_GET_ITEM(state, 0)) \|\|
510	4	!PyLong_Check(PyTuple_GET_ITEM(state, 1))) {
511	0	Py_DECREF(state);
512	0	PyErr_SetString(PyExc_TypeError,
513	0	"incremental decoder getstate() must return (bytes, int)");
514	0	tok->done = E_DECODE;
515	0	return -1;
516	0	}
517	4	int pending = PyBytes_GET_SIZE(PyTuple_GET_ITEM(state, 0)) != 0;
518	4	Py_DECREF(state);
519	4	return pending;
520	4	}