/src/cpython/Parser/tokenizer/decoder.c

Source
#include "Python.h"
#include "pycore_codecs.h"
#include "pycore_global_strings.h"
#include "pycore_runtime.h"
#include "errcode.h"

#include "reader_internal.h"
#include "helpers.h"
#include "../lexer/state.h"

#define UNICODE_BOM 0xFEFF
#define UTF16_LE_BOM "\xff\xfe"
#define UTF16_BE_BOM "\xfe\xff"
#define UTF16_BOM_LENGTH ((Py_ssize_t)(sizeof(UTF16_LE_BOM) - 1))
#define UTF8_BOM "\xef\xbb\xbf"
#define UTF8_BOM_LENGTH ((Py_ssize_t)(sizeof(UTF8_BOM) - 1))

char *
_PyTok_CopyBytes(const char *data, Py_ssize_t len)
{
    if (len < 0 || len == PY_SSIZE_T_MAX) {
        PyErr_NoMemory();
        return NULL;
    }
    char *copy = PyMem_Malloc((size_t)len + 1);
    if (copy == NULL) {
        PyErr_NoMemory();
        return NULL;
    }
    memcpy(copy, data, (size_t)len);
    copy[len] = '\0';
    return copy;
}

static void
chunk_release_data(_PyTok_Chunk *chunk)
{
    switch (chunk->ownership) {
        case _PYTOK_CHUNK_BORROWED:
            break;
        case _PYTOK_CHUNK_PYMEM:
            PyMem_Free(chunk->data);
            break;
        case _PYTOK_CHUNK_PYOBJECT:
            Py_DECREF(chunk->owner);
            break;
    }
}

void
_PyTok_ChunkClear(_PyTok_Chunk *chunk)
{
    chunk_release_data(chunk);
    *chunk = (_PyTok_Chunk){0};
}

static int
chunk_set_unicode(struct tok_state *tok, _PyTok_Chunk *chunk,
                  PyObject *unicode, int strip_bom)
{
    Py_ssize_t utf8_len;
    const char *utf8 = PyUnicode_AsUTF8AndSize(unicode, &utf8_len);
    if (utf8 == NULL) {
        Py_DECREF(unicode);
        tok->done = PyErr_ExceptionMatches(PyExc_MemoryError)
            ? E_NOMEM : E_DECODE;
        return -1;
    }
    if (strip_bom && PyUnicode_GET_LENGTH(unicode) > 0 &&
            PyUnicode_ReadChar(unicode, 0) == UNICODE_BOM) {
        utf8 += UTF8_BOM_LENGTH;
        utf8_len -= UTF8_BOM_LENGTH;
    }
    chunk_release_data(chunk);
    chunk->owner = unicode;
    chunk->data = (char *)utf8;
    chunk->len = utf8_len;
    chunk->ownership = _PYTOK_CHUNK_PYOBJECT;
    return 0;
}

char *
_PyTok_NormalizeNewlines(const char *data, Py_ssize_t len,
                         Py_ssize_t *out_len)
{
    if (len < 0 || len == PY_SSIZE_T_MAX) {
        PyErr_NoMemory();
        return NULL;
    }
    char *result = PyMem_Malloc((size_t)len + 1);
    if (result == NULL) {
        PyErr_NoMemory();
        return NULL;
    }
    Py_ssize_t write = 0;
    for (Py_ssize_t read = 0; read < len; read++) {
        char c = data[read];
        if (c == '\r') {
            if (read + 1 < len && data[read + 1] == '\n') {
                read++;
            }
            c = '\n';
        }
        result[write++] = c;
    }
    result[write] = '\0';
    *out_len = write;
    return result;
}

int
_PyTok_SetEncoding(struct tok_state *tok, const char *encoding)
{
    char *copy = _PyTok_CopyBytes(encoding, strlen(encoding));
    if (copy == NULL) {
        tok->done = E_NOMEM;
        return -1;
    }
    PyMem_Free(tok->reader->encoding);
    tok->reader->encoding = copy;
    return 0;
}

static int
find_cookie(const char *line, Py_ssize_t len, char **encoding, int *scan_next)
{
    Py_ssize_t i = 0;
    *encoding = NULL;
    *scan_next = 1;
    for (; i < len; i++) {
        if (line[i] == '#') {
            break;
        }
        if (line[i] == '\n' || line[i] == '\r') {
            return 0;
        }
        if (line[i] != ' ' && line[i] != '\t' && line[i] != '\f') {
            *scan_next = 0;
            return 0;
        }
    }
    const Py_ssize_t marker_len = sizeof("coding") - 1;
    for (; i < len - marker_len; i++) {
        if (memcmp(line + i, "coding", marker_len) != 0) {
            continue;
        }
        const char *cursor = line + i + marker_len;
        if (*cursor != ':' && *cursor != '=') {
            continue;
        }
        do {
            cursor++;
        } while (cursor < line + len &&
                 (*cursor == ' ' || *cursor == '\t'));
        const char *start = cursor;
        while (cursor < line + len &&
                (Py_ISALNUM(*cursor) || *cursor == '-' ||
                 *cursor == '_' || *cursor == '.')) {
            cursor++;
        }
        if (cursor == start) {
            continue;
        }
        Py_ssize_t encoding_len = cursor - start;
        // Include the longest recognized prefix and its NUL terminator.
        char normalized[sizeof("iso-latin-1-")];
        const Py_ssize_t prefix_len = Py_MIN(
            encoding_len, (Py_ssize_t)sizeof(normalized) - 1);
        Py_ssize_t n;
        for (n = 0; n < prefix_len; n++) {
            normalized[n] = start[n] == '_' ? '-' : Py_TOLOWER(start[n]);
        }
        normalized[n] = '\0';
        const char *canonical = start;
        if (strcmp(normalized, "utf-8") == 0 ||
                strncmp(normalized, "utf-8-", sizeof("utf-8-") - 1) == 0) {
            canonical = "utf-8";
        }
        else if (strcmp(normalized, "latin-1") == 0 ||
                 strcmp(normalized, "iso-8859-1") == 0 ||
                 strcmp(normalized, "iso-latin-1") == 0 ||
                 strncmp(normalized, "latin-1-", sizeof("latin-1-") - 1) == 0 ||
                 strncmp(normalized, "iso-8859-1-", sizeof("iso-8859-1-") - 1) == 0 ||
                 strncmp(normalized, "iso-latin-1-", sizeof("iso-latin-1-") - 1) == 0) {
            canonical = "iso-8859-1";
        }
        if (canonical != start) {
            encoding_len = strlen(canonical);
        }
        *encoding = _PyTok_CopyBytes(canonical, encoding_len);
        if (*encoding == NULL) {
            return -1;
        }
        *scan_next = 0;
        return 0;
    }
    return 0;
}

_PyTok_EncodingResult
_PyTok_DetectEncoding(struct tok_state *tok, const _PyTok_Chunk *first,
                      const _PyTok_Chunk *second, int final,
                      Py_ssize_t *bom_len)
{
    int bom = first->len >= UTF8_BOM_LENGTH &&
        memcmp(first->data, UTF8_BOM, UTF8_BOM_LENGTH) == 0;
    *bom_len = bom ? UTF8_BOM_LENGTH : 0;

    char *cookie = NULL;
    int scan_next = 0;
    int cookie_line = 1;
    const char *first_data = first->data + *bom_len;
    Py_ssize_t first_len = first->len - *bom_len;
    if (find_cookie(first_data, first_len, &cookie, &scan_next) < 0) {
        return _PYTOK_ENCODING_ERROR;
    }
    if (cookie == NULL && scan_next && second != NULL) {
        if (find_cookie(second->data, second->len, &cookie, &scan_next) < 0) {
            return _PYTOK_ENCODING_ERROR;
        }
        cookie_line = 2;
    }
    else if (cookie == NULL && scan_next && !final) {
        return _PYTOK_ENCODING_NEED_SECOND_LINE;
    }

    if (cookie == NULL) {
        if (bom && _PyTok_SetEncoding(tok, "utf-8") < 0) {
            return _PYTOK_ENCODING_ERROR;
        }
        return _PYTOK_ENCODING_DONE;
    }
    if (bom && strcmp(cookie, "utf-8") != 0) {
        const _PyTok_Chunk *line = cookie_line == 2 ? second : first;
        const char *line_data = line->data + (cookie_line == 1 ? *bom_len : 0);
        Py_ssize_t line_len = line->len - (cookie_line == 1 ? *bom_len : 0);
        int end_col = (int)Py_MIN(line_len, INT_MAX);
        if (end_col > 0 && (line_data[end_col - 1] == '\n' ||
                            line_data[end_col - 1] == '\r')) {
            end_col--;
        }
        _PyTokenizer_syntaxerror_at(
            tok, line_data, 0, cookie_line, 0, end_col,
            "encoding problem: %s with BOM", cookie);
        PyMem_Free(cookie);
        return _PYTOK_ENCODING_ERROR;
    }
    PyMem_Free(tok->reader->encoding);
    tok->reader->encoding = cookie;
    return _PYTOK_ENCODING_DONE;
}

int
_PyTok_DecodeOnce(struct tok_state *tok, _PyTok_Chunk *chunk,
                  const char *encoding, const char *errors)
{
    PyObject *unicode = PyUnicode_Decode(
        chunk->data, chunk->len, encoding, errors);
    if (unicode == NULL) {
        tok->done = PyErr_ExceptionMatches(PyExc_MemoryError)
            ? E_NOMEM : E_DECODE;
        return -1;
    }
    return chunk_set_unicode(tok, chunk, unicode, 0);
}

static Py_ssize_t
raw_line_length(const char *data, Py_ssize_t len)
{
    for (Py_ssize_t i = 0; i < len; i++) {
        if (data[i] == '\n') {
            return i + 1;
        }
        if (data[i] == '\r') {
            return i + 1 < len && data[i + 1] == '\n' ? i + 2 : i + 1;
        }
    }
    return len;
}

static int
store_prepared_source(struct tok_state *tok, const char *data, Py_ssize_t len,
                      int preserve_crlf, int add_final_newline)
{
    Py_ssize_t pos = 0;
    while (pos < len) {
        Py_ssize_t raw_line_len;
        if (preserve_crlf) {
            const char *newline = memchr(data + pos, '\n', len - pos);
            raw_line_len = newline == NULL
                ? len - pos : newline - data - pos + 1;
        }
        else {
            raw_line_len = raw_line_length(data + pos, len - pos);
        }
        int terminated = preserve_crlf
            ? data[pos + raw_line_len - 1] == '\n'
            : data[pos + raw_line_len - 1] == '\n' ||
              data[pos + raw_line_len - 1] == '\r';
        int add_newline = add_final_newline &&
            pos + raw_line_len == len && !terminated;
        const char *line = data + pos;
        // raw_line_length stops at the first CR or LF, so any CR is in
        // the line terminator.
        Py_ssize_t terminator_len = 0;
        if (!preserve_crlf) {
            if (line[raw_line_len - 1] == '\r') {
                terminator_len = 1;
            }
            else if (raw_line_len > 1 && line[raw_line_len - 2] == '\r') {
                terminator_len = 2;
            }
        }
        _PyTok_Off appended;
        if (terminator_len || add_newline) {
            appended = _PyTok_SourceAppendLineWithNewline(
                &tok->source, line, raw_line_len - terminator_len);
        }
        else {
            appended = _PyTok_SourceAppendLine(&tok->source, line, raw_line_len);
        }
        if (appended < 0) {
            tok->done = PyErr_ExceptionMatches(PyExc_MemoryError)
                ? E_NOMEM : E_ERROR;
            return -1;
        }
        tok->reader->prepared_final_newline = add_newline;
        pos += raw_line_len;
    }
    // Partial input includes the newline submitted by the next Enter key.
    // A final CR pairs with that newline instead of starting a blank line.
    if (add_final_newline == _PyTokenizer_APPEND_NEWLINE &&
        !tok->reader->prepared_final_newline &&
        (len == 0 || data[len - 1] != '\r'))
    {
        if (_PyTok_SourceAppendLine(&tok->source, "\n", 1) < 0) {
            return -1;
        }
        tok->reader->prepared_final_newline = _PyTokenizer_APPEND_NEWLINE;
    }
    return 0;
}

int
_PyTok_PrepareString(struct tok_state *tok, const char *input, int utf8_only,
                     int exec_input, int preserve_crlf)
{
    Py_ssize_t raw_len = strlen(input);
    char *raw = (char *)input;

    if (utf8_only) {
        if (_PyTok_SetEncoding(tok, "utf-8") < 0) {
            return -1;
        }
    }
    else {
        Py_ssize_t first_original_len = raw_line_length(raw, raw_len);
        _PyTok_Chunk first = {
            .data = raw,
            .len = first_original_len,
            .ownership = _PYTOK_CHUNK_BORROWED,
        };
        _PyTok_Chunk second = {0};
        int have_second = first_original_len < raw_len;
        if (have_second) {
            second.data = raw + first_original_len;
            second.len = raw_line_length(second.data,
                                         raw_len - first_original_len);
        }
        Py_ssize_t bom_len;
        _PyTok_EncodingResult detection = _PyTok_DetectEncoding(
            tok, &first, have_second ? &second : NULL, 1, &bom_len);
        if (detection == _PYTOK_ENCODING_ERROR) {
            return -1;
        }
        raw += bom_len;
        raw_len -= bom_len;
    }

    _PyTok_Chunk decoded = {
        .data = raw,
        .len = raw_len,
        .ownership = _PYTOK_CHUNK_BORROWED,
    };
    const char *encoding = tok->reader->encoding;
    if (encoding != NULL && strcmp(encoding, "utf-8") != 0) {
        if (_PyTok_DecodeOnce(
                tok, &decoded, encoding, NULL) < 0) {
            return -1;
        }
    }

    int stored = store_prepared_source(
        tok, decoded.data, decoded.len, preserve_crlf, exec_input);
    _PyTok_ChunkClear(&decoded);
    if (stored < 0) {
        return -1;
    }
    if (!utf8_only &&
            (encoding == NULL || strcmp(encoding, "utf-8") == 0) &&
            !_PyTokenizer_ensure_utf8(_PyTok_SourceData(&tok->source), tok, 1)) {
        return -1;
    }
    return 0;
}

int
_PyTok_StartDecoder(struct tok_state *tok, const char *errors)
{
    _PyTok_Reader *reader = tok->reader;
    if (reader->encoding == NULL || reader->decoder != NULL) {
        return 0;
    }
    if (reader->kind == _PYTOK_READER_FILE &&
            strcmp(reader->encoding, "utf-8") == 0) {
        return 0;
    }

    PyObject *codec = _PyCodec_LookupTextEncoding(reader->encoding, NULL);
    if (codec != NULL) {
        reader->decoder = _PyCodecInfo_GetIncrementalDecoder(codec, errors);
        Py_DECREF(codec);
    }
    if (reader->decoder == NULL) {
        tok->done = PyErr_ExceptionMatches(PyExc_MemoryError)
            ? E_NOMEM : E_DECODE;
        if (reader->kind == _PYTOK_READER_FILE) {
            _PyTokenizer_raise_init_error(
                tok->filename != NULL ? tok->filename : Py_None);
        }
        return -1;
    }
    return 0;
}

int
_PyTok_DecodeChunk(struct tok_state *tok, _PyTok_Chunk *chunk, int final)
{
    _PyTok_Reader *reader = tok->reader;
    if (reader->decoder == NULL) {
        return 0;
    }
    int strip_bom = reader->kind == _PYTOK_READER_READLINE &&
        chunk->len >= UTF16_BOM_LENGTH &&
        (memcmp(chunk->data, UTF16_LE_BOM, UTF16_BOM_LENGTH) == 0 ||
         memcmp(chunk->data, UTF16_BE_BOM, UTF16_BOM_LENGTH) == 0);
    PyObject *input;
    if (chunk->ownership == _PYTOK_CHUNK_PYOBJECT &&
            PyBytes_Check(chunk->owner) &&
            chunk->data == PyBytes_AS_STRING(chunk->owner)) {
        input = Py_NewRef(chunk->owner);
    }
    else {
        input = PyBytes_FromStringAndSize(chunk->data, chunk->len);
    }
    if (input == NULL) {
        tok->done = E_NOMEM;
        return -1;
    }
    PyObject *unicode = PyObject_CallMethodObjArgs(
        reader->decoder, &_Py_ID(decode), input,
        final ? Py_True : Py_False, NULL);
    Py_DECREF(input);
    if (unicode == NULL) {
        tok->done = PyErr_ExceptionMatches(PyExc_MemoryError)
            ? E_NOMEM : E_DECODE;
        if (reader->kind == _PYTOK_READER_FILE) {
            _PyTokenizer_raise_init_error(
                tok->filename != NULL ? tok->filename : Py_None);
        }
        return -1;
    }
    if (!PyUnicode_Check(unicode)) {
        PyErr_Format(PyExc_TypeError,
                     "decoder should return a string result, not '%.200s'",
                     Py_TYPE(unicode)->tp_name);
        Py_DECREF(unicode);
        tok->done = E_DECODE;
        return -1;
    }
    return chunk_set_unicode(tok, chunk, unicode, strip_bom);
}

int
_PyTok_DecoderHasBufferedInput(struct tok_state *tok)
{
    if (tok->reader->decoder == NULL) {
        return 0;
    }
    PyObject *state = PyObject_CallMethodNoArgs(
        tok->reader->decoder, &_Py_ID(getstate));
    if (state == NULL) {
        tok->done = PyErr_ExceptionMatches(PyExc_MemoryError)
            ? E_NOMEM : E_DECODE;
        return -1;
    }
    if (!PyTuple_Check(state) || PyTuple_GET_SIZE(state) != 2 ||
            !PyBytes_Check(PyTuple_GET_ITEM(state, 0)) ||
            !PyLong_Check(PyTuple_GET_ITEM(state, 1))) {
        Py_DECREF(state);
        PyErr_SetString(PyExc_TypeError,
                        "incremental decoder getstate() must return (bytes, int)");
        tok->done = E_DECODE;
        return -1;
    }
    int pending = PyBytes_GET_SIZE(PyTuple_GET_ITEM(state, 0)) != 0;
    Py_DECREF(state);
    return pending;
}

Coverage Report

Created: 2026-10-31 06:21

Line	Count	Source
1		#include "Python.h"
2		#include "pycore_codecs.h"
3		#include "pycore_global_strings.h"
4		#include "pycore_runtime.h"
5		#include "errcode.h"
6
7		#include "reader_internal.h"
8		#include "helpers.h"
9		#include "../lexer/state.h"
10
11	0	#define UNICODE_BOM 0xFEFF
12	0	#define UTF16_LE_BOM "\xff\xfe"
13	0	#define UTF16_BE_BOM "\xfe\xff"
14	0	#define UTF16_BOM_LENGTH ((Py_ssize_t)(sizeof(UTF16_LE_BOM) - 1))
15	67.2k	#define UTF8_BOM "\xef\xbb\xbf"
16	68.0k	#define UTF8_BOM_LENGTH ((Py_ssize_t)(sizeof(UTF8_BOM) - 1))
17
18		char *
19		_PyTok_CopyBytes(const char *data, Py_ssize_t len)
20	58.2k	{
21	58.2k	if (len < 0 \|\| len == PY_SSIZE_T_MAX) {
22	0	PyErr_NoMemory();
23	0	return NULL;
24	0	}
25	58.2k	char *copy = PyMem_Malloc((size_t)len + 1);
26	58.2k	if (copy == NULL) {
27	0	PyErr_NoMemory();
28	0	return NULL;
29	0	}
30	58.2k	memcpy(copy, data, (size_t)len);
31	58.2k	copy[len] = '\0';
32	58.2k	return copy;
33	58.2k	}
34
35		static void
36		chunk_release_data(_PyTok_Chunk *chunk)
37	502k	{
38	502k	switch (chunk->ownership) {
39	498k	case _PYTOK_CHUNK_BORROWED:
40	498k	break;
41	0	case _PYTOK_CHUNK_PYMEM:
42	0	PyMem_Free(chunk->data);
43	0	break;
44	3.43k	case _PYTOK_CHUNK_PYOBJECT:
45	3.43k	Py_DECREF(chunk->owner);
46	3.43k	break;
47	502k	}
48	502k	}
49
50		void
51		_PyTok_ChunkClear(_PyTok_Chunk *chunk)
52	499k	{
53	499k	chunk_release_data(chunk);
54	499k	*chunk = (_PyTok_Chunk){0};
55	499k	}
56
57		static int
58		chunk_set_unicode(struct tok_state tok, _PyTok_Chunk chunk,
59		PyObject *unicode, int strip_bom)
60	3.16k	{
61	3.16k	Py_ssize_t utf8_len;
62	3.16k	const char *utf8 = PyUnicode_AsUTF8AndSize(unicode, &utf8_len);
63	3.16k	if (utf8 == NULL) {
64	63	Py_DECREF(unicode);
65	63	tok->done = PyErr_ExceptionMatches(PyExc_MemoryError)
66	63	? E_NOMEM : E_DECODE;
67	63	return -1;
68	63	}
69	3.09k	if (strip_bom && PyUnicode_GET_LENGTH(unicode) > 0 &&
70	0	PyUnicode_ReadChar(unicode, 0) == UNICODE_BOM) {
71	0	utf8 += UTF8_BOM_LENGTH;
72	0	utf8_len -= UTF8_BOM_LENGTH;
73	0	}
74	3.09k	chunk_release_data(chunk);
75	3.09k	chunk->owner = unicode;
76	3.09k	chunk->data = (char *)utf8;
77	3.09k	chunk->len = utf8_len;
78	3.09k	chunk->ownership = _PYTOK_CHUNK_PYOBJECT;
79	3.09k	return 0;
80	3.16k	}
81
82		char *
83		_PyTok_NormalizeNewlines(const char *data, Py_ssize_t len,
84		Py_ssize_t *out_len)
85	0	{
86	0	if (len < 0 \|\| len == PY_SSIZE_T_MAX) {
87	0	PyErr_NoMemory();
88	0	return NULL;
89	0	}
90	0	char *result = PyMem_Malloc((size_t)len + 1);
91	0	if (result == NULL) {
92	0	PyErr_NoMemory();
93	0	return NULL;
94	0	}
95	0	Py_ssize_t write = 0;
96	0	for (Py_ssize_t read = 0; read < len; read++) {
97	0	char c = data[read];
98	0	if (c == '\r') {
99	0	if (read + 1 < len && data[read + 1] == '\n') {
100	0	read++;
101	0	}
102	0	c = '\n';
103	0	}
104	0	result[write++] = c;
105	0	}
106	0	result[write] = '\0';
107	0	*out_len = write;
108	0	return result;
109	0	}
110
111		int
112		_PyTok_SetEncoding(struct tok_state tok, const char encoding)
113	53.0k	{
114	53.0k	char *copy = _PyTok_CopyBytes(encoding, strlen(encoding));
115	53.0k	if (copy == NULL) {
116	0	tok->done = E_NOMEM;
117	0	return -1;
118	0	}
119	53.0k	PyMem_Free(tok->reader->encoding);
120	53.0k	tok->reader->encoding = copy;
121	53.0k	return 0;
122	53.0k	}
123
124		static int
125		find_cookie(const char line, Py_ssize_t len, char encoding, int scan_next)
126	23.5k	{
127	23.5k	Py_ssize_t i = 0;
128	23.5k	*encoding = NULL;
129	23.5k	*scan_next = 1;
130	24.4k	for (; i < len; i++) {
131	24.4k	if (line[i] == '#') {
132	5.59k	break;
133	5.59k	}
134	18.8k	if (line[i] == '\n' \|\| line[i] == '\r') {
135	380	return 0;
136	380	}
137	18.4k	if (line[i] != ' ' && line[i] != '\t' && line[i] != '\f') {
138	17.5k	*scan_next = 0;
139	17.5k	return 0;
140	17.5k	}
141	18.4k	}
142	5.61k	const Py_ssize_t marker_len = sizeof("coding") - 1;
143	277k	for (; i < len - marker_len; i++) {
144	277k	if (memcmp(line + i, "coding", marker_len) != 0) {
145	271k	continue;
146	271k	}
147	5.88k	const char *cursor = line + i + marker_len;
148	5.88k	if (cursor != ':' && cursor != '=') {
149	266	continue;
150	266	}
151	6.48k	do {
152	6.48k	cursor++;
153	6.48k	} while (cursor < line + len &&
154	6.46k	(cursor == ' ' \|\| cursor == '\t'));
155	5.61k	const char *start = cursor;
156	43.2k	while (cursor < line + len &&
157	41.7k	(Py_ISALNUM(cursor) \|\| cursor == '-' \|\|
158	37.6k	cursor == '_' \|\| cursor == '.')) {
159	37.6k	cursor++;
160	37.6k	}
161	5.61k	if (cursor == start) {
162	415	continue;
163	415	}
164	5.20k	Py_ssize_t encoding_len = cursor - start;
165		// Include the longest recognized prefix and its NUL terminator.
166	5.20k	char normalized[sizeof("iso-latin-1-")];
167	5.20k	const Py_ssize_t prefix_len = Py_MIN(
168	5.20k	encoding_len, (Py_ssize_t)sizeof(normalized) - 1);
169	5.20k	Py_ssize_t n;
170	34.8k	for (n = 0; n < prefix_len; n++) {
171	29.6k	normalized[n] = start[n] == '_' ? '-' : Py_TOLOWER(start[n]);
172	29.6k	}
173	5.20k	normalized[n] = '\0';
174	5.20k	const char *canonical = start;
175	5.20k	if (strcmp(normalized, "utf-8") == 0 \|\|
176	5.20k	strncmp(normalized, "utf-8-", sizeof("utf-8-") - 1) == 0) {
177	3	canonical = "utf-8";
178	3	}
179	5.20k	else if (strcmp(normalized, "latin-1") == 0 \|\|
180	5.18k	strcmp(normalized, "iso-8859-1") == 0 \|\|
181	5.18k	strcmp(normalized, "iso-latin-1") == 0 \|\|
182	5.18k	strncmp(normalized, "latin-1-", sizeof("latin-1-") - 1) == 0 \|\|
183	5.18k	strncmp(normalized, "iso-8859-1-", sizeof("iso-8859-1-") - 1) == 0 \|\|
184	5.18k	strncmp(normalized, "iso-latin-1-", sizeof("iso-latin-1-") - 1) == 0) {
185	18	canonical = "iso-8859-1";
186	18	}
187	5.20k	if (canonical != start) {
188	21	encoding_len = strlen(canonical);
189	21	}
190	5.20k	*encoding = _PyTok_CopyBytes(canonical, encoding_len);
191	5.20k	if (*encoding == NULL) {
192	0	return -1;
193	0	}
194	5.20k	*scan_next = 0;
195	5.20k	return 0;
196	5.20k	}
197	416	return 0;
198	5.61k	}
199
200		_PyTok_EncodingResult
201		_PyTok_DetectEncoding(struct tok_state tok, const _PyTok_Chunk first,
202		const _PyTok_Chunk *second, int final,
203		Py_ssize_t *bom_len)
204	22.9k	{
205	22.9k	int bom = first->len >= UTF8_BOM_LENGTH &&
206	22.1k	memcmp(first->data, UTF8_BOM, UTF8_BOM_LENGTH) == 0;
207	22.9k	*bom_len = bom ? UTF8_BOM_LENGTH : 0;
208
209	22.9k	char *cookie = NULL;
210	22.9k	int scan_next = 0;
211	22.9k	int cookie_line = 1;
212	22.9k	const char first_data = first->data + bom_len;
213	22.9k	Py_ssize_t first_len = first->len - *bom_len;
214	22.9k	if (find_cookie(first_data, first_len, &cookie, &scan_next) < 0) {
215	0	return _PYTOK_ENCODING_ERROR;
216	0	}
217	22.9k	if (cookie == NULL && scan_next && second != NULL) {
218	569	if (find_cookie(second->data, second->len, &cookie, &scan_next) < 0) {
219	0	return _PYTOK_ENCODING_ERROR;
220	0	}
221	569	cookie_line = 2;
222	569	}
223	22.4k	else if (cookie == NULL && scan_next && !final) {
224	0	return _PYTOK_ENCODING_NEED_SECOND_LINE;
225	0	}
226
227	22.9k	if (cookie == NULL) {
228	17.7k	if (bom && _PyTok_SetEncoding(tok, "utf-8") < 0) {
229	0	return _PYTOK_ENCODING_ERROR;
230	0	}
231	17.7k	return _PYTOK_ENCODING_DONE;
232	17.7k	}
233	5.20k	if (bom && strcmp(cookie, "utf-8") != 0) {
234	5	const _PyTok_Chunk *line = cookie_line == 2 ? second : first;
235	5	const char line_data = line->data + (cookie_line == 1 ? bom_len : 0);
236	5	Py_ssize_t line_len = line->len - (cookie_line == 1 ? *bom_len : 0);
237	5	int end_col = (int)Py_MIN(line_len, INT_MAX);
238	5	if (end_col > 0 && (line_data[end_col - 1] == '\n' \|\|
239	4	line_data[end_col - 1] == '\r')) {
240	3	end_col--;
241	3	}
242	5	_PyTokenizer_syntaxerror_at(
243	5	tok, line_data, 0, cookie_line, 0, end_col,
244	5	"encoding problem: %s with BOM", cookie);
245	5	PyMem_Free(cookie);
246	5	return _PYTOK_ENCODING_ERROR;
247	5	}
248	5.19k	PyMem_Free(tok->reader->encoding);
249	5.19k	tok->reader->encoding = cookie;
250	5.19k	return _PYTOK_ENCODING_DONE;
251	5.20k	}
252
253		int
254		_PyTok_DecodeOnce(struct tok_state tok, _PyTok_Chunk chunk,
255		const char encoding, const char errors)
256	5.19k	{
257	5.19k	PyObject *unicode = PyUnicode_Decode(
258	5.19k	chunk->data, chunk->len, encoding, errors);
259	5.19k	if (unicode == NULL) {
260	2.03k	tok->done = PyErr_ExceptionMatches(PyExc_MemoryError)
261	2.03k	? E_NOMEM : E_DECODE;
262	2.03k	return -1;
263	2.03k	}
264	3.16k	return chunk_set_unicode(tok, chunk, unicode, 0);
265	5.19k	}
266
267		static Py_ssize_t
268		raw_line_length(const char *data, Py_ssize_t len)
269	303k	{
270	9.26M	for (Py_ssize_t i = 0; i < len; i++) {
271	9.17M	if (data[i] == '\n') {
272	193k	return i + 1;
273	193k	}
274	8.97M	if (data[i] == '\r') {
275	16.6k	return i + 1 < len && data[i + 1] == '\n' ? i + 2 : i + 1;
276	16.6k	}
277	8.97M	}
278	92.8k	return len;
279	303k	}
280
281		static int
282		store_prepared_source(struct tok_state tok, const char data, Py_ssize_t len,
283		int preserve_crlf, int add_final_newline)
284	73.9k	{
285	73.9k	Py_ssize_t pos = 0;
286	348k	while (pos < len) {
287	274k	Py_ssize_t raw_line_len;
288	274k	if (preserve_crlf) {
289	0	const char *newline = memchr(data + pos, '\n', len - pos);
290	0	raw_line_len = newline == NULL
291	0	? len - pos : newline - data - pos + 1;
292	0	}
293	274k	else {
294	274k	raw_line_len = raw_line_length(data + pos, len - pos);
295	274k	}
296	274k	int terminated = preserve_crlf
297	274k	? data[pos + raw_line_len - 1] == '\n'
298	274k	: data[pos + raw_line_len - 1] == '\n' \|\|
299	87.9k	data[pos + raw_line_len - 1] == '\r';
300	274k	int add_newline = add_final_newline &&
301	274k	pos + raw_line_len == len && !terminated;
302	274k	const char *line = data + pos;
303		// raw_line_length stops at the first CR or LF, so any CR is in
304		// the line terminator.
305	274k	Py_ssize_t terminator_len = 0;
306	274k	if (!preserve_crlf) {
307	274k	if (line[raw_line_len - 1] == '\r') {
308	14.8k	terminator_len = 1;
309	14.8k	}
310	259k	else if (raw_line_len > 1 && line[raw_line_len - 2] == '\r') {
311	713	terminator_len = 2;
312	713	}
313	274k	}
314	274k	_PyTok_Off appended;
315	274k	if (terminator_len \|\| add_newline) {
316	88.4k	appended = _PyTok_SourceAppendLineWithNewline(
317	88.4k	&tok->source, line, raw_line_len - terminator_len);
318	88.4k	}
319	186k	else {
320	186k	appended = _PyTok_SourceAppendLine(&tok->source, line, raw_line_len);
321	186k	}
322	274k	if (appended < 0) {
323	0	tok->done = PyErr_ExceptionMatches(PyExc_MemoryError)
324	0	? E_NOMEM : E_ERROR;
325	0	return -1;
326	0	}
327	274k	tok->reader->prepared_final_newline = add_newline;
328	274k	pos += raw_line_len;
329	274k	}
330		// Partial input includes the newline submitted by the next Enter key.
331		// A final CR pairs with that newline instead of starting a blank line.
332	73.9k	if (add_final_newline == _PyTokenizer_APPEND_NEWLINE &&
333	0	!tok->reader->prepared_final_newline &&
334	0	(len == 0 \|\| data[len - 1] != '\r'))
335	0	{
336	0	if (_PyTok_SourceAppendLine(&tok->source, "\n", 1) < 0) {
337	0	return -1;
338	0	}
339	0	tok->reader->prepared_final_newline = _PyTokenizer_APPEND_NEWLINE;
340	0	}
341	73.9k	return 0;
342	73.9k	}
343
344		int
345		_PyTok_PrepareString(struct tok_state tok, const char input, int utf8_only,
346		int exec_input, int preserve_crlf)
347	76.0k	{
348	76.0k	Py_ssize_t raw_len = strlen(input);
349	76.0k	char raw = (char )input;
350
351	76.0k	if (utf8_only) {
352	53.0k	if (_PyTok_SetEncoding(tok, "utf-8") < 0) {
353	0	return -1;
354	0	}
355	53.0k	}
356	22.9k	else {
357	22.9k	Py_ssize_t first_original_len = raw_line_length(raw, raw_len);
358	22.9k	_PyTok_Chunk first = {
359	22.9k	.data = raw,
360	22.9k	.len = first_original_len,
361	22.9k	.ownership = _PYTOK_CHUNK_BORROWED,
362	22.9k	};
363	22.9k	_PyTok_Chunk second = {0};
364	22.9k	int have_second = first_original_len < raw_len;
365	22.9k	if (have_second) {
366	5.70k	second.data = raw + first_original_len;
367	5.70k	second.len = raw_line_length(second.data,
368	5.70k	raw_len - first_original_len);
369	5.70k	}
370	22.9k	Py_ssize_t bom_len;
371	22.9k	_PyTok_EncodingResult detection = _PyTok_DetectEncoding(
372	22.9k	tok, &first, have_second ? &second : NULL, 1, &bom_len);
373	22.9k	if (detection == _PYTOK_ENCODING_ERROR) {
374	5	return -1;
375	5	}
376	22.9k	raw += bom_len;
377	22.9k	raw_len -= bom_len;
378	22.9k	}
379
380	76.0k	_PyTok_Chunk decoded = {
381	76.0k	.data = raw,
382	76.0k	.len = raw_len,
383	76.0k	.ownership = _PYTOK_CHUNK_BORROWED,
384	76.0k	};
385	76.0k	const char *encoding = tok->reader->encoding;
386	76.0k	if (encoding != NULL && strcmp(encoding, "utf-8") != 0) {
387	5.19k	if (_PyTok_DecodeOnce(
388	5.19k	tok, &decoded, encoding, NULL) < 0) {
389	2.09k	return -1;
390	2.09k	}
391	5.19k	}
392
393	73.9k	int stored = store_prepared_source(
394	73.9k	tok, decoded.data, decoded.len, preserve_crlf, exec_input);
395	73.9k	_PyTok_ChunkClear(&decoded);
396	73.9k	if (stored < 0) {
397	0	return -1;
398	0	}
399	73.9k	if (!utf8_only &&
400	20.8k	(encoding == NULL \|\| strcmp(encoding, "utf-8") == 0) &&
401	17.7k	!_PyTokenizer_ensure_utf8(_PyTok_SourceData(&tok->source), tok, 1)) {
402	558	return -1;
403	558	}
404	73.3k	return 0;
405	73.9k	}
406
407		int
408		_PyTok_StartDecoder(struct tok_state tok, const char errors)
409	0	{
410	0	_PyTok_Reader *reader = tok->reader;
411	0	if (reader->encoding == NULL \|\| reader->decoder != NULL) {
412	0	return 0;
413	0	}
414	0	if (reader->kind == _PYTOK_READER_FILE &&
415	0	strcmp(reader->encoding, "utf-8") == 0) {
416	0	return 0;
417	0	}
418
419	0	PyObject *codec = _PyCodec_LookupTextEncoding(reader->encoding, NULL);
420	0	if (codec != NULL) {
421	0	reader->decoder = _PyCodecInfo_GetIncrementalDecoder(codec, errors);
422	0	Py_DECREF(codec);
423	0	}
424	0	if (reader->decoder == NULL) {
425	0	tok->done = PyErr_ExceptionMatches(PyExc_MemoryError)
426	0	? E_NOMEM : E_DECODE;
427	0	if (reader->kind == _PYTOK_READER_FILE) {
428	0	_PyTokenizer_raise_init_error(
429	0	tok->filename != NULL ? tok->filename : Py_None);
430	0	}
431	0	return -1;
432	0	}
433	0	return 0;
434	0	}
435
436		int
437		_PyTok_DecodeChunk(struct tok_state tok, _PyTok_Chunk chunk, int final)
438	0	{
439	0	_PyTok_Reader *reader = tok->reader;
440	0	if (reader->decoder == NULL) {
441	0	return 0;
442	0	}
443	0	int strip_bom = reader->kind == _PYTOK_READER_READLINE &&
444	0	chunk->len >= UTF16_BOM_LENGTH &&
445	0	(memcmp(chunk->data, UTF16_LE_BOM, UTF16_BOM_LENGTH) == 0 \|\|
446	0	memcmp(chunk->data, UTF16_BE_BOM, UTF16_BOM_LENGTH) == 0);
447	0	PyObject *input;
448	0	if (chunk->ownership == _PYTOK_CHUNK_PYOBJECT &&
449	0	PyBytes_Check(chunk->owner) &&
450	0	chunk->data == PyBytes_AS_STRING(chunk->owner)) {
451	0	input = Py_NewRef(chunk->owner);
452	0	}
453	0	else {
454	0	input = PyBytes_FromStringAndSize(chunk->data, chunk->len);
455	0	}
456	0	if (input == NULL) {
457	0	tok->done = E_NOMEM;
458	0	return -1;
459	0	}
460	0	PyObject *unicode = PyObject_CallMethodObjArgs(
461	0	reader->decoder, &_Py_ID(decode), input,
462	0	final ? Py_True : Py_False, NULL);
463	0	Py_DECREF(input);
464	0	if (unicode == NULL) {
465	0	tok->done = PyErr_ExceptionMatches(PyExc_MemoryError)
466	0	? E_NOMEM : E_DECODE;
467	0	if (reader->kind == _PYTOK_READER_FILE) {
468	0	_PyTokenizer_raise_init_error(
469	0	tok->filename != NULL ? tok->filename : Py_None);
470	0	}
471	0	return -1;
472	0	}
473	0	if (!PyUnicode_Check(unicode)) {
474	0	PyErr_Format(PyExc_TypeError,
475	0	"decoder should return a string result, not '%.200s'",
476	0	Py_TYPE(unicode)->tp_name);
477	0	Py_DECREF(unicode);
478	0	tok->done = E_DECODE;
479	0	return -1;
480	0	}
481	0	return chunk_set_unicode(tok, chunk, unicode, strip_bom);
482	0	}
483
484		int
485		_PyTok_DecoderHasBufferedInput(struct tok_state *tok)
486	0	{
487	0	if (tok->reader->decoder == NULL) {
488	0	return 0;
489	0	}
490	0	PyObject *state = PyObject_CallMethodNoArgs(
491	0	tok->reader->decoder, &_Py_ID(getstate));
492	0	if (state == NULL) {
493	0	tok->done = PyErr_ExceptionMatches(PyExc_MemoryError)
494	0	? E_NOMEM : E_DECODE;
495	0	return -1;
496	0	}
497	0	if (!PyTuple_Check(state) \|\| PyTuple_GET_SIZE(state) != 2 \|\|
498	0	!PyBytes_Check(PyTuple_GET_ITEM(state, 0)) \|\|
499	0	!PyLong_Check(PyTuple_GET_ITEM(state, 1))) {
500	0	Py_DECREF(state);
501	0	PyErr_SetString(PyExc_TypeError,
502	0	"incremental decoder getstate() must return (bytes, int)");
503	0	tok->done = E_DECODE;
504	0	return -1;
505	0	}
506	0	int pending = PyBytes_GET_SIZE(PyTuple_GET_ITEM(state, 0)) != 0;
507	0	Py_DECREF(state);
508	0	return pending;
509	0	}