/src/cpython/Python/Python-tokenize.c

Source
#include "Python.h"
#include "errcode.h"
#include "internal/pycore_critical_section.h"   // Py_BEGIN_CRITICAL_SECTION
#include "../Parser/tokenizer/tokenizer.h"
#include "../Parser/pegen.h"                    // _PyPegen_byte_offset_to_character_offset()

static struct PyModuleDef _tokenizemodule;

typedef struct {
    PyTypeObject *TokenizerIter;
} tokenize_state;

static tokenize_state *
get_tokenize_state(PyObject *module) {
    return (tokenize_state *)PyModule_GetState(module);
}

#define _tokenize_get_state_by_type(type) \
    get_tokenize_state(PyType_GetModuleByDef(type, &_tokenizemodule))

#include "pycore_runtime.h"

/*[clinic input]
module _tokenizer
class _tokenizer.tokenizeriter "tokenizeriterobject *" "_tokenize_get_state_by_type(type)->TokenizerIter"
[clinic start generated code]*/
/*[clinic end generated code: output=da39a3ee5e6b4b0d input=96d98ee2fef7a8bc]*/

typedef struct
{
    PyObject_HEAD struct tok_state *tok;
    int done;
    int extra_tokens;

    /* Needed to cache line for performance */
    PyObject *last_line;
    Py_ssize_t last_lineno;
    Py_ssize_t byte_col_offset_diff;
} tokenizeriterobject;

#include "clinic/Python-tokenize.c.h"

/*[clinic input]
@classmethod
_tokenizer.tokenizeriter.__new__ as tokenizeriter_new

    readline: object
    /
    *
    extra_tokens: bool
    encoding: str(c_default="NULL") = 'utf-8'
[clinic start generated code]*/

static PyObject *
tokenizeriter_new_impl(PyTypeObject *type, PyObject *readline,
                       int extra_tokens, const char *encoding)
/*[clinic end generated code: output=7501a1211683ce16 input=f7dddf8a613ae8bd]*/
{
    struct tok_state *tok = _PyTokenizer_FromReadline(readline, encoding);
    if (tok == NULL) {
        return NULL;
    }
    _Py_DECLARE_STR(anon_string, "<string>");
    _PyTokenizer_SetContext(tok, &_Py_STR(anon_string), NULL);
    tokenizeriterobject *self = (tokenizeriterobject *)type->tp_alloc(type, 0);
    if (self == NULL) {
        _PyTokenizer_Free(tok);
        return NULL;
    }
    self->tok = tok;
    _PyTokenizer_SetOptions(self->tok, extra_tokens, 0);
    self->extra_tokens = extra_tokens;
    self->done = 0;

    self->last_line = NULL;
    self->byte_col_offset_diff = 0;
    self->last_lineno = 0;

    return (PyObject *)self;
}

static void
_tokenizer_error(tokenizeriterobject *it)
{
    _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(it);
    assert(!PyErr_Occurred());

    const char *msg = NULL;
    PyObject* errtype = PyExc_SyntaxError;
    struct tok_state *tok = it->tok;
    _PyTokenizer_Info info = _PyTokenizer_GetInfo(tok);
    switch (info.status) {
        case E_TOKEN:
            msg = "invalid token";
            break;
        case E_EOF:
            PyErr_SetString(PyExc_SyntaxError, "unexpected EOF in multi-line statement");
            PyErr_SyntaxLocationObject(
                info.filename, info.location.lineno,
                (int)Py_MAX(0, info.input_span.end - info.input_span.start));
            return;
        case E_DEDENT:
            msg = "unindent does not match any outer indentation level";
            errtype = PyExc_IndentationError;
            break;
        case E_INTR:
            PyErr_SetNone(PyExc_KeyboardInterrupt);
            return;
        case E_NOMEM:
            PyErr_NoMemory();
            return;
        case E_TABSPACE:
            errtype = PyExc_TabError;
            msg = "inconsistent use of tabs and spaces in indentation";
            break;
        case E_TOODEEP:
            errtype = PyExc_IndentationError;
            msg = "too many levels of indentation";
            break;
        case E_LINECONT: {
            msg = "unexpected character after line continuation character";
            break;
        }
        default:
            msg = "unknown tokenization error";
    }

    PyObject* error_line = NULL;
    PyObject* value = NULL;

    Py_ssize_t input_size;
    const char *input = _PyTokenizer_SpanView(
        tok, info.input_span, &input_size);
    Py_ssize_t size = input_size;
    assert(input[size-1] == '\n');
    size -= 1; // Remove the newline character from the end of the line
    error_line = PyUnicode_DecodeUTF8(input, size, "replace");
    if (!error_line) {
        goto exit;
    }

    Py_ssize_t offset = _PyPegen_byte_offset_to_character_offset(error_line, input_size);
    if (offset == -1) {
        goto exit;
    }
    value = Py_BuildValue("(s(OinOOO))", msg, info.filename,
                          info.location.lineno, offset, error_line,
                          Py_None, Py_None);
    if (!value) {
        goto exit;
    }

    PyErr_SetObject(errtype, value);

exit:
    Py_XDECREF(error_line);
    Py_XDECREF(value);
}

static PyObject *
_get_current_line(tokenizeriterobject *it, int current_lineno,
                  const _PyToken_View *view, int *line_changed)
{
    _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(it);
    if (current_lineno != it->last_lineno) {
        Py_ssize_t size;
        const char *line_start = _PyTokenizer_SpanView(
            it->tok, view->line_span, &size);
        if (size > 0 && view->implicit_newline) {
            size--;
        }
        Py_XDECREF(it->last_line);
        it->last_line = PyUnicode_DecodeUTF8(line_start, size, "replace");
        it->byte_col_offset_diff = 0;
    }
    else {
        *line_changed = 0;
    }
    return it->last_line;
}

static int
_get_col_offsets(tokenizeriterobject *it, const struct token *token,
                 const _PyToken_View *view, PyObject *line, int line_changed,
                 Py_ssize_t *col_offset, Py_ssize_t *end_col_offset)
{
    _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(it);
    _PyTok_Off token_start = token->span.start;
    _PyTok_Off token_end = token->span.end;
    Py_ssize_t lineno = token->start_loc.lineno;
    Py_ssize_t end_lineno = token->end_loc.lineno;
    Py_ssize_t byte_offset = -1;
    if (token_start >= view->line_span.start) {
        byte_offset = token_start - view->line_span.start;
        if (line_changed) {
            *col_offset = _PyPegen_byte_offset_to_character_offset_line(line, 0, byte_offset);
            if (*col_offset < 0) {
                return -1;
            }
            it->byte_col_offset_diff = byte_offset - *col_offset;
        }
        else {
            *col_offset = byte_offset - it->byte_col_offset_diff;
        }
    }

    if (token_end >= view->end_line_start) {
        Py_ssize_t end_byte_offset = token_end - view->end_line_start;
        if (lineno == end_lineno) {
            // Avoid rescanning the prefix of a very long line.
            Py_ssize_t token_col_offset = _PyPegen_byte_offset_to_character_offset_line(line, byte_offset, end_byte_offset);
            if (token_col_offset < 0) {
                return -1;
            }
            *end_col_offset = *col_offset + token_col_offset;
            it->byte_col_offset_diff += token_end - token_start - token_col_offset;
        }
        else {
            *end_col_offset = _PyPegen_byte_offset_to_character_offset_line(
                line, view->end_line_start - view->line_span.start,
                token_end - view->line_span.start);
            if (*end_col_offset < 0) {
                return -1;
            }
            it->byte_col_offset_diff += end_byte_offset - *end_col_offset;
        }
    }
    it->last_lineno = lineno;
    return 0;
}

/*[clinic input]
@critical_section
_tokenizer.tokenizeriter.__next__ as tokenizeriter_next
[clinic start generated code]*/

static PyObject *
tokenizeriter_next_impl(tokenizeriterobject *self)
/*[clinic end generated code: output=745c9c646781ff45 input=66f70c965ba8ccfe]*/
{
    tokenizeriterobject *it = self;
    PyObject* result = NULL;

    if (it->done) {
        PyErr_SetString(PyExc_StopIteration, "EOF");
        return NULL;
    }

    struct token token;
    _PyToken_Init(&token);

    _PyTokenizer_Get(it->tok, &token);
    int type = token.type;
    if (type == ERRORTOKEN) {
        if(!PyErr_Occurred()) {
            _tokenizer_error(it);
            assert(PyErr_Occurred());
        }
        goto exit;
    }
    _PyToken_View view;
    _PyToken_GetView(it->tok, &token, &view);
    PyObject *str = PyUnicode_FromStringAndSize(view.text, view.length);
    if (str == NULL) {
        goto exit;
    }

    int is_trailing_token = type == ENDMARKER || (type == DEDENT && view.at_eof);

    PyObject* line = NULL;
    int line_changed = 1;
    if (it->extra_tokens && is_trailing_token) {
        line = Py_GetConstant(Py_CONSTANT_EMPTY_STR);
    } else {
        line = _get_current_line(
            it, token.end_loc.lineno, &view, &line_changed);
    }
    if (line == NULL) {
        Py_DECREF(str);
        goto exit;
    }

    Py_ssize_t lineno = token.start_loc.lineno;
    Py_ssize_t end_lineno = token.end_loc.lineno;
    Py_ssize_t col_offset = -1;
    Py_ssize_t end_col_offset = -1;
    if (_get_col_offsets(it, &token, &view, line,
                         line_changed, &col_offset, &end_col_offset) < 0) {
        Py_DECREF(str);
        goto exit;
    }

    if (it->extra_tokens) {
        if (is_trailing_token) {
            lineno = end_lineno = lineno + 1;
            col_offset = end_col_offset = 0;
        }
        // Necessary adjustments to match the original Python tokenize
        // implementation
        if (type > DEDENT && type < OP) {
            type = OP;
        }
        else if (type == NEWLINE) {
            if (!view.implicit_newline) {
                Py_DECREF(str);
                if (view.text[0] == '\r') {
                    str = PyUnicode_FromString("\r\n");
                } else {
                    str = PyUnicode_FromString("\n");
                }
            }
            end_col_offset++;
        }
        else if (type == NL) {
            if (view.implicit_newline) {
                Py_DECREF(str);
                str = Py_GetConstant(Py_CONSTANT_EMPTY_STR);
            }
        }

        if (str == NULL) {
            goto exit;
        }
    }

    result = Py_BuildValue("(iN(nn)(nn)O)", type, str, lineno, col_offset, end_lineno, end_col_offset, line);
exit:
    _PyToken_Free(&token);
    if (type == ENDMARKER) {
        it->done = 1;
    }
    return result;
}

static void
tokenizeriter_dealloc(PyObject *op)
{
    tokenizeriterobject *it = (tokenizeriterobject*)op;
    PyTypeObject *tp = Py_TYPE(it);
    PyObject_GC_UnTrack(it);
    Py_XDECREF(it->last_line);
    _PyTokenizer_Free(it->tok);
    tp->tp_free(it);
    Py_DECREF(tp);
}

static int
tokenizeriter_traverse(PyObject *op, visitproc visit, void *arg)
{
    tokenizeriterobject *it = (tokenizeriterobject *)op;
    Py_VISIT(Py_TYPE(it));
    Py_VISIT(it->last_line);
    return _PyTokenizer_Traverse(it->tok, visit, arg);
}

static PyType_Slot tokenizeriter_slots[] = {
    {Py_tp_new, tokenizeriter_new},
    {Py_tp_dealloc, tokenizeriter_dealloc},
    {Py_tp_traverse, tokenizeriter_traverse},
    {Py_tp_getattro, PyObject_GenericGetAttr},
    {Py_tp_iter, PyObject_SelfIter},
    {Py_tp_iternext, tokenizeriter_next},
    {0, NULL},
};

static PyType_Spec tokenizeriter_spec = {
    .name = "_tokenize.TokenizerIter",
    .basicsize = sizeof(tokenizeriterobject),
    .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_IMMUTABLETYPE | Py_TPFLAGS_HAVE_GC),
    .slots = tokenizeriter_slots,
};

static int
tokenizemodule_exec(PyObject *m)
{
    tokenize_state *state = get_tokenize_state(m);
    if (state == NULL) {
        return -1;
    }

    state->TokenizerIter = (PyTypeObject *)PyType_FromModuleAndSpec(m, &tokenizeriter_spec, NULL);
    if (state->TokenizerIter == NULL) {
        return -1;
    }
    if (PyModule_AddType(m, state->TokenizerIter) < 0) {
        return -1;
    }

    return 0;
}

static PyMethodDef tokenize_methods[] = {
    {NULL, NULL, 0, NULL} /* Sentinel */
};

static PyModuleDef_Slot tokenizemodule_slots[] = {
    _Py_ABI_SLOT,
    {Py_mod_exec, tokenizemodule_exec},
    {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED},
    {Py_mod_gil, Py_MOD_GIL_NOT_USED},
    {0, NULL}
};

static int
tokenizemodule_traverse(PyObject *m, visitproc visit, void *arg)
{
    tokenize_state *state = get_tokenize_state(m);
    Py_VISIT(state->TokenizerIter);
    return 0;
}

static int
tokenizemodule_clear(PyObject *m)
{
    tokenize_state *state = get_tokenize_state(m);
    Py_CLEAR(state->TokenizerIter);
    return 0;
}

static void
tokenizemodule_free(void *m)
{
    tokenizemodule_clear((PyObject *)m);
}

static struct PyModuleDef _tokenizemodule = {
    PyModuleDef_HEAD_INIT,
    .m_name = "_tokenize",
    .m_size = sizeof(tokenize_state),
    .m_slots = tokenizemodule_slots,
    .m_methods = tokenize_methods,
    .m_traverse = tokenizemodule_traverse,
    .m_clear = tokenizemodule_clear,
    .m_free = tokenizemodule_free,
};

PyMODINIT_FUNC
PyInit__tokenize(void)
{
    return PyModuleDef_Init(&_tokenizemodule);
}

Coverage Report

Created: 2026-10-31 06:21

Line	Count	Source
1		#include "Python.h"
2		#include "errcode.h"
3		#include "internal/pycore_critical_section.h" // Py_BEGIN_CRITICAL_SECTION
4		#include "../Parser/tokenizer/tokenizer.h"
5		#include "../Parser/pegen.h" // _PyPegen_byte_offset_to_character_offset()
6
7		static struct PyModuleDef _tokenizemodule;
8
9		typedef struct {
10		PyTypeObject *TokenizerIter;
11		} tokenize_state;
12
13		static tokenize_state *
14	674	get_tokenize_state(PyObject *module) {
15	674	return (tokenize_state *)PyModule_GetState(module);
16	674	}
17
18		#define _tokenize_get_state_by_type(type) \
19		get_tokenize_state(PyType_GetModuleByDef(type, &_tokenizemodule))
20
21		#include "pycore_runtime.h"
22
23		/*[clinic input]
24		module _tokenizer
25		class _tokenizer.tokenizeriter "tokenizeriterobject *" "_tokenize_get_state_by_type(type)->TokenizerIter"
26		[clinic start generated code]*/
27		/[clinic end generated code: output=da39a3ee5e6b4b0d input=96d98ee2fef7a8bc]/
28
29		typedef struct
30		{
31		PyObject_HEAD struct tok_state *tok;
32		int done;
33		int extra_tokens;
34
35		/* Needed to cache line for performance */
36		PyObject *last_line;
37		Py_ssize_t last_lineno;
38		Py_ssize_t byte_col_offset_diff;
39		} tokenizeriterobject;
40
41		#include "clinic/Python-tokenize.c.h"
42
43		/*[clinic input]
44		@classmethod
45		_tokenizer.tokenizeriter.__new__ as tokenizeriter_new
46
47		readline: object
48		/
49		*
50		extra_tokens: bool
51		encoding: str(c_default="NULL") = 'utf-8'
52		[clinic start generated code]*/
53
54		static PyObject *
55		tokenizeriter_new_impl(PyTypeObject type, PyObject readline,
56		int extra_tokens, const char *encoding)
57		/[clinic end generated code: output=7501a1211683ce16 input=f7dddf8a613ae8bd]/
58	16	{
59	16	struct tok_state *tok = _PyTokenizer_FromReadline(readline, encoding);
60	16	if (tok == NULL) {
61	0	return NULL;
62	0	}
63	16	_Py_DECLARE_STR(anon_string, "<string>");
64	16	_PyTokenizer_SetContext(tok, &_Py_STR(anon_string), NULL);
65	16	tokenizeriterobject self = (tokenizeriterobject )type->tp_alloc(type, 0);
66	16	if (self == NULL) {
67	0	_PyTokenizer_Free(tok);
68	0	return NULL;
69	0	}
70	16	self->tok = tok;
71	16	_PyTokenizer_SetOptions(self->tok, extra_tokens, 0);
72	16	self->extra_tokens = extra_tokens;
73	16	self->done = 0;
74
75	16	self->last_line = NULL;
76	16	self->byte_col_offset_diff = 0;
77	16	self->last_lineno = 0;
78
79	16	return (PyObject *)self;
80	16	}
81
82		static void
83		_tokenizer_error(tokenizeriterobject *it)
84	0	{
85	0	_Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(it);
86	0	assert(!PyErr_Occurred());
87
88	0	const char *msg = NULL;
89	0	PyObject* errtype = PyExc_SyntaxError;
90	0	struct tok_state *tok = it->tok;
91	0	_PyTokenizer_Info info = _PyTokenizer_GetInfo(tok);
92	0	switch (info.status) {
93	0	case E_TOKEN:
94	0	msg = "invalid token";
95	0	break;
96	0	case E_EOF:
97	0	PyErr_SetString(PyExc_SyntaxError, "unexpected EOF in multi-line statement");
98	0	PyErr_SyntaxLocationObject(
99	0	info.filename, info.location.lineno,
100	0	(int)Py_MAX(0, info.input_span.end - info.input_span.start));
101	0	return;
102	0	case E_DEDENT:
103	0	msg = "unindent does not match any outer indentation level";
104	0	errtype = PyExc_IndentationError;
105	0	break;
106	0	case E_INTR:
107	0	PyErr_SetNone(PyExc_KeyboardInterrupt);
108	0	return;
109	0	case E_NOMEM:
110	0	PyErr_NoMemory();
111	0	return;
112	0	case E_TABSPACE:
113	0	errtype = PyExc_TabError;
114	0	msg = "inconsistent use of tabs and spaces in indentation";
115	0	break;
116	0	case E_TOODEEP:
117	0	errtype = PyExc_IndentationError;
118	0	msg = "too many levels of indentation";
119	0	break;
120	0	case E_LINECONT: {
121	0	msg = "unexpected character after line continuation character";
122	0	break;
123	0	}
124	0	default:
125	0	msg = "unknown tokenization error";
126	0	}
127
128	0	PyObject* error_line = NULL;
129	0	PyObject* value = NULL;
130
131	0	Py_ssize_t input_size;
132	0	const char *input = _PyTokenizer_SpanView(
133	0	tok, info.input_span, &input_size);
134	0	Py_ssize_t size = input_size;
135	0	assert(input[size-1] == '\n');
136	0	size -= 1; // Remove the newline character from the end of the line
137	0	error_line = PyUnicode_DecodeUTF8(input, size, "replace");
138	0	if (!error_line) {
139	0	goto exit;
140	0	}
141
142	0	Py_ssize_t offset = _PyPegen_byte_offset_to_character_offset(error_line, input_size);
143	0	if (offset == -1) {
144	0	goto exit;
145	0	}
146	0	value = Py_BuildValue("(s(OinOOO))", msg, info.filename,
147	0	info.location.lineno, offset, error_line,
148	0	Py_None, Py_None);
149	0	if (!value) {
150	0	goto exit;
151	0	}
152
153	0	PyErr_SetObject(errtype, value);
154
155	0	exit:
156	0	Py_XDECREF(error_line);
157	0	Py_XDECREF(value);
158	0	}
159
160		static PyObject *
161		_get_current_line(tokenizeriterobject *it, int current_lineno,
162		const _PyToken_View view, int line_changed)
163	1.99k	{
164	1.99k	_Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(it);
165	1.99k	if (current_lineno != it->last_lineno) {
166	316	Py_ssize_t size;
167	316	const char *line_start = _PyTokenizer_SpanView(
168	316	it->tok, view->line_span, &size);
169	316	if (size > 0 && view->implicit_newline) {
170	0	size--;
171	0	}
172	316	Py_XDECREF(it->last_line);
173	316	it->last_line = PyUnicode_DecodeUTF8(line_start, size, "replace");
174	316	it->byte_col_offset_diff = 0;
175	316	}
176	1.67k	else {
177	1.67k	*line_changed = 0;
178	1.67k	}
179	1.99k	return it->last_line;
180	1.99k	}
181
182		static int
183		_get_col_offsets(tokenizeriterobject it, const struct token token,
184		const _PyToken_View view, PyObject line, int line_changed,
185		Py_ssize_t col_offset, Py_ssize_t end_col_offset)
186	2.00k	{
187	2.00k	_Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(it);
188	2.00k	_PyTok_Off token_start = token->span.start;
189	2.00k	_PyTok_Off token_end = token->span.end;
190	2.00k	Py_ssize_t lineno = token->start_loc.lineno;
191	2.00k	Py_ssize_t end_lineno = token->end_loc.lineno;
192	2.00k	Py_ssize_t byte_offset = -1;
193	2.00k	if (token_start >= view->line_span.start) {
194	2.00k	byte_offset = token_start - view->line_span.start;
195	2.00k	if (line_changed) {
196	328	*col_offset = _PyPegen_byte_offset_to_character_offset_line(line, 0, byte_offset);
197	328	if (*col_offset < 0) {
198	0	return -1;
199	0	}
200	328	it->byte_col_offset_diff = byte_offset - *col_offset;
201	328	}
202	1.67k	else {
203	1.67k	*col_offset = byte_offset - it->byte_col_offset_diff;
204	1.67k	}
205	2.00k	}
206
207	2.00k	if (token_end >= view->end_line_start) {
208	2.00k	Py_ssize_t end_byte_offset = token_end - view->end_line_start;
209	2.00k	if (lineno == end_lineno) {
210		// Avoid rescanning the prefix of a very long line.
211	2.00k	Py_ssize_t token_col_offset = _PyPegen_byte_offset_to_character_offset_line(line, byte_offset, end_byte_offset);
212	2.00k	if (token_col_offset < 0) {
213	0	return -1;
214	0	}
215	2.00k	end_col_offset = col_offset + token_col_offset;
216	2.00k	it->byte_col_offset_diff += token_end - token_start - token_col_offset;
217	2.00k	}
218	4	else {
219	4	*end_col_offset = _PyPegen_byte_offset_to_character_offset_line(
220	4	line, view->end_line_start - view->line_span.start,
221	4	token_end - view->line_span.start);
222	4	if (*end_col_offset < 0) {
223	0	return -1;
224	0	}
225	4	it->byte_col_offset_diff += end_byte_offset - *end_col_offset;
226	4	}
227	2.00k	}
228	2.00k	it->last_lineno = lineno;
229	2.00k	return 0;
230	2.00k	}
231
232		/*[clinic input]
233		@critical_section
234		_tokenizer.tokenizeriter.__next__ as tokenizeriter_next
235		[clinic start generated code]*/
236
237		static PyObject *
238		tokenizeriter_next_impl(tokenizeriterobject *self)
239		/[clinic end generated code: output=745c9c646781ff45 input=66f70c965ba8ccfe]/
240	2.00k	{
241	2.00k	tokenizeriterobject *it = self;
242	2.00k	PyObject* result = NULL;
243
244	2.00k	if (it->done) {
245	0	PyErr_SetString(PyExc_StopIteration, "EOF");
246	0	return NULL;
247	0	}
248
249	2.00k	struct token token;
250	2.00k	_PyToken_Init(&token);
251
252	2.00k	_PyTokenizer_Get(it->tok, &token);
253	2.00k	int type = token.type;
254	2.00k	if (type == ERRORTOKEN) {
255	0	if(!PyErr_Occurred()) {
256	0	_tokenizer_error(it);
257	0	assert(PyErr_Occurred());
258	0	}
259	0	goto exit;
260	0	}
261	2.00k	_PyToken_View view;
262	2.00k	_PyToken_GetView(it->tok, &token, &view);
263	2.00k	PyObject *str = PyUnicode_FromStringAndSize(view.text, view.length);
264	2.00k	if (str == NULL) {
265	0	goto exit;
266	0	}
267
268	2.00k	int is_trailing_token = type == ENDMARKER \|\| (type == DEDENT && view.at_eof);
269
270	2.00k	PyObject* line = NULL;
271	2.00k	int line_changed = 1;
272	2.00k	if (it->extra_tokens && is_trailing_token) {
273	16	line = Py_GetConstant(Py_CONSTANT_EMPTY_STR);
274	1.99k	} else {
275	1.99k	line = _get_current_line(
276	1.99k	it, token.end_loc.lineno, &view, &line_changed);
277	1.99k	}
278	2.00k	if (line == NULL) {
279	0	Py_DECREF(str);
280	0	goto exit;
281	0	}
282
283	2.00k	Py_ssize_t lineno = token.start_loc.lineno;
284	2.00k	Py_ssize_t end_lineno = token.end_loc.lineno;
285	2.00k	Py_ssize_t col_offset = -1;
286	2.00k	Py_ssize_t end_col_offset = -1;
287	2.00k	if (_get_col_offsets(it, &token, &view, line,
288	2.00k	line_changed, &col_offset, &end_col_offset) < 0) {
289	0	Py_DECREF(str);
290	0	goto exit;
291	0	}
292
293	2.00k	if (it->extra_tokens) {
294	2.00k	if (is_trailing_token) {
295	16	lineno = end_lineno = lineno + 1;
296	16	col_offset = end_col_offset = 0;
297	16	}
298		// Necessary adjustments to match the original Python tokenize
299		// implementation
300	2.00k	if (type > DEDENT && type < OP) {
301	764	type = OP;
302	764	}
303	1.24k	else if (type == NEWLINE) {
304	160	if (!view.implicit_newline) {
305	160	Py_DECREF(str);
306	160	if (view.text[0] == '\r') {
307	0	str = PyUnicode_FromString("\r\n");
308	160	} else {
309	160	str = PyUnicode_FromString("\n");
310	160	}
311	160	}
312	160	end_col_offset++;
313	160	}
314	1.08k	else if (type == NL) {
315	140	if (view.implicit_newline) {
316	0	Py_DECREF(str);
317	0	str = Py_GetConstant(Py_CONSTANT_EMPTY_STR);
318	0	}
319	140	}
320
321	2.00k	if (str == NULL) {
322	0	goto exit;
323	0	}
324	2.00k	}
325
326	2.00k	result = Py_BuildValue("(iN(nn)(nn)O)", type, str, lineno, col_offset, end_lineno, end_col_offset, line);
327	2.00k	exit:
328	2.00k	_PyToken_Free(&token);
329	2.00k	if (type == ENDMARKER) {
330	4	it->done = 1;
331	4	}
332	2.00k	return result;
333	2.00k	}
334
335		static void
336		tokenizeriter_dealloc(PyObject *op)
337	16	{
338	16	tokenizeriterobject it = (tokenizeriterobject)op;
339	16	PyTypeObject *tp = Py_TYPE(it);
340	16	PyObject_GC_UnTrack(it);
341	16	Py_XDECREF(it->last_line);
342	16	_PyTokenizer_Free(it->tok);
343	16	tp->tp_free(it);
344	16	Py_DECREF(tp);
345	16	}
346
347		static int
348		tokenizeriter_traverse(PyObject op, visitproc visit, void arg)
349	0	{
350	0	tokenizeriterobject it = (tokenizeriterobject )op;
351	0	Py_VISIT(Py_TYPE(it));
352	0	Py_VISIT(it->last_line);
353	0	return _PyTokenizer_Traverse(it->tok, visit, arg);
354	0	}
355
356		static PyType_Slot tokenizeriter_slots[] = {
357		{Py_tp_new, tokenizeriter_new},
358		{Py_tp_dealloc, tokenizeriter_dealloc},
359		{Py_tp_traverse, tokenizeriter_traverse},
360		{Py_tp_getattro, PyObject_GenericGetAttr},
361		{Py_tp_iter, PyObject_SelfIter},
362		{Py_tp_iternext, tokenizeriter_next},
363		{0, NULL},
364		};
365
366		static PyType_Spec tokenizeriter_spec = {
367		.name = "_tokenize.TokenizerIter",
368		.basicsize = sizeof(tokenizeriterobject),
369		.flags = (Py_TPFLAGS_DEFAULT \| Py_TPFLAGS_IMMUTABLETYPE \| Py_TPFLAGS_HAVE_GC),
370		.slots = tokenizeriter_slots,
371		};
372
373		static int
374		tokenizemodule_exec(PyObject *m)
375	6	{
376	6	tokenize_state *state = get_tokenize_state(m);
377	6	if (state == NULL) {
378	0	return -1;
379	0	}
380
381	6	state->TokenizerIter = (PyTypeObject *)PyType_FromModuleAndSpec(m, &tokenizeriter_spec, NULL);
382	6	if (state->TokenizerIter == NULL) {
383	0	return -1;
384	0	}
385	6	if (PyModule_AddType(m, state->TokenizerIter) < 0) {
386	0	return -1;
387	0	}
388
389	6	return 0;
390	6	}
391
392		static PyMethodDef tokenize_methods[] = {
393		{NULL, NULL, 0, NULL} /* Sentinel */
394		};
395
396		static PyModuleDef_Slot tokenizemodule_slots[] = {
397		_Py_ABI_SLOT,
398		{Py_mod_exec, tokenizemodule_exec},
399		{Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED},
400		{Py_mod_gil, Py_MOD_GIL_NOT_USED},
401		{0, NULL}
402		};
403
404		static int
405		tokenizemodule_traverse(PyObject m, visitproc visit, void arg)
406	668	{
407	668	tokenize_state *state = get_tokenize_state(m);
408	668	Py_VISIT(state->TokenizerIter);
409	668	return 0;
410	668	}
411
412		static int
413		tokenizemodule_clear(PyObject *m)
414	0	{
415	0	tokenize_state *state = get_tokenize_state(m);
416	0	Py_CLEAR(state->TokenizerIter);
417	0	return 0;
418	0	}
419
420		static void
421		tokenizemodule_free(void *m)
422	0	{
423	0	tokenizemodule_clear((PyObject *)m);
424	0	}
425
426		static struct PyModuleDef _tokenizemodule = {
427		PyModuleDef_HEAD_INIT,
428		.m_name = "_tokenize",
429		.m_size = sizeof(tokenize_state),
430		.m_slots = tokenizemodule_slots,
431		.m_methods = tokenize_methods,
432		.m_traverse = tokenizemodule_traverse,
433		.m_clear = tokenizemodule_clear,
434		.m_free = tokenizemodule_free,
435		};
436
437		PyMODINIT_FUNC
438		PyInit__tokenize(void)
439	6	{
440	6	return PyModuleDef_Init(&_tokenizemodule);
441	6	}