/src/cpython/Parser/lexer/layout.c

Source
#include "Python.h"
#include "errcode.h"
#include "pycore_token.h"

#include "lexer_internal.h"
#include "../tokenizer/helpers.h"
#include "../tokenizer/reader.h"

#define TABSIZE 8
#define ALTTABSIZE 1

int
_PyLexer_ContinueLine(struct tok_state *tok)
{
    int c = tok_nextc(tok);
    if (c == '\r') {
        c = tok_nextc(tok);
    }
    if (c != '\n') {
        tok->done = E_LINECONT;
        return -1;
    }
    c = tok_nextc(tok);
    if (tok->implicit_newline == _PyTokenizer_APPEND_NEWLINE) {
        tok->needs_newline |= _PyTokenizer_NEWLINE_CONTINUATION;
    }
    if (c == EOF) {
        tok->done = E_EOF;
        tok->cur = tok->inp;
        return -1;
    } else {
        tok_backup(tok, c);
    }
    return c;
}


static int
update_indentation(struct tok_state *tok, int col, int altcol)
{
    lexer_layout_state *layout = &tok->layout;
    if (col == layout->stack[layout->depth].column) {
        if (altcol != layout->stack[layout->depth].alternate_column) {
            _PyTokenizer_indenterror(tok);
            return -1;
        }
    }
    else if (col > layout->stack[layout->depth].column) {
        if (layout->depth + 1 >= MAXINDENT) {
            tok->done = E_TOODEEP;
            tok->cur = tok->inp;
            return -1;
        }
        if (altcol <= layout->stack[layout->depth].alternate_column) {
            _PyTokenizer_indenterror(tok);
            return -1;
        }
        layout->pending++;
        layout->stack[++layout->depth] = (indentation_level){col, altcol};
    }
    else {
        while (layout->depth > 0 &&
            col < layout->stack[layout->depth].column) {
            layout->pending--;
            layout->depth--;
        }
        if (col != layout->stack[layout->depth].column) {
            tok->done = E_DEDENT;
            tok->cur = tok->inp;
            return -1;
        }
        if (altcol != layout->stack[layout->depth].alternate_column) {
            _PyTokenizer_indenterror(tok);
            return -1;
        }
    }
    return 0;
}

int
_PyLexer_BeginLine(struct tok_state *tok)
{
    assert(tok->layout.at_bol);
    int c;
    int blankline = 0;
    int col = 0;
    int altcol = 0;
    tok->layout.at_bol = 0;
    int cont_line_col = 0;
    for (;;) {
        c = tok_nextc(tok);
        if (c == ' ') {
            col++, altcol++;
        }
        else if (c == '\t') {
            col = (col / TABSIZE + 1) * TABSIZE;
            altcol = (altcol / ALTTABSIZE + 1) * ALTTABSIZE;
        }
        else if (c == '\014')  {/* Control-L (formfeed) */
            col = altcol = 0; /* For Emacs users */
        }
        else if (c == '\\') {
            // Indentation cannot be split over multiple physical lines
            // using backslashes. This means that if we found a backslash
            // preceded by whitespace, **the first one we find** determines
            // the level of indentation of whatever comes next.
            cont_line_col = cont_line_col ? cont_line_col : col;
            if ((c = _PyLexer_ContinueLine(tok)) == -1) {
                return -1;
            }
        }
        else if (c == EOF && PyErr_Occurred()) {
            return -1;
        }
        else {
            break;
        }
    }
    tok_backup(tok, c);
    if (c == '\n' && tok->implicit_newline == _PyTokenizer_IMPLY_NEWLINE &&
        tok->level == 0 && (cont_line_col || col)) {
        // Without the newline, a whitespace-only line affects indentation.
        tok->needs_newline |= _PyTokenizer_NEWLINE_INDENT;
    }
    if (c == '#' || c == '\n' || c == '\r') {
        int interactive = _PyTok_ReaderIsInteractive(tok);
        /* Lines with only whitespace and/or comments
           shouldn't affect the indentation and are
           not passed to the parser as NEWLINE tokens,
           except *totally* empty lines in interactive
           mode, which signal the end of a command group. */
        if (col == 0 && c == '\n' && interactive) {
            blankline = 0; /* Let it through */
        }
        else if (interactive && tok->lineno == 1) {
            /* In interactive mode, if the first line contains
               only spaces and/or a comment, let it through. */
            blankline = 0;
            col = altcol = 0;
        }
        else {
            blankline = 1; /* Ignore completely */
        }
    }
    if (!blankline && tok->level == 0) {
        col = cont_line_col ? cont_line_col : col;
        altcol = cont_line_col ? cont_line_col : altcol;
        if (update_indentation(tok, col, altcol) < 0) {
            return -1;
        }
    }
    return blankline;
}

int
_PyLexer_IndentationToken(struct tok_state *tok, struct token *token)
{
    assert(tok->layout.pending != 0);
    _PyTok_Off p_start = -1;
    _PyTok_Off p_end = -1;
    if (tok->layout.pending < 0) {
        if (tok->tok_extra_tokens) {
            p_start = tok->cur;
            p_end = tok->cur;
        }
        tok->layout.pending++;
        return _PyLexer_token_setup(tok, token, DEDENT, p_start, p_end);
    }
    else {
        if (tok->tok_extra_tokens) {
            p_start = tok->buf_offset;
            p_end = tok->cur;
        }
        tok->layout.pending--;
        return _PyLexer_token_setup(tok, token, INDENT, p_start, p_end);
    }
}

int
_PyLexer_Newline(struct tok_state *tok, struct token *token, int blankline)
{
    if (!blankline && tok->implicit_newline == _PyTokenizer_IMPLY_NEWLINE) {
        // The statement's terminating newline was not in the input.
        tok->needs_newline |= _PyTokenizer_NEWLINE_STATEMENT;
    }
    tok->layout.at_bol = 1;
    if (blankline || tok->level > 0) {
        if (!tok->tok_extra_tokens) {
            return 0;
        }
    }
    else if (!tok->layout.comment_newline || !tok->tok_extra_tokens) {
        return _PyLexer_token_setup(tok, token, NEWLINE,
                                    tok->start, tok->cur - 1);
    }
    tok->layout.comment_newline = 0;
    return _PyLexer_token_setup(tok, token, NL, tok->start, tok->cur);
}

void
_PyLexer_ImplyDedents(struct tok_state *tok)
{
    if (tok->layout.depth != 0) {
        tok->layout.pending = -tok->layout.depth;
        tok->layout.depth = 0;
    }
}

Coverage Report

Created: 2026-10-31 06:21

Line	Count	Source
1		#include "Python.h"
2		#include "errcode.h"
3		#include "pycore_token.h"
4
5		#include "lexer_internal.h"
6		#include "../tokenizer/helpers.h"
7		#include "../tokenizer/reader.h"
8
9	1.05k	#define TABSIZE 8
10	1.05k	#define ALTTABSIZE 1
11
12		int
13		_PyLexer_ContinueLine(struct tok_state *tok)
14	1.27k	{
15	1.27k	int c = tok_nextc(tok);
16	1.27k	if (c == '\r') {
17	0	c = tok_nextc(tok);
18	0	}
19	1.27k	if (c != '\n') {
20	69	tok->done = E_LINECONT;
21	69	return -1;
22	69	}
23	1.20k	c = tok_nextc(tok);
24	1.20k	if (tok->implicit_newline == _PyTokenizer_APPEND_NEWLINE) {
25	0	tok->needs_newline \|= _PyTokenizer_NEWLINE_CONTINUATION;
26	0	}
27	1.20k	if (c == EOF) {
28	51	tok->done = E_EOF;
29	51	tok->cur = tok->inp;
30	51	return -1;
31	1.15k	} else {
32	1.15k	tok_backup(tok, c);
33	1.15k	}
34	1.15k	return c;
35	1.20k	}
36
37
38		static int
39		update_indentation(struct tok_state *tok, int col, int altcol)
40	210k	{
41	210k	lexer_layout_state *layout = &tok->layout;
42	210k	if (col == layout->stack[layout->depth].column) {
43	193k	if (altcol != layout->stack[layout->depth].alternate_column) {
44	1	_PyTokenizer_indenterror(tok);
45	1	return -1;
46	1	}
47	193k	}
48	17.5k	else if (col > layout->stack[layout->depth].column) {
49	9.64k	if (layout->depth + 1 >= MAXINDENT) {
50	0	tok->done = E_TOODEEP;
51	0	tok->cur = tok->inp;
52	0	return -1;
53	0	}
54	9.64k	if (altcol <= layout->stack[layout->depth].alternate_column) {
55	1	_PyTokenizer_indenterror(tok);
56	1	return -1;
57	1	}
58	9.64k	layout->pending++;
59	9.64k	layout->stack[++layout->depth] = (indentation_level){col, altcol};
60	9.64k	}
61	7.86k	else {
62	16.9k	while (layout->depth > 0 &&
63	12.7k	col < layout->stack[layout->depth].column) {
64	9.10k	layout->pending--;
65	9.10k	layout->depth--;
66	9.10k	}
67	7.86k	if (col != layout->stack[layout->depth].column) {
68	10	tok->done = E_DEDENT;
69	10	tok->cur = tok->inp;
70	10	return -1;
71	10	}
72	7.85k	if (altcol != layout->stack[layout->depth].alternate_column) {
73	1	_PyTokenizer_indenterror(tok);
74	1	return -1;
75	1	}
76	7.85k	}
77	210k	return 0;
78	210k	}
79
80		int
81		_PyLexer_BeginLine(struct tok_state *tok)
82	329k	{
83	329k	assert(tok->layout.at_bol);
84	329k	int c;
85	329k	int blankline = 0;
86	329k	int col = 0;
87	329k	int altcol = 0;
88	329k	tok->layout.at_bol = 0;
89	329k	int cont_line_col = 0;
90	583k	for (;;) {
91	583k	c = tok_nextc(tok);
92	583k	if (c == ' ') {
93	251k	col++, altcol++;
94	251k	}
95	331k	else if (c == '\t') {
96	527	col = (col / TABSIZE + 1) * TABSIZE;
97	527	altcol = (altcol / ALTTABSIZE + 1) * ALTTABSIZE;
98	527	}
99	331k	else if (c == '\014') {/* Control-L (formfeed) */
100	806	col = altcol = 0; /* For Emacs users */
101	806	}
102	330k	else if (c == '\\') {
103		// Indentation cannot be split over multiple physical lines
104		// using backslashes. This means that if we found a backslash
105		// preceded by whitespace, the first one we find determines
106		// the level of indentation of whatever comes next.
107	907	cont_line_col = cont_line_col ? cont_line_col : col;
108	907	if ((c = _PyLexer_ContinueLine(tok)) == -1) {
109	32	return -1;
110	32	}
111	907	}
112	329k	else if (c == EOF && PyErr_Occurred()) {
113	88	return -1;
114	88	}
115	329k	else {
116	329k	break;
117	329k	}
118	583k	}
119	329k	tok_backup(tok, c);
120	329k	if (c == '\n' && tok->implicit_newline == _PyTokenizer_IMPLY_NEWLINE &&
121	49	tok->level == 0 && (cont_line_col \|\| col)) {
122		// Without the newline, a whitespace-only line affects indentation.
123	36	tok->needs_newline \|= _PyTokenizer_NEWLINE_INDENT;
124	36	}
125	329k	if (c == '#' \|\| c == '\n' \|\| c == '\r') {
126	82.4k	int interactive = _PyTok_ReaderIsInteractive(tok);
127		/* Lines with only whitespace and/or comments
128		shouldn't affect the indentation and are
129		not passed to the parser as NEWLINE tokens,
130		except totally empty lines in interactive
131		mode, which signal the end of a command group. */
132	82.4k	if (col == 0 && c == '\n' && interactive) {
133	0	blankline = 0; /* Let it through */
134	0	}
135	82.4k	else if (interactive && tok->lineno == 1) {
136		/* In interactive mode, if the first line contains
137		only spaces and/or a comment, let it through. */
138	0	blankline = 0;
139	0	col = altcol = 0;
140	0	}
141	82.4k	else {
142	82.4k	blankline = 1; /* Ignore completely */
143	82.4k	}
144	82.4k	}
145	329k	if (!blankline && tok->level == 0) {
146	210k	col = cont_line_col ? cont_line_col : col;
147	210k	altcol = cont_line_col ? cont_line_col : altcol;
148	210k	if (update_indentation(tok, col, altcol) < 0) {
149	13	return -1;
150	13	}
151	210k	}
152	329k	return blankline;
153	329k	}
154
155		int
156		_PyLexer_IndentationToken(struct tok_state tok, struct token token)
157	18.7k	{
158	18.7k	assert(tok->layout.pending != 0);
159	18.7k	_PyTok_Off p_start = -1;
160	18.7k	_PyTok_Off p_end = -1;
161	18.7k	if (tok->layout.pending < 0) {
162	9.06k	if (tok->tok_extra_tokens) {
163	60	p_start = tok->cur;
164	60	p_end = tok->cur;
165	60	}
166	9.06k	tok->layout.pending++;
167	9.06k	return _PyLexer_token_setup(tok, token, DEDENT, p_start, p_end);
168	9.06k	}
169	9.64k	else {
170	9.64k	if (tok->tok_extra_tokens) {
171	64	p_start = tok->buf_offset;
172	64	p_end = tok->cur;
173	64	}
174	9.64k	tok->layout.pending--;
175	9.64k	return _PyLexer_token_setup(tok, token, INDENT, p_start, p_end);
176	9.64k	}
177	18.7k	}
178
179		int
180		_PyLexer_Newline(struct tok_state tok, struct token token, int blankline)
181	256k	{
182	256k	if (!blankline && tok->implicit_newline == _PyTokenizer_IMPLY_NEWLINE) {
183		// The statement's terminating newline was not in the input.
184	68.1k	tok->needs_newline \|= _PyTokenizer_NEWLINE_STATEMENT;
185	68.1k	}
186	256k	tok->layout.at_bol = 1;
187	256k	if (blankline \|\| tok->level > 0) {
188	119k	if (!tok->tok_extra_tokens) {
189	118k	return 0;
190	118k	}
191	119k	}
192	137k	else if (!tok->layout.comment_newline \|\| !tok->tok_extra_tokens) {
193	137k	return _PyLexer_token_setup(tok, token, NEWLINE,
194	137k	tok->start, tok->cur - 1);
195	137k	}
196	140	tok->layout.comment_newline = 0;
197	140	return _PyLexer_token_setup(tok, token, NL, tok->start, tok->cur);
198	256k	}
199
200		void
201		_PyLexer_ImplyDedents(struct tok_state *tok)
202	0	{
203	0	if (tok->layout.depth != 0) {
204	0	tok->layout.pending = -tok->layout.depth;
205	0	tok->layout.depth = 0;
206	0	}
207	0	}