/src/cpython/Parser/lexer/layout.c

Source
#include "Python.h"
#include "errcode.h"
#include "pycore_token.h"

#include "lexer_internal.h"
#include "../tokenizer/helpers.h"
#include "../tokenizer/reader.h"

#define TABSIZE 8
#define ALTTABSIZE 1

int
_PyLexer_ContinueLine(struct tok_state *tok)
{
    int c = tok_nextc(tok);
    if (c == '\r') {
        c = tok_nextc(tok);
    }
    if (c != '\n') {
        tok->done = E_LINECONT;
        return -1;
    }
    c = tok_nextc(tok);
    if (c == EOF) {
        tok->done = E_EOF;
        tok->cur = tok->inp;
        return -1;
    } else {
        tok_backup(tok, c);
    }
    return c;
}


static int
update_indentation(struct tok_state *tok, int col, int altcol)
{
    lexer_layout_state *layout = &tok->layout;
    if (col == layout->stack[layout->depth].column) {
        if (altcol != layout->stack[layout->depth].alternate_column) {
            _PyTokenizer_indenterror(tok);
            return -1;
        }
    }
    else if (col > layout->stack[layout->depth].column) {
        if (layout->depth + 1 >= MAXINDENT) {
            tok->done = E_TOODEEP;
            tok->cur = tok->inp;
            return -1;
        }
        if (altcol <= layout->stack[layout->depth].alternate_column) {
            _PyTokenizer_indenterror(tok);
            return -1;
        }
        layout->pending++;
        layout->stack[++layout->depth] = (indentation_level){col, altcol};
    }
    else {
        while (layout->depth > 0 &&
            col < layout->stack[layout->depth].column) {
            layout->pending--;
            layout->depth--;
        }
        if (col != layout->stack[layout->depth].column) {
            tok->done = E_DEDENT;
            tok->cur = tok->inp;
            return -1;
        }
        if (altcol != layout->stack[layout->depth].alternate_column) {
            _PyTokenizer_indenterror(tok);
            return -1;
        }
    }
    return 0;
}

int
_PyLexer_BeginLine(struct tok_state *tok)
{
    assert(tok->layout.at_bol);
    int c;
    int blankline = 0;
    int col = 0;
    int altcol = 0;
    tok->layout.at_bol = 0;
    int cont_line_col = 0;
    for (;;) {
        c = tok_nextc(tok);
        if (c == ' ') {
            col++, altcol++;
        }
        else if (c == '\t') {
            col = (col / TABSIZE + 1) * TABSIZE;
            altcol = (altcol / ALTTABSIZE + 1) * ALTTABSIZE;
        }
        else if (c == '\014')  {/* Control-L (formfeed) */
            col = altcol = 0; /* For Emacs users */
        }
        else if (c == '\\') {
            // Indentation cannot be split over multiple physical lines
            // using backslashes. This means that if we found a backslash
            // preceded by whitespace, **the first one we find** determines
            // the level of indentation of whatever comes next.
            cont_line_col = cont_line_col ? cont_line_col : col;
            if ((c = _PyLexer_ContinueLine(tok)) == -1) {
                return -1;
            }
        }
        else if (c == EOF && PyErr_Occurred()) {
            return -1;
        }
        else {
            break;
        }
    }
    tok_backup(tok, c);
    if (c == '#' || c == '\n' || c == '\r') {
        int interactive = _PyTok_ReaderIsInteractive(tok);
        /* Lines with only whitespace and/or comments
           shouldn't affect the indentation and are
           not passed to the parser as NEWLINE tokens,
           except *totally* empty lines in interactive
           mode, which signal the end of a command group. */
        if (col == 0 && c == '\n' && interactive) {
            blankline = 0; /* Let it through */
        }
        else if (interactive && tok->lineno == 1) {
            /* In interactive mode, if the first line contains
               only spaces and/or a comment, let it through. */
            blankline = 0;
            col = altcol = 0;
        }
        else {
            blankline = 1; /* Ignore completely */
        }
    }
    if (!blankline && tok->level == 0) {
        col = cont_line_col ? cont_line_col : col;
        altcol = cont_line_col ? cont_line_col : altcol;
        if (update_indentation(tok, col, altcol) < 0) {
            return -1;
        }
    }
    return blankline;
}

int
_PyLexer_IndentationToken(struct tok_state *tok, struct token *token)
{
    assert(tok->layout.pending != 0);
    _PyTok_Off p_start = -1;
    _PyTok_Off p_end = -1;
    if (tok->layout.pending < 0) {
        if (tok->tok_extra_tokens) {
            p_start = tok->cur;
            p_end = tok->cur;
        }
        tok->layout.pending++;
        return _PyLexer_token_setup(tok, token, DEDENT, p_start, p_end);
    }
    else {
        if (tok->tok_extra_tokens) {
            p_start = tok->buf_offset;
            p_end = tok->cur;
        }
        tok->layout.pending--;
        return _PyLexer_token_setup(tok, token, INDENT, p_start, p_end);
    }
}

int
_PyLexer_Newline(struct tok_state *tok, struct token *token, int blankline)
{
    tok->layout.at_bol = 1;
    if (blankline || tok->level > 0) {
        if (!tok->tok_extra_tokens) {
            return 0;
        }
    }
    else if (!tok->layout.comment_newline || !tok->tok_extra_tokens) {
        return _PyLexer_token_setup(tok, token, NEWLINE,
                                    tok->start, tok->cur - 1);
    }
    tok->layout.comment_newline = 0;
    return _PyLexer_token_setup(tok, token, NL, tok->start, tok->cur);
}

void
_PyLexer_ImplyDedents(struct tok_state *tok)
{
    if (tok->layout.depth != 0) {
        tok->layout.pending = -tok->layout.depth;
        tok->layout.depth = 0;
    }
}

Line	Count	Source
1		#include "Python.h"
2		#include "errcode.h"
3		#include "pycore_token.h"
4
5		#include "lexer_internal.h"
6		#include "../tokenizer/helpers.h"
7		#include "../tokenizer/reader.h"
8
9	1.29k	#define TABSIZE 8
10	1.29k	#define ALTTABSIZE 1
11
12		int
13		_PyLexer_ContinueLine(struct tok_state *tok)
14	1.23k	{
15	1.23k	int c = tok_nextc(tok);
16	1.23k	if (c == '\r') {
17	0	c = tok_nextc(tok);
18	0	}
19	1.23k	if (c != '\n') {
20	63	tok->done = E_LINECONT;
21	63	return -1;
22	63	}
23	1.17k	c = tok_nextc(tok);
24	1.17k	if (c == EOF) {
25	42	tok->done = E_EOF;
26	42	tok->cur = tok->inp;
27	42	return -1;
28	1.13k	} else {
29	1.13k	tok_backup(tok, c);
30	1.13k	}
31	1.13k	return c;
32	1.17k	}
33
34
35		static int
36		update_indentation(struct tok_state *tok, int col, int altcol)
37	188k	{
38	188k	lexer_layout_state *layout = &tok->layout;
39	188k	if (col == layout->stack[layout->depth].column) {
40	170k	if (altcol != layout->stack[layout->depth].alternate_column) {
41	1	_PyTokenizer_indenterror(tok);
42	1	return -1;
43	1	}
44	170k	}
45	18.2k	else if (col > layout->stack[layout->depth].column) {
46	10.0k	if (layout->depth + 1 >= MAXINDENT) {
47	0	tok->done = E_TOODEEP;
48	0	tok->cur = tok->inp;
49	0	return -1;
50	0	}
51	10.0k	if (altcol <= layout->stack[layout->depth].alternate_column) {
52	1	_PyTokenizer_indenterror(tok);
53	1	return -1;
54	1	}
55	10.0k	layout->pending++;
56	10.0k	layout->stack[++layout->depth] = (indentation_level){col, altcol};
57	10.0k	}
58	8.16k	else {
59	17.7k	while (layout->depth > 0 &&
60	13.7k	col < layout->stack[layout->depth].column) {
61	9.56k	layout->pending--;
62	9.56k	layout->depth--;
63	9.56k	}
64	8.16k	if (col != layout->stack[layout->depth].column) {
65	10	tok->done = E_DEDENT;
66	10	tok->cur = tok->inp;
67	10	return -1;
68	10	}
69	8.15k	if (altcol != layout->stack[layout->depth].alternate_column) {
70	1	_PyTokenizer_indenterror(tok);
71	1	return -1;
72	1	}
73	8.15k	}
74	188k	return 0;
75	188k	}
76
77		int
78		_PyLexer_BeginLine(struct tok_state *tok)
79	303k	{
80	303k	assert(tok->layout.at_bol);
81	303k	int c;
82	303k	int blankline = 0;
83	303k	int col = 0;
84	303k	int altcol = 0;
85	303k	tok->layout.at_bol = 0;
86	303k	int cont_line_col = 0;
87	573k	for (;;) {
88	573k	c = tok_nextc(tok);
89	573k	if (c == ' ') {
90	268k	col++, altcol++;
91	268k	}
92	305k	else if (c == '\t') {
93	649	col = (col / TABSIZE + 1) * TABSIZE;
94	649	altcol = (altcol / ALTTABSIZE + 1) * ALTTABSIZE;
95	649	}
96	305k	else if (c == '\014') {/* Control-L (formfeed) */
97	673	col = altcol = 0; /* For Emacs users */
98	673	}
99	304k	else if (c == '\\') {
100		// Indentation cannot be split over multiple physical lines
101		// using backslashes. This means that if we found a backslash
102		// preceded by whitespace, the first one we find determines
103		// the level of indentation of whatever comes next.
104	871	cont_line_col = cont_line_col ? cont_line_col : col;
105	871	if ((c = _PyLexer_ContinueLine(tok)) == -1) {
106	26	return -1;
107	26	}
108	871	}
109	303k	else if (c == EOF && PyErr_Occurred()) {
110	85	return -1;
111	85	}
112	303k	else {
113	303k	break;
114	303k	}
115	573k	}
116	303k	tok_backup(tok, c);
117	303k	if (c == '#' \|\| c == '\n' \|\| c == '\r') {
118	79.7k	int interactive = _PyTok_ReaderIsInteractive(tok);
119		/* Lines with only whitespace and/or comments
120		shouldn't affect the indentation and are
121		not passed to the parser as NEWLINE tokens,
122		except totally empty lines in interactive
123		mode, which signal the end of a command group. */
124	79.7k	if (col == 0 && c == '\n' && interactive) {
125	0	blankline = 0; /* Let it through */
126	0	}
127	79.7k	else if (interactive && tok->lineno == 1) {
128		/* In interactive mode, if the first line contains
129		only spaces and/or a comment, let it through. */
130	0	blankline = 0;
131	0	col = altcol = 0;
132	0	}
133	79.7k	else {
134	79.7k	blankline = 1; /* Ignore completely */
135	79.7k	}
136	79.7k	}
137	303k	if (!blankline && tok->level == 0) {
138	188k	col = cont_line_col ? cont_line_col : col;
139	188k	altcol = cont_line_col ? cont_line_col : altcol;
140	188k	if (update_indentation(tok, col, altcol) < 0) {
141	13	return -1;
142	13	}
143	188k	}
144	303k	return blankline;
145	303k	}
146
147		int
148		_PyLexer_IndentationToken(struct tok_state tok, struct token token)
149	19.6k	{
150	19.6k	assert(tok->layout.pending != 0);
151	19.6k	_PyTok_Off p_start = -1;
152	19.6k	_PyTok_Off p_end = -1;
153	19.6k	if (tok->layout.pending < 0) {
154	9.52k	if (tok->tok_extra_tokens) {
155	60	p_start = tok->cur;
156	60	p_end = tok->cur;
157	60	}
158	9.52k	tok->layout.pending++;
159	9.52k	return _PyLexer_token_setup(tok, token, DEDENT, p_start, p_end);
160	9.52k	}
161	10.0k	else {
162	10.0k	if (tok->tok_extra_tokens) {
163	64	p_start = tok->buf_offset;
164	64	p_end = tok->cur;
165	64	}
166	10.0k	tok->layout.pending--;
167	10.0k	return _PyLexer_token_setup(tok, token, INDENT, p_start, p_end);
168	10.0k	}
169	19.6k	}
170
171		int
172		_PyLexer_Newline(struct tok_state tok, struct token token, int blankline)
173	241k	{
174	241k	tok->layout.at_bol = 1;
175	241k	if (blankline \|\| tok->level > 0) {
176	114k	if (!tok->tok_extra_tokens) {
177	114k	return 0;
178	114k	}
179	114k	}
180	126k	else if (!tok->layout.comment_newline \|\| !tok->tok_extra_tokens) {
181	126k	return _PyLexer_token_setup(tok, token, NEWLINE,
182	126k	tok->start, tok->cur - 1);
183	126k	}
184	140	tok->layout.comment_newline = 0;
185	140	return _PyLexer_token_setup(tok, token, NL, tok->start, tok->cur);
186	241k	}
187
188		void
189		_PyLexer_ImplyDedents(struct tok_state *tok)
190	0	{
191	0	if (tok->layout.depth != 0) {
192	0	tok->layout.pending = -tok->layout.depth;
193	0	tok->layout.depth = 0;
194	0	}
195	0	}

Coverage Report

Created: 2026-10-17 06:15