Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/libcst/_parser/py_whitespace_parser.py: 12%
114 statements
« prev ^ index » next coverage.py v7.3.1, created at 2023-09-25 06:43 +0000
1# Copyright (c) Meta Platforms, Inc. and affiliates.
2#
3# This source code is licensed under the MIT license found in the
4# LICENSE file in the root directory of this source tree.
6from typing import List, Optional, Sequence, Tuple, Union
8from libcst._nodes.whitespace import (
9 Comment,
10 COMMENT_RE,
11 EmptyLine,
12 Newline,
13 NEWLINE_RE,
14 ParenthesizedWhitespace,
15 SIMPLE_WHITESPACE_RE,
16 SimpleWhitespace,
17 TrailingWhitespace,
18)
19from libcst._parser.types.config import BaseWhitespaceParserConfig
20from libcst._parser.types.whitespace_state import WhitespaceState as State
22# BEGIN PARSER ENTRYPOINTS
25def parse_simple_whitespace(
26 config: BaseWhitespaceParserConfig, state: State
27) -> SimpleWhitespace:
28 # The match never fails because the pattern can match an empty string
29 lines = config.lines
30 # pyre-fixme[16]: Optional type has no attribute `group`.
31 ws_line = SIMPLE_WHITESPACE_RE.match(lines[state.line - 1], state.column).group(0)
32 ws_line_list = [ws_line]
33 while "\\" in ws_line:
34 # continuation character
35 state.line += 1
36 state.column = 0
37 ws_line = SIMPLE_WHITESPACE_RE.match(lines[state.line - 1], state.column).group(
38 0
39 )
40 ws_line_list.append(ws_line)
42 # TODO: we could special-case the common case where there's no continuation
43 # character to avoid list construction and joining.
45 # once we've finished collecting continuation characters
46 state.column += len(ws_line)
47 return SimpleWhitespace("".join(ws_line_list))
50def parse_empty_lines(
51 config: BaseWhitespaceParserConfig,
52 state: State,
53 *,
54 override_absolute_indent: Optional[str] = None,
55) -> Sequence[EmptyLine]:
56 # If override_absolute_indent is true, then we need to parse all lines up
57 # to and including the last line that is indented at our level. These all
58 # belong to the footer and not to the next line's leading_lines. All lines
59 # that have indent=False and come after the last line where indent=True
60 # do not belong to this node.
61 state_for_line = State(
62 state.line, state.column, state.absolute_indent, state.is_parenthesized
63 )
64 lines: List[Tuple[State, EmptyLine]] = []
65 while True:
66 el = _parse_empty_line(
67 config, state_for_line, override_absolute_indent=override_absolute_indent
68 )
69 if el is None:
70 break
72 # Store the updated state with the element we parsed. Then make a new state
73 # clone for the next element.
74 lines.append((state_for_line, el))
75 state_for_line = State(
76 state_for_line.line,
77 state_for_line.column,
78 state.absolute_indent,
79 state.is_parenthesized,
80 )
82 if override_absolute_indent is not None:
83 # We need to find the last element that is indented, and then split the list
84 # at that point.
85 for i in range(len(lines) - 1, -1, -1):
86 if lines[i][1].indent:
87 lines = lines[: (i + 1)]
88 break
89 else:
90 # We didn't find any lines, throw them all away
91 lines = []
93 if lines:
94 # Update the state line and column to match the last line actually parsed.
95 final_state: State = lines[-1][0]
96 state.line = final_state.line
97 state.column = final_state.column
98 return [r[1] for r in lines]
101def parse_trailing_whitespace(
102 config: BaseWhitespaceParserConfig, state: State
103) -> TrailingWhitespace:
104 trailing_whitespace = _parse_trailing_whitespace(config, state)
105 if trailing_whitespace is None:
106 raise Exception(
107 "Internal Error: Failed to parse TrailingWhitespace. This should never "
108 + "happen because a TrailingWhitespace is never optional in the grammar, "
109 + "so this error should've been caught by parso first."
110 )
111 return trailing_whitespace
114def parse_parenthesizable_whitespace(
115 config: BaseWhitespaceParserConfig, state: State
116) -> Union[SimpleWhitespace, ParenthesizedWhitespace]:
117 if state.is_parenthesized:
118 # First, try parenthesized (don't need speculation because it either
119 # parses or doesn't modify state).
120 parenthesized_whitespace = _parse_parenthesized_whitespace(config, state)
121 if parenthesized_whitespace is not None:
122 return parenthesized_whitespace
123 # Now, just parse and return a simple whitespace
124 return parse_simple_whitespace(config, state)
127# END PARSER ENTRYPOINTS
128# BEGIN PARSER INTERNAL PRODUCTIONS
131def _parse_empty_line(
132 config: BaseWhitespaceParserConfig,
133 state: State,
134 *,
135 override_absolute_indent: Optional[str] = None,
136) -> Optional[EmptyLine]:
137 # begin speculative parsing
138 speculative_state = State(
139 state.line, state.column, state.absolute_indent, state.is_parenthesized
140 )
141 try:
142 indent = _parse_indent(
143 config, speculative_state, override_absolute_indent=override_absolute_indent
144 )
145 except Exception:
146 # We aren't on a new line, speculative parsing failed
147 return None
148 whitespace = parse_simple_whitespace(config, speculative_state)
149 comment = _parse_comment(config, speculative_state)
150 newline = _parse_newline(config, speculative_state)
151 if newline is None:
152 # speculative parsing failed
153 return None
154 # speculative parsing succeeded
155 state.line = speculative_state.line
156 state.column = speculative_state.column
157 # don't need to copy absolute_indent/is_parenthesized because they don't change.
158 return EmptyLine(indent, whitespace, comment, newline)
161def _parse_indent(
162 config: BaseWhitespaceParserConfig,
163 state: State,
164 *,
165 override_absolute_indent: Optional[str] = None,
166) -> bool:
167 """
168 Returns True if indentation was found, otherwise False.
169 """
170 absolute_indent = (
171 override_absolute_indent
172 if override_absolute_indent is not None
173 else state.absolute_indent
174 )
175 line_str = config.lines[state.line - 1]
176 if state.column != 0:
177 if state.column == len(line_str) and state.line == len(config.lines):
178 # We're at EOF, treat this as a failed speculative parse
179 return False
180 raise Exception("Internal Error: Column should be 0 when parsing an indent.")
181 if line_str.startswith(absolute_indent, state.column):
182 state.column += len(absolute_indent)
183 return True
184 return False
187def _parse_comment(
188 config: BaseWhitespaceParserConfig, state: State
189) -> Optional[Comment]:
190 comment_match = COMMENT_RE.match(config.lines[state.line - 1], state.column)
191 if comment_match is None:
192 return None
193 comment = comment_match.group(0)
194 state.column += len(comment)
195 return Comment(comment)
198def _parse_newline(
199 config: BaseWhitespaceParserConfig, state: State
200) -> Optional[Newline]:
201 # begin speculative parsing
202 line_str = config.lines[state.line - 1]
203 newline_match = NEWLINE_RE.match(line_str, state.column)
204 if newline_match is not None:
205 # speculative parsing succeeded
206 newline_str = newline_match.group(0)
207 state.column += len(newline_str)
208 if state.column != len(line_str):
209 raise Exception("Internal Error: Found a newline, but it wasn't the EOL.")
210 if state.line < len(config.lines):
211 # this newline was the end of a line, and there's another line,
212 # therefore we should move to the next line
213 state.line += 1
214 state.column = 0
215 if newline_str == config.default_newline:
216 # Just inherit it from the Module instead of explicitly setting it.
217 return Newline()
218 else:
219 return Newline(newline_str)
220 else: # no newline was found, speculative parsing failed
221 return None
224def _parse_trailing_whitespace(
225 config: BaseWhitespaceParserConfig, state: State
226) -> Optional[TrailingWhitespace]:
227 # Begin speculative parsing
228 speculative_state = State(
229 state.line, state.column, state.absolute_indent, state.is_parenthesized
230 )
231 whitespace = parse_simple_whitespace(config, speculative_state)
232 comment = _parse_comment(config, speculative_state)
233 newline = _parse_newline(config, speculative_state)
234 if newline is None:
235 # Speculative parsing failed
236 return None
237 # Speculative parsing succeeded
238 state.line = speculative_state.line
239 state.column = speculative_state.column
240 # don't need to copy absolute_indent/is_parenthesized because they don't change.
241 return TrailingWhitespace(whitespace, comment, newline)
244def _parse_parenthesized_whitespace(
245 config: BaseWhitespaceParserConfig, state: State
246) -> Optional[ParenthesizedWhitespace]:
247 first_line = _parse_trailing_whitespace(config, state)
248 if first_line is None:
249 # Speculative parsing failed
250 return None
251 empty_lines = ()
252 while True:
253 empty_line = _parse_empty_line(config, state)
254 if empty_line is None:
255 # This isn't an empty line, so parse it below
256 break
257 empty_lines = empty_lines + (empty_line,)
258 indent = _parse_indent(config, state)
259 last_line = parse_simple_whitespace(config, state)
260 return ParenthesizedWhitespace(first_line, empty_lines, indent, last_line)