Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/libcst/_parser/grammar.py: 33%
94 statements
« prev ^ index » next coverage.py v7.3.1, created at 2023-09-25 06:43 +0000
1# Copyright (c) Meta Platforms, Inc. and affiliates.
2#
3# This source code is licensed under the MIT license found in the
4# LICENSE file in the root directory of this source tree.
6import re
7from functools import lru_cache
8from typing import FrozenSet, Iterator, Mapping, Optional, Tuple, Union
10from libcst._parser.conversions.expression import (
11 convert_arg_assign_comp_for,
12 convert_arglist,
13 convert_argument,
14 convert_atom,
15 convert_atom_basic,
16 convert_atom_curlybraces,
17 convert_atom_ellipses,
18 convert_atom_expr,
19 convert_atom_expr_await,
20 convert_atom_expr_trailer,
21 convert_atom_parens,
22 convert_atom_squarebrackets,
23 convert_atom_string,
24 convert_binop,
25 convert_boolop,
26 convert_comp_for,
27 convert_comp_if,
28 convert_comp_op,
29 convert_comparison,
30 convert_dictorsetmaker,
31 convert_expression_input,
32 convert_factor,
33 convert_fstring,
34 convert_fstring_content,
35 convert_fstring_conversion,
36 convert_fstring_equality,
37 convert_fstring_expr,
38 convert_fstring_format_spec,
39 convert_lambda,
40 convert_namedexpr_test,
41 convert_not_test,
42 convert_power,
43 convert_sliceop,
44 convert_star_arg,
45 convert_star_expr,
46 convert_subscript,
47 convert_subscriptlist,
48 convert_sync_comp_for,
49 convert_test,
50 convert_test_nocond,
51 convert_test_or_expr_list,
52 convert_testlist_comp_list,
53 convert_testlist_comp_tuple,
54 convert_trailer,
55 convert_trailer_arglist,
56 convert_trailer_attribute,
57 convert_trailer_subscriptlist,
58 convert_yield_arg,
59 convert_yield_expr,
60)
61from libcst._parser.conversions.module import convert_file_input
62from libcst._parser.conversions.params import (
63 convert_argslist,
64 convert_fpdef,
65 convert_fpdef_assign,
66 convert_fpdef_slash,
67 convert_fpdef_star,
68 convert_fpdef_starstar,
69)
70from libcst._parser.conversions.statement import (
71 convert_annassign,
72 convert_assert_stmt,
73 convert_assign,
74 convert_asyncable_funcdef,
75 convert_asyncable_stmt,
76 convert_augassign,
77 convert_break_stmt,
78 convert_classdef,
79 convert_compound_stmt,
80 convert_continue_stmt,
81 convert_decorated,
82 convert_decorator,
83 convert_decorators,
84 convert_del_stmt,
85 convert_dotted_as_name,
86 convert_dotted_as_names,
87 convert_dotted_name,
88 convert_except_clause,
89 convert_expr_stmt,
90 convert_for_stmt,
91 convert_funcdef,
92 convert_funcdef_annotation,
93 convert_global_stmt,
94 convert_if_stmt,
95 convert_if_stmt_elif,
96 convert_if_stmt_else,
97 convert_import_as_name,
98 convert_import_as_names,
99 convert_import_from,
100 convert_import_name,
101 convert_import_relative,
102 convert_import_stmt,
103 convert_indented_suite,
104 convert_nonlocal_stmt,
105 convert_parameters,
106 convert_pass_stmt,
107 convert_raise_stmt,
108 convert_return_stmt,
109 convert_simple_stmt_line,
110 convert_simple_stmt_partial,
111 convert_simple_stmt_suite,
112 convert_small_stmt,
113 convert_stmt,
114 convert_stmt_input,
115 convert_suite,
116 convert_try_stmt,
117 convert_while_stmt,
118 convert_with_item,
119 convert_with_stmt,
120)
121from libcst._parser.conversions.terminals import (
122 convert_ASYNC,
123 convert_AWAIT,
124 convert_DEDENT,
125 convert_ENDMARKER,
126 convert_FSTRING_END,
127 convert_FSTRING_START,
128 convert_FSTRING_STRING,
129 convert_INDENT,
130 convert_NAME,
131 convert_NEWLINE,
132 convert_NUMBER,
133 convert_OP,
134 convert_STRING,
135)
136from libcst._parser.parso.pgen2.generator import generate_grammar, Grammar
137from libcst._parser.parso.python.token import PythonTokenTypes, TokenType
138from libcst._parser.parso.utils import parse_version_string, PythonVersionInfo
139from libcst._parser.production_decorator import get_productions
140from libcst._parser.types.config import AutoConfig
141from libcst._parser.types.conversions import NonterminalConversion, TerminalConversion
142from libcst._parser.types.production import Production
144# Keep this sorted alphabetically
145_TERMINAL_CONVERSIONS_SEQUENCE: Tuple[TerminalConversion, ...] = (
146 convert_DEDENT,
147 convert_ENDMARKER,
148 convert_INDENT,
149 convert_NAME,
150 convert_NEWLINE,
151 convert_NUMBER,
152 convert_OP,
153 convert_STRING,
154 convert_FSTRING_START,
155 convert_FSTRING_END,
156 convert_FSTRING_STRING,
157 convert_ASYNC,
158 convert_AWAIT,
159)
161# Try to match the order of https://docs.python.org/3/reference/grammar.html
162_NONTERMINAL_CONVERSIONS_SEQUENCE: Tuple[NonterminalConversion, ...] = (
163 convert_file_input,
164 convert_stmt_input, # roughly equivalent to single_input
165 convert_expression_input, # roughly equivalent to eval_input
166 convert_stmt,
167 convert_simple_stmt_partial,
168 convert_simple_stmt_line,
169 convert_simple_stmt_suite,
170 convert_small_stmt,
171 convert_expr_stmt,
172 convert_annassign,
173 convert_augassign,
174 convert_assign,
175 convert_pass_stmt,
176 convert_continue_stmt,
177 convert_break_stmt,
178 convert_del_stmt,
179 convert_import_stmt,
180 convert_import_name,
181 convert_import_relative,
182 convert_import_from,
183 convert_import_as_name,
184 convert_dotted_as_name,
185 convert_import_as_names,
186 convert_dotted_as_names,
187 convert_dotted_name,
188 convert_return_stmt,
189 convert_raise_stmt,
190 convert_global_stmt,
191 convert_nonlocal_stmt,
192 convert_assert_stmt,
193 convert_compound_stmt,
194 convert_if_stmt,
195 convert_if_stmt_elif,
196 convert_if_stmt_else,
197 convert_while_stmt,
198 convert_for_stmt,
199 convert_try_stmt,
200 convert_except_clause,
201 convert_with_stmt,
202 convert_with_item,
203 convert_asyncable_funcdef,
204 convert_funcdef,
205 convert_classdef,
206 convert_decorator,
207 convert_decorators,
208 convert_decorated,
209 convert_asyncable_stmt,
210 convert_parameters,
211 convert_argslist,
212 convert_fpdef_slash,
213 convert_fpdef_star,
214 convert_fpdef_starstar,
215 convert_fpdef_assign,
216 convert_fpdef,
217 convert_funcdef_annotation,
218 convert_suite,
219 convert_indented_suite,
220 convert_namedexpr_test,
221 convert_test,
222 convert_test_nocond,
223 convert_lambda,
224 convert_boolop,
225 convert_not_test,
226 convert_comparison,
227 convert_comp_op,
228 convert_star_expr,
229 convert_binop,
230 convert_factor,
231 convert_power,
232 convert_atom_expr,
233 convert_atom_expr_await,
234 convert_atom_expr_trailer,
235 convert_trailer,
236 convert_trailer_attribute,
237 convert_trailer_subscriptlist,
238 convert_subscriptlist,
239 convert_subscript,
240 convert_sliceop,
241 convert_trailer_arglist,
242 convert_atom,
243 convert_atom_basic,
244 convert_atom_parens,
245 convert_atom_squarebrackets,
246 convert_atom_curlybraces,
247 convert_atom_string,
248 convert_fstring,
249 convert_fstring_content,
250 convert_fstring_conversion,
251 convert_fstring_equality,
252 convert_fstring_expr,
253 convert_fstring_format_spec,
254 convert_atom_ellipses,
255 convert_testlist_comp_tuple,
256 convert_testlist_comp_list,
257 convert_test_or_expr_list,
258 convert_dictorsetmaker,
259 convert_arglist,
260 convert_argument,
261 convert_arg_assign_comp_for,
262 convert_star_arg,
263 convert_sync_comp_for,
264 convert_comp_for,
265 convert_comp_if,
266 convert_yield_expr,
267 convert_yield_arg,
268)
271def get_grammar_str(version: PythonVersionInfo, future_imports: FrozenSet[str]) -> str:
272 """
273 Returns an BNF-like grammar text that `parso.pgen2.generator.generate_grammar` can
274 handle.
276 While you should generally use `get_grammar` instead, this can be useful for
277 debugging the grammar.
278 """
279 lines = []
280 for p in get_nonterminal_productions(version, future_imports):
281 lines.append(str(p))
282 return "\n".join(lines) + "\n"
285# TODO: We should probably provide an on-disk cache like parso and lib2to3 do. Because
286# of how we're defining our grammar, efficient cache invalidation is harder, though not
287# impossible.
288@lru_cache()
289def get_grammar(
290 version: PythonVersionInfo,
291 future_imports: Union[FrozenSet[str], AutoConfig],
292) -> "Grammar[TokenType]":
293 if isinstance(future_imports, AutoConfig):
294 # For easier testing, if not provided assume no __future__ imports
295 future_imports = frozenset(())
296 return generate_grammar(get_grammar_str(version, future_imports), PythonTokenTypes)
299@lru_cache()
300def get_terminal_conversions() -> Mapping[str, TerminalConversion]:
301 """
302 Returns a mapping from terminal type name to the conversion function that should be
303 called by the parser.
304 """
305 return {
306 # pyre-fixme[16]: Optional type has no attribute `group`.
307 re.match("convert_(.*)", fn.__name__).group(1): fn
308 for fn in _TERMINAL_CONVERSIONS_SEQUENCE
309 }
312@lru_cache()
313def validate_grammar() -> None:
314 for fn in _NONTERMINAL_CONVERSIONS_SEQUENCE:
315 fn_productions = get_productions(fn)
316 if all(p.name == fn_productions[0].name for p in fn_productions):
317 # all the production names are the same, ensure that the `convert_` function
318 # is named correctly
319 production_name = fn_productions[0].name
320 expected_name = f"convert_{production_name}"
321 if fn.__name__ != expected_name:
322 raise Exception(
323 f"The conversion function for '{production_name}' "
324 + f"must be called '{expected_name}', not '{fn.__name__}'."
325 )
328def _get_version_comparison(version: str) -> Tuple[str, PythonVersionInfo]:
329 if version[:2] in (">=", "<=", "==", "!="):
330 return (version[:2], parse_version_string(version[2:].strip()))
331 if version[:1] in (">", "<"):
332 return (version[:1], parse_version_string(version[1:].strip()))
333 raise Exception(f"Invalid version comparison specifier '{version}'")
336def _compare_versions(
337 requested_version: PythonVersionInfo,
338 actual_version: PythonVersionInfo,
339 comparison: str,
340) -> bool:
341 if comparison == ">=":
342 return actual_version >= requested_version
343 if comparison == "<=":
344 return actual_version <= requested_version
345 if comparison == "==":
346 return actual_version == requested_version
347 if comparison == "!=":
348 return actual_version != requested_version
349 if comparison == ">":
350 return actual_version > requested_version
351 if comparison == "<":
352 return actual_version < requested_version
353 raise Exception(f"Invalid version comparison specifier '{comparison}'")
356def _should_include(
357 requested_version: Optional[str], actual_version: PythonVersionInfo
358) -> bool:
359 if requested_version is None:
360 return True
361 for version in requested_version.split(","):
362 comparison, parsed_version = _get_version_comparison(version.strip())
363 if not _compare_versions(parsed_version, actual_version, comparison):
364 return False
365 return True
368def _should_include_future(
369 future: Optional[str],
370 future_imports: FrozenSet[str],
371) -> bool:
372 if future is None:
373 return True
374 if future[:1] == "!":
375 return future[1:] not in future_imports
376 return future in future_imports
379def get_nonterminal_productions(
380 version: PythonVersionInfo, future_imports: FrozenSet[str]
381) -> Iterator[Production]:
382 for conversion in _NONTERMINAL_CONVERSIONS_SEQUENCE:
383 for production in get_productions(conversion):
384 if not _should_include(production.version, version):
385 continue
386 if not _should_include_future(production.future, future_imports):
387 continue
388 yield production
391@lru_cache()
392def get_nonterminal_conversions(
393 version: PythonVersionInfo,
394 future_imports: FrozenSet[str],
395) -> Mapping[str, NonterminalConversion]:
396 """
397 Returns a mapping from nonterminal production name to the conversion function that
398 should be called by the parser.
399 """
400 conversions = {}
401 for fn in _NONTERMINAL_CONVERSIONS_SEQUENCE:
402 for fn_production in get_productions(fn):
403 if not _should_include(fn_production.version, version):
404 continue
405 if not _should_include_future(fn_production.future, future_imports):
406 continue
407 if fn_production.name in conversions:
408 raise Exception(
409 f"Found duplicate '{fn_production.name}' production in grammar"
410 )
411 conversions[fn_production.name] = fn
413 return conversions