Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/astroid/builder.py: 16%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# Licensed under the LGPL: https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html
2# For details: https://github.com/pylint-dev/astroid/blob/main/LICENSE
3# Copyright (c) https://github.com/pylint-dev/astroid/blob/main/CONTRIBUTORS.txt
5"""The AstroidBuilder makes astroid from living object and / or from _ast.
7The builder is not thread safe and can't be used to parse different sources
8at the same time.
9"""
11from __future__ import annotations
13import ast
14import os
15import re
16import textwrap
17import types
18import warnings
19from collections.abc import Collection, Iterator, Sequence
20from io import TextIOWrapper
21from tokenize import detect_encoding
22from typing import TYPE_CHECKING, cast
24from astroid import bases, modutils, nodes, raw_building, rebuilder, util
25from astroid._ast import ParserModule, get_parser_module
26from astroid.const import PY312_PLUS, PY314_PLUS
27from astroid.exceptions import AstroidBuildingError, AstroidSyntaxError, InferenceError
29if TYPE_CHECKING:
30 from astroid.manager import AstroidManager
32# The name of the transient function that is used to
33# wrap expressions to be extracted when calling
34# extract_node.
35_TRANSIENT_FUNCTION = "__"
37# The comment used to select a statement to be extracted
38# when calling extract_node.
39_STATEMENT_SELECTOR = "#@"
41if PY312_PLUS:
42 warnings.filterwarnings("ignore", ".*invalid escape sequence", SyntaxWarning)
43if PY314_PLUS:
44 warnings.filterwarnings(
45 "ignore", "'(return|continue|break)' in a 'finally'", SyntaxWarning
46 )
49def open_source_file(filename: str) -> tuple[TextIOWrapper, str, str]:
50 # pylint: disable=consider-using-with
51 with open(filename, "rb") as byte_stream:
52 encoding = detect_encoding(byte_stream.readline)[0]
53 stream = open(filename, newline=None, encoding=encoding)
54 data = stream.read()
55 return stream, encoding, data
58def _can_assign_attr(node: nodes.ClassDef, attrname: str | None) -> bool:
59 try:
60 slots = node.slots()
61 except NotImplementedError:
62 pass
63 else:
64 if slots and attrname not in {slot.value for slot in slots}:
65 return False
66 return node.qname() != "builtins.object"
69class AstroidBuilder(raw_building.InspectBuilder):
70 """Class for building an astroid tree from source code or from a live module.
72 The param *manager* specifies the manager class which should be used. The
73 param *apply_transforms* determines if the transforms should be
74 applied after the tree was built from source or from a live object,
75 by default being True.
76 """
78 def __init__(self, manager: AstroidManager, apply_transforms: bool = True) -> None:
79 super().__init__(manager)
80 self._apply_transforms = apply_transforms
81 if not raw_building.InspectBuilder.bootstrapped:
82 manager.bootstrap()
84 def module_build(
85 self, module: types.ModuleType, modname: str | None = None
86 ) -> nodes.Module:
87 """Build an astroid from a living module instance."""
88 node = None
89 path = getattr(module, "__file__", None)
90 loader = getattr(module, "__loader__", None)
91 # Prefer the loader to get the source rather than assuming we have a
92 # filesystem to read the source file from ourselves.
93 if loader:
94 modname = modname or module.__name__
95 source = loader.get_source(modname)
96 if source:
97 node = self.string_build(source, modname, path=path)
98 if node is None and path is not None:
99 path_, ext = os.path.splitext(modutils._path_from_filename(path))
100 if ext in {".py", ".pyc", ".pyo"} and os.path.exists(path_ + ".py"):
101 node = self.file_build(path_ + ".py", modname)
102 if node is None:
103 # this is a built-in module
104 # get a partial representation by introspection
105 node = self.inspect_build(module, modname=modname, path=path)
106 if self._apply_transforms:
107 # We have to handle transformation by ourselves since the
108 # rebuilder isn't called for builtin nodes
109 node = self._manager.visit_transforms(node)
110 assert isinstance(node, nodes.Module)
111 return node
113 def file_build(self, path: str, modname: str | None = None) -> nodes.Module:
114 """Build astroid from a source code file (i.e. from an ast).
116 *path* is expected to be a python source file
117 """
118 try:
119 stream, encoding, data = open_source_file(path)
120 except OSError as exc:
121 raise AstroidBuildingError(
122 "Unable to load file {path}:\n{error}",
123 modname=modname,
124 path=path,
125 error=exc,
126 ) from exc
127 except (SyntaxError, LookupError) as exc:
128 raise AstroidSyntaxError(
129 "Python 3 encoding specification error or unknown encoding:\n"
130 "{error}",
131 modname=modname,
132 path=path,
133 error=exc,
134 ) from exc
135 except UnicodeError as exc: # wrong encoding
136 # detect_encoding returns utf-8 if no encoding specified
137 raise AstroidBuildingError(
138 "Wrong or no encoding specified for {filename}.", filename=path
139 ) from exc
140 with stream:
141 # get module name if necessary
142 if modname is None:
143 try:
144 modname = ".".join(modutils.modpath_from_file(path))
145 except ImportError:
146 modname = os.path.splitext(os.path.basename(path))[0]
147 # build astroid representation
148 module, builder = self._data_build(data, modname, path)
149 return self._post_build(module, builder, encoding)
151 def string_build(
152 self, data: str, modname: str = "", path: str | None = None
153 ) -> nodes.Module:
154 """Build astroid from source code string."""
155 module, builder = self._data_build(data, modname, path)
156 module.file_bytes = data.encode("utf-8")
157 return self._post_build(module, builder, "utf-8")
159 def _post_build(
160 self, module: nodes.Module, builder: rebuilder.TreeRebuilder, encoding: str
161 ) -> nodes.Module:
162 """Handles encoding and delayed nodes after a module has been built."""
163 module.file_encoding = encoding
164 self._manager.cache_module(module)
165 # post tree building steps after we stored the module in the cache:
166 for from_node, global_names in builder._import_from_nodes:
167 if from_node.modname == "__future__":
168 for symbol, _ in from_node.names:
169 module.future_imports.add(symbol)
170 self.add_from_names_to_locals(from_node, global_names)
171 # handle delayed assattr nodes
172 for delayed in builder._delayed_assattr:
173 self.delayed_assattr(delayed)
175 # Visit the transforms
176 if self._apply_transforms:
177 module = self._manager.visit_transforms(module)
178 return module
180 def _data_build(
181 self, data: str, modname: str, path: str | None
182 ) -> tuple[nodes.Module, rebuilder.TreeRebuilder]:
183 """Build tree node from data and add some informations."""
184 try:
185 node, parser_module = _parse_string(
186 data, type_comments=True, modname=modname
187 )
188 except (TypeError, ValueError, SyntaxError, MemoryError) as exc:
189 raise AstroidSyntaxError(
190 "Parsing Python code failed:\n{error}",
191 source=data,
192 modname=modname,
193 path=path,
194 error=exc,
195 ) from exc
197 if path is not None:
198 node_file = os.path.abspath(path)
199 else:
200 node_file = "<?>"
201 if modname.endswith(".__init__"):
202 modname = modname[:-9]
203 package = True
204 else:
205 package = (
206 path is not None
207 and os.path.splitext(os.path.basename(path))[0] == "__init__"
208 )
209 builder = rebuilder.TreeRebuilder(self._manager, parser_module, data)
210 module = builder.visit_module(node, modname, node_file, package)
211 return module, builder
213 def add_from_names_to_locals(
214 self, node: nodes.ImportFrom, global_name: Collection[str]
215 ) -> None:
216 """Store imported names to the locals.
218 Resort the locals if coming from a delayed node
219 """
221 def add_local(parent_or_root: nodes.NodeNG, name: str) -> None:
222 parent_or_root.set_local(name, node)
223 my_list = parent_or_root.scope().locals[name]
224 if TYPE_CHECKING:
225 my_list = cast(list[nodes.NodeNG], my_list)
226 my_list.sort(key=lambda n: n.fromlineno or 0)
228 assert node.parent # It should always default to the module
229 module = node.root()
230 for name, asname in node.names:
231 if name == "*":
232 try:
233 imported = node.do_import_module()
234 except AstroidBuildingError:
235 continue
236 for name in imported.public_names():
237 if name in global_name:
238 add_local(module, name)
239 else:
240 add_local(node.parent, name)
241 else:
242 name = asname or name
243 if name in global_name:
244 add_local(module, name)
245 else:
246 add_local(node.parent, name)
248 def delayed_assattr(self, node: nodes.AssignAttr) -> None:
249 """Visit an AssignAttr node.
251 This adds name to locals and handle members definition.
252 """
253 from astroid import objects # pylint: disable=import-outside-toplevel
255 try:
256 for inferred in node.expr.infer():
257 if isinstance(inferred, util.UninferableBase):
258 continue
259 try:
260 # We want a narrow check on the parent type, not all of its subclasses
261 if type(inferred) in {bases.Instance, objects.ExceptionInstance}:
262 inferred = inferred._proxied
263 iattrs = inferred.instance_attrs
264 if not _can_assign_attr(inferred, node.attrname):
265 continue
266 elif isinstance(inferred, bases.Instance):
267 # Const, Tuple or other containers that inherit from
268 # `Instance`
269 continue
270 elif isinstance(inferred, (bases.Proxy, util.UninferableBase)):
271 continue
272 elif inferred.is_function:
273 iattrs = inferred.instance_attrs
274 else:
275 iattrs = inferred.locals
276 except AttributeError:
277 # XXX log error
278 continue
279 values = iattrs.setdefault(node.attrname, [])
280 if node in values:
281 continue
282 values.append(node)
283 except InferenceError:
284 pass
287def build_namespace_package_module(name: str, path: Sequence[str]) -> nodes.Module:
288 module = nodes.Module(name, path=path, package=True)
289 module.postinit(body=[], doc_node=None)
290 return module
293def parse(
294 code: str,
295 module_name: str = "",
296 path: str | None = None,
297 apply_transforms: bool = True,
298) -> nodes.Module:
299 """Parses a source string in order to obtain an astroid AST from it.
301 :param str code: The code for the module.
302 :param str module_name: The name for the module, if any
303 :param str path: The path for the module
304 :param bool apply_transforms:
305 Apply the transforms for the give code. Use it if you
306 don't want the default transforms to be applied.
307 """
308 # pylint: disable-next=import-outside-toplevel
309 from astroid.manager import AstroidManager
311 code = textwrap.dedent(code)
312 builder = AstroidBuilder(AstroidManager(), apply_transforms=apply_transforms)
313 return builder.string_build(code, modname=module_name, path=path)
316def _extract_expressions(node: nodes.NodeNG) -> Iterator[nodes.NodeNG]:
317 """Find expressions in a call to _TRANSIENT_FUNCTION and extract them.
319 The function walks the AST recursively to search for expressions that
320 are wrapped into a call to _TRANSIENT_FUNCTION. If it finds such an
321 expression, it completely removes the function call node from the tree,
322 replacing it by the wrapped expression inside the parent.
324 :param node: An astroid node.
325 :type node: astroid.bases.NodeNG
326 :yields: The sequence of wrapped expressions on the modified tree
327 expression can be found.
328 """
329 if (
330 isinstance(node, nodes.Call)
331 and isinstance(node.func, nodes.Name)
332 and node.func.name == _TRANSIENT_FUNCTION
333 and node.args
334 ):
335 real_expr = node.args[0]
336 assert node.parent
337 real_expr.parent = node.parent
338 # Search for node in all _astng_fields (the fields checked when
339 # get_children is called) of its parent. Some of those fields may
340 # be lists or tuples, in which case the elements need to be checked.
341 # When we find it, replace it by real_expr, so that the AST looks
342 # like no call to _TRANSIENT_FUNCTION ever took place.
343 for name in node.parent._astroid_fields:
344 child = getattr(node.parent, name)
345 if isinstance(child, list):
346 for idx, compound_child in enumerate(child):
347 if compound_child is node:
348 child[idx] = real_expr
349 elif child is node:
350 setattr(node.parent, name, real_expr)
351 yield real_expr
352 else:
353 for child in node.get_children():
354 yield from _extract_expressions(child)
357def _find_statement_by_line(node: nodes.NodeNG, line: int) -> nodes.NodeNG | None:
358 """Extracts the statement on a specific line from an AST.
360 If the line number of node matches line, it will be returned;
361 otherwise its children are iterated and the function is called
362 recursively.
364 :param node: An astroid node.
365 :type node: astroid.bases.NodeNG
366 :param line: The line number of the statement to extract.
367 :type line: int
368 :returns: The statement on the line, or None if no statement for the line
369 can be found.
370 :rtype: astroid.bases.NodeNG or None
371 """
372 if isinstance(node, (nodes.ClassDef, nodes.FunctionDef, nodes.MatchCase)):
373 # This is an inaccuracy in the AST: the nodes that can be
374 # decorated do not carry explicit information on which line
375 # the actual definition (class/def), but .fromline seems to
376 # be close enough.
377 node_line = node.fromlineno
378 else:
379 node_line = node.lineno
381 if node_line == line:
382 return node
384 for child in node.get_children():
385 result = _find_statement_by_line(child, line)
386 if result:
387 return result
389 return None
392def extract_node(code: str, module_name: str = "") -> nodes.NodeNG | list[nodes.NodeNG]:
393 """Parses some Python code as a module and extracts a designated AST node.
395 Statements:
396 To extract one or more statement nodes, append #@ to the end of the line
398 Examples:
399 >>> def x():
400 >>> def y():
401 >>> return 1 #@
403 The return statement will be extracted.
405 >>> class X(object):
406 >>> def meth(self): #@
407 >>> pass
409 The function object 'meth' will be extracted.
411 Expressions:
412 To extract arbitrary expressions, surround them with the fake
413 function call __(...). After parsing, the surrounded expression
414 will be returned and the whole AST (accessible via the returned
415 node's parent attribute) will look like the function call was
416 never there in the first place.
418 Examples:
419 >>> a = __(1)
421 The const node will be extracted.
423 >>> def x(d=__(foo.bar)): pass
425 The node containing the default argument will be extracted.
427 >>> def foo(a, b):
428 >>> return 0 < __(len(a)) < b
430 The node containing the function call 'len' will be extracted.
432 If no statements or expressions are selected, the last toplevel
433 statement will be returned.
435 If the selected statement is a discard statement, (i.e. an expression
436 turned into a statement), the wrapped expression is returned instead.
438 For convenience, singleton lists are unpacked.
440 :param str code: A piece of Python code that is parsed as
441 a module. Will be passed through textwrap.dedent first.
442 :param str module_name: The name of the module.
443 :returns: The designated node from the parse tree, or a list of nodes.
444 """
446 def _extract(node: nodes.NodeNG | None) -> nodes.NodeNG | None:
447 if isinstance(node, nodes.Expr):
448 return node.value
450 return node
452 requested_lines: list[int] = []
453 for idx, line in enumerate(code.splitlines()):
454 if line.strip().endswith(_STATEMENT_SELECTOR):
455 requested_lines.append(idx + 1)
457 tree = parse(code, module_name=module_name)
458 if not tree.body:
459 raise ValueError("Empty tree, cannot extract from it")
461 extracted: list[nodes.NodeNG | None] = []
462 if requested_lines:
463 extracted = [_find_statement_by_line(tree, line) for line in requested_lines]
465 # Modifies the tree.
466 extracted.extend(_extract_expressions(tree))
468 if not extracted:
469 extracted.append(tree.body[-1])
471 extracted = [_extract(node) for node in extracted]
472 extracted_without_none = [node for node in extracted if node is not None]
473 if len(extracted_without_none) == 1:
474 return extracted_without_none[0]
475 return extracted_without_none
478def _extract_single_node(code: str, module_name: str = "") -> nodes.NodeNG:
479 """Call extract_node while making sure that only one value is returned."""
480 ret = extract_node(code, module_name)
481 if isinstance(ret, list):
482 return ret[0]
483 return ret
486def _parse_string(
487 data: str, type_comments: bool = True, modname: str | None = None
488) -> tuple[ast.Module, ParserModule]:
489 parser_module = get_parser_module(type_comments=type_comments)
490 try:
491 parsed = parser_module.parse(
492 data + "\n", type_comments=type_comments, filename=modname
493 )
494 except SyntaxError as exc:
495 # If the type annotations are misplaced for some reason, we do not want
496 # to fail the entire parsing of the file, so we need to retry the
497 # parsing without type comment support. We use a heuristic for
498 # determining if the error is due to type annotations.
499 type_annot_related = re.search(r"#\s+type:", exc.text or "")
500 if not (type_annot_related and type_comments):
501 raise
503 parser_module = get_parser_module(type_comments=False)
504 parsed = parser_module.parse(data + "\n", type_comments=False)
505 return parsed, parser_module