Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/astroid/builder.py: 50%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# Licensed under the LGPL: https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html
2# For details: https://github.com/pylint-dev/astroid/blob/main/LICENSE
3# Copyright (c) https://github.com/pylint-dev/astroid/blob/main/CONTRIBUTORS.txt
5"""The AstroidBuilder makes astroid from living object and / or from _ast.
7The builder is not thread safe and can't be used to parse different sources
8at the same time.
9"""
11from __future__ import annotations
13import ast
14import os
15import re
16import textwrap
17import types
18import warnings
19from collections.abc import Collection, Iterator, Sequence
20from io import TextIOWrapper
21from tokenize import detect_encoding
22from typing import TYPE_CHECKING, cast
24from astroid import bases, modutils, nodes, raw_building, rebuilder, util
25from astroid.const import PY312_PLUS, PY314_PLUS
26from astroid.exceptions import AstroidBuildingError, AstroidSyntaxError, InferenceError
28if TYPE_CHECKING:
29 from astroid.manager import AstroidManager
31# The name of the transient function that is used to
32# wrap expressions to be extracted when calling
33# extract_node.
34_TRANSIENT_FUNCTION = "__"
36# The comment used to select a statement to be extracted
37# when calling extract_node.
38_STATEMENT_SELECTOR = "#@"
40if PY312_PLUS:
41 warnings.filterwarnings("ignore", ".*invalid escape sequence", SyntaxWarning)
42if PY314_PLUS:
43 warnings.filterwarnings(
44 "ignore", "'(return|continue|break)' in a 'finally'", SyntaxWarning
45 )
48def open_source_file(filename: str) -> tuple[TextIOWrapper, str, str]:
49 # pylint: disable=consider-using-with
50 with open(filename, "rb") as byte_stream:
51 encoding = detect_encoding(byte_stream.readline)[0]
52 stream = open(filename, newline=None, encoding=encoding)
53 try:
54 data = stream.read()
55 except BaseException:
56 stream.close()
57 raise
58 return stream, encoding, data
61def _can_assign_attr(node: nodes.ClassDef, attrname: str | None) -> bool:
62 try:
63 slots = node.slots()
64 except NotImplementedError:
65 pass
66 else:
67 if slots and attrname not in {slot.value for slot in slots}:
68 return False
69 return node.qname() != "builtins.object"
72class AstroidBuilder(raw_building.InspectBuilder):
73 """Class for building an astroid tree from source code or from a live module.
75 The param *manager* specifies the manager class which should be used. The
76 param *apply_transforms* determines if the transforms should be
77 applied after the tree was built from source or from a live object,
78 by default being True.
79 """
81 def __init__(self, manager: AstroidManager, apply_transforms: bool = True) -> None:
82 super().__init__(manager)
83 self._apply_transforms = apply_transforms
84 if not raw_building.InspectBuilder.bootstrapped:
85 manager.bootstrap()
87 def module_build(
88 self, module: types.ModuleType, modname: str | None = None
89 ) -> nodes.Module:
90 """Build an astroid from a living module instance."""
91 node = None
92 path = getattr(module, "__file__", None)
93 loader = getattr(module, "__loader__", None)
94 # Prefer the loader to get the source rather than assuming we have a
95 # filesystem to read the source file from ourselves.
96 if loader:
97 modname = modname or module.__name__
98 source = loader.get_source(modname)
99 if source:
100 node = self.string_build(source, modname, path=path)
101 if node is None and path is not None:
102 path_, ext = os.path.splitext(modutils._path_from_filename(path))
103 if ext in {".py", ".pyc", ".pyo"} and os.path.exists(path_ + ".py"):
104 node = self.file_build(path_ + ".py", modname)
105 if node is None:
106 # this is a built-in module
107 # get a partial representation by introspection
108 node = self.inspect_build(module, modname=modname, path=path)
109 if self._apply_transforms:
110 # We have to handle transformation by ourselves since the
111 # rebuilder isn't called for builtin nodes
112 node = self._manager.visit_transforms(node)
113 assert isinstance(node, nodes.Module)
114 return node
116 def file_build(self, path: str, modname: str | None = None) -> nodes.Module:
117 """Build astroid from a source code file (i.e. from an ast).
119 *path* is expected to be a python source file
120 """
121 try:
122 stream, encoding, data = open_source_file(path)
123 except OSError as exc:
124 raise AstroidBuildingError(
125 "Unable to load file {path}:\n{error}",
126 modname=modname,
127 path=path,
128 error=exc,
129 ) from exc
130 except (SyntaxError, LookupError) as exc:
131 raise AstroidSyntaxError(
132 "Python 3 encoding specification error or unknown encoding:\n"
133 "{error}",
134 modname=modname,
135 path=path,
136 error=exc,
137 ) from exc
138 except UnicodeError as exc: # wrong encoding
139 # detect_encoding returns utf-8 if no encoding specified
140 raise AstroidBuildingError(
141 "Wrong or no encoding specified for {filename}.", filename=path
142 ) from exc
143 with stream:
144 # get module name if necessary
145 if modname is None:
146 try:
147 modname = ".".join(modutils.modpath_from_file(path))
148 except ImportError:
149 modname = os.path.splitext(os.path.basename(path))[0]
150 # build astroid representation
151 module, builder = self._data_build(data, modname, path)
152 return self._post_build(module, builder, encoding)
154 def string_build(
155 self,
156 data: str,
157 modname: str = "",
158 path: str | None = None,
159 is_stub: bool | None = None,
160 ) -> nodes.Module:
161 """Build astroid from source code string."""
162 module, builder = self._data_build(data, modname, path, is_stub=is_stub)
163 module.file_bytes = data.encode("utf-8")
164 return self._post_build(module, builder, "utf-8")
166 def _post_build(
167 self, module: nodes.Module, builder: rebuilder.TreeRebuilder, encoding: str
168 ) -> nodes.Module:
169 """Handles encoding and delayed nodes after a module has been built."""
170 module.file_encoding = encoding
171 self._manager.cache_module(module)
172 # post tree building steps after we stored the module in the cache:
173 for from_node, global_names in builder._import_from_nodes:
174 if from_node.modname == "__future__":
175 for symbol, _ in from_node.names:
176 module.future_imports.add(symbol)
177 self.add_from_names_to_locals(from_node, global_names)
178 # handle delayed assattr nodes
179 for delayed in builder._delayed_assattr:
180 self.delayed_assattr(delayed)
182 # Visit the transforms
183 if self._apply_transforms:
184 module = self._manager.visit_transforms(module)
185 return module
187 def _data_build(
188 self,
189 data: str,
190 modname: str,
191 path: str | None,
192 is_stub: bool | None = None,
193 ) -> tuple[nodes.Module, rebuilder.TreeRebuilder]:
194 """Build tree node from data and add some informations."""
195 try:
196 node = _parse_string(data, type_comments=True, modname=modname)
197 except (TypeError, ValueError, SyntaxError, MemoryError) as exc:
198 raise AstroidSyntaxError(
199 "Parsing Python code failed:\n{error}",
200 source=data,
201 modname=modname,
202 path=path,
203 error=exc,
204 ) from exc
206 if path is not None:
207 node_file = os.path.abspath(path)
208 else:
209 node_file = "<?>"
210 if modname.endswith(".__init__"):
211 modname = modname[:-9]
212 package = True
213 else:
214 package = (
215 path is not None
216 and os.path.splitext(os.path.basename(path))[0] == "__init__"
217 )
218 if is_stub is None:
219 is_stub = path is not None and path.endswith(".pyi")
220 builder = rebuilder.TreeRebuilder(self._manager, data, is_stub=is_stub)
221 module = builder.visit_module(node, modname, node_file, package)
222 return module, builder
224 def add_from_names_to_locals(
225 self, node: nodes.ImportFrom, global_name: Collection[str]
226 ) -> None:
227 """Store imported names to the locals.
229 Resort the locals if coming from a delayed node
230 """
232 def add_local(parent_or_root: nodes.NodeNG, name: str) -> None:
233 parent_or_root.set_local(name, node)
234 my_list = parent_or_root.scope().locals[name]
235 if TYPE_CHECKING:
236 my_list = cast(list[nodes.NodeNG], my_list)
237 my_list.sort(key=lambda n: n.fromlineno or 0)
239 assert node.parent # It should always default to the module
240 module = node.root()
241 for name, asname in node.names:
242 if name == "*":
243 try:
244 imported = node.do_import_module()
245 except AstroidBuildingError:
246 continue
247 for name in imported.public_names():
248 if name in global_name:
249 add_local(module, name)
250 else:
251 add_local(node.parent, name)
252 else:
253 name = asname or name
254 if name in global_name:
255 add_local(module, name)
256 else:
257 add_local(node.parent, name)
259 def delayed_assattr(self, node: nodes.AssignAttr) -> None:
260 """Visit an AssignAttr node.
262 This adds name to locals and handle members definition.
263 """
264 from astroid import objects # pylint: disable=import-outside-toplevel
266 try:
267 for inferred in node.expr.infer():
268 if isinstance(inferred, util.UninferableBase):
269 continue
270 try:
271 # We want a narrow check on the parent type, not all of its subclasses
272 if type(inferred) in {bases.Instance, objects.ExceptionInstance}:
273 inferred = inferred._proxied
274 iattrs = inferred.instance_attrs
275 if not _can_assign_attr(inferred, node.attrname):
276 continue
277 elif isinstance(inferred, bases.Instance):
278 # Const, Tuple or other containers that inherit from
279 # `Instance`
280 continue
281 elif isinstance(inferred, (bases.Proxy, util.UninferableBase)):
282 continue
283 elif inferred.is_function:
284 iattrs = inferred.instance_attrs
285 else:
286 iattrs = inferred.locals
287 except AttributeError:
288 # XXX log error
289 continue
290 values = iattrs.setdefault(node.attrname, [])
291 if node in values:
292 continue
293 values.append(node)
294 except InferenceError:
295 pass
298def build_namespace_package_module(name: str, path: Sequence[str]) -> nodes.Module:
299 module = nodes.Module(name, path=path, package=True)
300 module.postinit(body=[], doc_node=None)
301 return module
304def parse(
305 code: str,
306 module_name: str = "",
307 path: str | None = None,
308 apply_transforms: bool = True,
309 is_stub: bool | None = None,
310) -> nodes.Module:
311 """Parses a source string in order to obtain an astroid AST from it.
313 :param code: The code for the module.
314 :param module_name: The name for the module, if any
315 :param path: The path for the module
316 :param apply_transforms:
317 Apply the transforms for the give code. Use it if you
318 don't want the default transforms to be applied.
319 :param is_stub: Whether to treat the code as a ``.pyi`` stub.
320 When ``None``, auto-detected from *path*.
321 """
322 # pylint: disable-next=import-outside-toplevel
323 from astroid.manager import AstroidManager
325 code = textwrap.dedent(code)
326 builder = AstroidBuilder(AstroidManager(), apply_transforms=apply_transforms)
327 return builder.string_build(code, modname=module_name, path=path, is_stub=is_stub)
330def _extract_expressions(node: nodes.NodeNG) -> Iterator[nodes.NodeNG]:
331 """Find expressions in a call to _TRANSIENT_FUNCTION and extract them.
333 The function walks the AST recursively to search for expressions that
334 are wrapped into a call to _TRANSIENT_FUNCTION. If it finds such an
335 expression, it completely removes the function call node from the tree,
336 replacing it by the wrapped expression inside the parent.
338 :param node: An astroid node.
339 :yields: The sequence of wrapped expressions on the modified tree
340 expression can be found.
341 """
342 if (
343 isinstance(node, nodes.Call)
344 and isinstance(node.func, nodes.Name)
345 and node.func.name == _TRANSIENT_FUNCTION
346 and node.args
347 ):
348 real_expr = node.args[0]
349 assert node.parent
350 real_expr.parent = node.parent
351 # Search for node in all _astng_fields (the fields checked when
352 # get_children is called) of its parent. Some of those fields may
353 # be lists or tuples, in which case the elements need to be checked.
354 # When we find it, replace it by real_expr, so that the AST looks
355 # like no call to _TRANSIENT_FUNCTION ever took place.
356 for name in node.parent._astroid_fields:
357 child = getattr(node.parent, name)
358 if isinstance(child, list):
359 for idx, compound_child in enumerate(child):
360 if compound_child is node:
361 child[idx] = real_expr
362 elif child is node:
363 setattr(node.parent, name, real_expr)
364 yield real_expr
365 else:
366 for child in node.get_children():
367 yield from _extract_expressions(child)
370def _find_statement_by_line(node: nodes.NodeNG, line: int) -> nodes.NodeNG | None:
371 """Extracts the statement on a specific line from an AST.
373 If the line number of node matches line, it will be returned;
374 otherwise its children are iterated and the function is called
375 recursively.
377 :param node: An astroid node.
378 :param line: The line number of the statement to extract.
379 :returns: The statement on the line, or None if no statement for the line
380 can be found.
381 """
382 if isinstance(node, (nodes.ClassDef, nodes.FunctionDef, nodes.MatchCase)):
383 # This is an inaccuracy in the AST: the nodes that can be
384 # decorated do not carry explicit information on which line
385 # the actual definition (class/def), but .fromline seems to
386 # be close enough.
387 node_line = node.fromlineno
388 else:
389 node_line = node.lineno
391 if node_line == line:
392 return node
394 for child in node.get_children():
395 result = _find_statement_by_line(child, line)
396 if result:
397 return result
399 return None
402def extract_node(code: str, module_name: str = "") -> nodes.NodeNG | list[nodes.NodeNG]:
403 """Parses some Python code as a module and extracts a designated AST node.
405 Statements:
406 To extract one or more statement nodes, append #@ to the end of the line
408 Examples::
410 def x():
411 def y():
412 return 1 #@
414 The return statement will be extracted.
416 ::
418 class X(object):
419 def meth(self): #@
420 pass
422 The function object 'meth' will be extracted.
424 Expressions:
425 To extract arbitrary expressions, surround them with the fake
426 function call __(...). After parsing, the surrounded expression
427 will be returned and the whole AST (accessible via the returned
428 node's parent attribute) will look like the function call was
429 never there in the first place.
431 Examples::
433 a = __(1)
435 The const node will be extracted.
437 ::
439 def x(d=__(foo.bar)): pass
441 The node containing the default argument will be extracted.
443 ::
445 def foo(a, b):
446 return 0 < __(len(a)) < b
448 The node containing the function call 'len' will be extracted.
450 If no statements or expressions are selected, the last toplevel
451 statement will be returned.
453 If the selected statement is a discard statement, (i.e. an expression
454 turned into a statement), the wrapped expression is returned instead.
456 For convenience, singleton lists are unpacked.
458 :param code: A piece of Python code that is parsed as
459 a module. Will be passed through textwrap.dedent first.
460 :param module_name: The name of the module.
461 :returns: The designated node from the parse tree, or a list of nodes.
462 """
464 def _extract(node: nodes.NodeNG | None) -> nodes.NodeNG | None:
465 if isinstance(node, nodes.Expr):
466 return node.value
468 return node
470 requested_lines: list[int] = []
471 for idx, line in enumerate(code.splitlines()):
472 if line.strip().endswith(_STATEMENT_SELECTOR):
473 requested_lines.append(idx + 1)
475 tree = parse(code, module_name=module_name)
476 if not tree.body:
477 raise ValueError("Empty tree, cannot extract from it")
479 extracted: list[nodes.NodeNG | None] = []
480 if requested_lines:
481 extracted = [_find_statement_by_line(tree, line) for line in requested_lines]
483 # Modifies the tree.
484 extracted.extend(_extract_expressions(tree))
486 if not extracted:
487 extracted.append(tree.body[-1])
489 extracted = [_extract(node) for node in extracted]
490 extracted_without_none = [node for node in extracted if node is not None]
491 if len(extracted_without_none) == 1:
492 return extracted_without_none[0]
493 return extracted_without_none
496def _extract_single_node(code: str, module_name: str = "") -> nodes.NodeNG:
497 """Call extract_node while making sure that only one value is returned."""
498 ret = extract_node(code, module_name)
499 if isinstance(ret, list):
500 return ret[0]
501 return ret
504def _parse_string(
505 data: str, type_comments: bool = True, modname: str | None = None
506) -> ast.Module:
507 try:
508 parsed = ast.parse(
509 data + "\n", filename=modname or "<unknown>", type_comments=type_comments
510 )
511 except SyntaxError as exc:
512 # If the type annotations are misplaced for some reason, we do not want
513 # to fail the entire parsing of the file, so we need to retry the
514 # parsing without type comment support. We use a heuristic for
515 # determining if the error is due to type annotations.
516 type_annot_related = re.search(r"#\s+type:", exc.text or "")
517 if not (type_annot_related and type_comments):
518 raise
520 parsed = ast.parse(data + "\n", type_comments=False)
521 return parsed