Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.10/site-packages/astroid/builder.py: 44%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# Licensed under the LGPL: https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html
2# For details: https://github.com/pylint-dev/astroid/blob/main/LICENSE
3# Copyright (c) https://github.com/pylint-dev/astroid/blob/main/CONTRIBUTORS.txt
5"""The AstroidBuilder makes astroid from living object and / or from _ast.
7The builder is not thread safe and can't be used to parse different sources
8at the same time.
9"""
11from __future__ import annotations
13import ast
14import os
15import re
16import textwrap
17import types
18import warnings
19from collections.abc import Iterator, Sequence
20from io import TextIOWrapper
21from tokenize import detect_encoding
23from astroid import bases, modutils, nodes, raw_building, rebuilder, util
24from astroid._ast import ParserModule, get_parser_module
25from astroid.const import PY312_PLUS
26from astroid.exceptions import AstroidBuildingError, AstroidSyntaxError, InferenceError
27from astroid.manager import AstroidManager
29# The name of the transient function that is used to
30# wrap expressions to be extracted when calling
31# extract_node.
32_TRANSIENT_FUNCTION = "__"
34# The comment used to select a statement to be extracted
35# when calling extract_node.
36_STATEMENT_SELECTOR = "#@"
38if PY312_PLUS:
39 warnings.filterwarnings("ignore", "invalid escape sequence", SyntaxWarning)
42def open_source_file(filename: str) -> tuple[TextIOWrapper, str, str]:
43 # pylint: disable=consider-using-with
44 with open(filename, "rb") as byte_stream:
45 encoding = detect_encoding(byte_stream.readline)[0]
46 stream = open(filename, newline=None, encoding=encoding)
47 data = stream.read()
48 return stream, encoding, data
51def _can_assign_attr(node: nodes.ClassDef, attrname: str | None) -> bool:
52 try:
53 slots = node.slots()
54 except NotImplementedError:
55 pass
56 else:
57 if slots and attrname not in {slot.value for slot in slots}:
58 return False
59 return node.qname() != "builtins.object"
62class AstroidBuilder(raw_building.InspectBuilder):
63 """Class for building an astroid tree from source code or from a live module.
65 The param *manager* specifies the manager class which should be used.
66 If no manager is given, then the default one will be used. The
67 param *apply_transforms* determines if the transforms should be
68 applied after the tree was built from source or from a live object,
69 by default being True.
70 """
72 def __init__(
73 self, manager: AstroidManager | None = None, apply_transforms: bool = True
74 ) -> None:
75 super().__init__(manager)
76 self._apply_transforms = apply_transforms
77 if not raw_building.InspectBuilder.bootstrapped:
78 raw_building._astroid_bootstrapping()
80 def module_build(
81 self, module: types.ModuleType, modname: str | None = None
82 ) -> nodes.Module:
83 """Build an astroid from a living module instance."""
84 node = None
85 path = getattr(module, "__file__", None)
86 loader = getattr(module, "__loader__", None)
87 # Prefer the loader to get the source rather than assuming we have a
88 # filesystem to read the source file from ourselves.
89 if loader:
90 modname = modname or module.__name__
91 source = loader.get_source(modname)
92 if source:
93 node = self.string_build(source, modname, path=path)
94 if node is None and path is not None:
95 path_, ext = os.path.splitext(modutils._path_from_filename(path))
96 if ext in {".py", ".pyc", ".pyo"} and os.path.exists(path_ + ".py"):
97 node = self.file_build(path_ + ".py", modname)
98 if node is None:
99 # this is a built-in module
100 # get a partial representation by introspection
101 node = self.inspect_build(module, modname=modname, path=path)
102 if self._apply_transforms:
103 # We have to handle transformation by ourselves since the
104 # rebuilder isn't called for builtin nodes
105 node = self._manager.visit_transforms(node)
106 assert isinstance(node, nodes.Module)
107 return node
109 def file_build(self, path: str, modname: str | None = None) -> nodes.Module:
110 """Build astroid from a source code file (i.e. from an ast).
112 *path* is expected to be a python source file
113 """
114 try:
115 stream, encoding, data = open_source_file(path)
116 except OSError as exc:
117 raise AstroidBuildingError(
118 "Unable to load file {path}:\n{error}",
119 modname=modname,
120 path=path,
121 error=exc,
122 ) from exc
123 except (SyntaxError, LookupError) as exc:
124 raise AstroidSyntaxError(
125 "Python 3 encoding specification error or unknown encoding:\n"
126 "{error}",
127 modname=modname,
128 path=path,
129 error=exc,
130 ) from exc
131 except UnicodeError as exc: # wrong encoding
132 # detect_encoding returns utf-8 if no encoding specified
133 raise AstroidBuildingError(
134 "Wrong or no encoding specified for {filename}.", filename=path
135 ) from exc
136 with stream:
137 # get module name if necessary
138 if modname is None:
139 try:
140 modname = ".".join(modutils.modpath_from_file(path))
141 except ImportError:
142 modname = os.path.splitext(os.path.basename(path))[0]
143 # build astroid representation
144 module, builder = self._data_build(data, modname, path)
145 return self._post_build(module, builder, encoding)
147 def string_build(
148 self, data: str, modname: str = "", path: str | None = None
149 ) -> nodes.Module:
150 """Build astroid from source code string."""
151 module, builder = self._data_build(data, modname, path)
152 module.file_bytes = data.encode("utf-8")
153 return self._post_build(module, builder, "utf-8")
155 def _post_build(
156 self, module: nodes.Module, builder: rebuilder.TreeRebuilder, encoding: str
157 ) -> nodes.Module:
158 """Handles encoding and delayed nodes after a module has been built."""
159 module.file_encoding = encoding
160 self._manager.cache_module(module)
161 # post tree building steps after we stored the module in the cache:
162 for from_node in builder._import_from_nodes:
163 if from_node.modname == "__future__":
164 for symbol, _ in from_node.names:
165 module.future_imports.add(symbol)
166 self.add_from_names_to_locals(from_node)
167 # handle delayed assattr nodes
168 for delayed in builder._delayed_assattr:
169 self.delayed_assattr(delayed)
171 # Visit the transforms
172 if self._apply_transforms:
173 module = self._manager.visit_transforms(module)
174 return module
176 def _data_build(
177 self, data: str, modname: str, path: str | None
178 ) -> tuple[nodes.Module, rebuilder.TreeRebuilder]:
179 """Build tree node from data and add some informations."""
180 try:
181 node, parser_module = _parse_string(
182 data, type_comments=True, modname=modname
183 )
184 except (TypeError, ValueError, SyntaxError) as exc:
185 raise AstroidSyntaxError(
186 "Parsing Python code failed:\n{error}",
187 source=data,
188 modname=modname,
189 path=path,
190 error=exc,
191 ) from exc
193 if path is not None:
194 node_file = os.path.abspath(path)
195 else:
196 node_file = "<?>"
197 if modname.endswith(".__init__"):
198 modname = modname[:-9]
199 package = True
200 else:
201 package = (
202 path is not None
203 and os.path.splitext(os.path.basename(path))[0] == "__init__"
204 )
205 builder = rebuilder.TreeRebuilder(self._manager, parser_module, data)
206 module = builder.visit_module(node, modname, node_file, package)
207 return module, builder
209 def add_from_names_to_locals(self, node: nodes.ImportFrom) -> None:
210 """Store imported names to the locals.
212 Resort the locals if coming from a delayed node
213 """
215 def _key_func(node: nodes.NodeNG) -> int:
216 return node.fromlineno or 0
218 def sort_locals(my_list: list[nodes.NodeNG]) -> None:
219 my_list.sort(key=_key_func)
221 assert node.parent # It should always default to the module
222 for name, asname in node.names:
223 if name == "*":
224 try:
225 imported = node.do_import_module()
226 except AstroidBuildingError:
227 continue
228 for name in imported.public_names():
229 node.parent.set_local(name, node)
230 sort_locals(node.parent.scope().locals[name]) # type: ignore[arg-type]
231 else:
232 node.parent.set_local(asname or name, node)
233 sort_locals(node.parent.scope().locals[asname or name]) # type: ignore[arg-type]
235 def delayed_assattr(self, node: nodes.AssignAttr) -> None:
236 """Visit an AssignAttr node.
238 This adds name to locals and handle members definition.
239 """
240 from astroid import objects # pylint: disable=import-outside-toplevel
242 try:
243 for inferred in node.expr.infer():
244 if isinstance(inferred, util.UninferableBase):
245 continue
246 try:
247 # We want a narrow check on the parent type, not all of its subclasses
248 if type(inferred) in {bases.Instance, objects.ExceptionInstance}:
249 inferred = inferred._proxied
250 iattrs = inferred.instance_attrs
251 if not _can_assign_attr(inferred, node.attrname):
252 continue
253 elif isinstance(inferred, bases.Instance):
254 # Const, Tuple or other containers that inherit from
255 # `Instance`
256 continue
257 elif isinstance(inferred, (bases.Proxy, util.UninferableBase)):
258 continue
259 elif inferred.is_function:
260 iattrs = inferred.instance_attrs
261 else:
262 iattrs = inferred.locals
263 except AttributeError:
264 # XXX log error
265 continue
266 values = iattrs.setdefault(node.attrname, [])
267 if node in values:
268 continue
269 values.append(node)
270 except InferenceError:
271 pass
274def build_namespace_package_module(name: str, path: Sequence[str]) -> nodes.Module:
275 module = nodes.Module(name, path=path, package=True)
276 module.postinit(body=[], doc_node=None)
277 return module
280def parse(
281 code: str,
282 module_name: str = "",
283 path: str | None = None,
284 apply_transforms: bool = True,
285) -> nodes.Module:
286 """Parses a source string in order to obtain an astroid AST from it.
288 :param str code: The code for the module.
289 :param str module_name: The name for the module, if any
290 :param str path: The path for the module
291 :param bool apply_transforms:
292 Apply the transforms for the give code. Use it if you
293 don't want the default transforms to be applied.
294 """
295 code = textwrap.dedent(code)
296 builder = AstroidBuilder(
297 manager=AstroidManager(), apply_transforms=apply_transforms
298 )
299 return builder.string_build(code, modname=module_name, path=path)
302def _extract_expressions(node: nodes.NodeNG) -> Iterator[nodes.NodeNG]:
303 """Find expressions in a call to _TRANSIENT_FUNCTION and extract them.
305 The function walks the AST recursively to search for expressions that
306 are wrapped into a call to _TRANSIENT_FUNCTION. If it finds such an
307 expression, it completely removes the function call node from the tree,
308 replacing it by the wrapped expression inside the parent.
310 :param node: An astroid node.
311 :type node: astroid.bases.NodeNG
312 :yields: The sequence of wrapped expressions on the modified tree
313 expression can be found.
314 """
315 if (
316 isinstance(node, nodes.Call)
317 and isinstance(node.func, nodes.Name)
318 and node.func.name == _TRANSIENT_FUNCTION
319 ):
320 real_expr = node.args[0]
321 assert node.parent
322 real_expr.parent = node.parent
323 # Search for node in all _astng_fields (the fields checked when
324 # get_children is called) of its parent. Some of those fields may
325 # be lists or tuples, in which case the elements need to be checked.
326 # When we find it, replace it by real_expr, so that the AST looks
327 # like no call to _TRANSIENT_FUNCTION ever took place.
328 for name in node.parent._astroid_fields:
329 child = getattr(node.parent, name)
330 if isinstance(child, list):
331 for idx, compound_child in enumerate(child):
332 if compound_child is node:
333 child[idx] = real_expr
334 elif child is node:
335 setattr(node.parent, name, real_expr)
336 yield real_expr
337 else:
338 for child in node.get_children():
339 yield from _extract_expressions(child)
342def _find_statement_by_line(node: nodes.NodeNG, line: int) -> nodes.NodeNG | None:
343 """Extracts the statement on a specific line from an AST.
345 If the line number of node matches line, it will be returned;
346 otherwise its children are iterated and the function is called
347 recursively.
349 :param node: An astroid node.
350 :type node: astroid.bases.NodeNG
351 :param line: The line number of the statement to extract.
352 :type line: int
353 :returns: The statement on the line, or None if no statement for the line
354 can be found.
355 :rtype: astroid.bases.NodeNG or None
356 """
357 if isinstance(node, (nodes.ClassDef, nodes.FunctionDef, nodes.MatchCase)):
358 # This is an inaccuracy in the AST: the nodes that can be
359 # decorated do not carry explicit information on which line
360 # the actual definition (class/def), but .fromline seems to
361 # be close enough.
362 node_line = node.fromlineno
363 else:
364 node_line = node.lineno
366 if node_line == line:
367 return node
369 for child in node.get_children():
370 result = _find_statement_by_line(child, line)
371 if result:
372 return result
374 return None
377def extract_node(code: str, module_name: str = "") -> nodes.NodeNG | list[nodes.NodeNG]:
378 """Parses some Python code as a module and extracts a designated AST node.
380 Statements:
381 To extract one or more statement nodes, append #@ to the end of the line
383 Examples:
384 >>> def x():
385 >>> def y():
386 >>> return 1 #@
388 The return statement will be extracted.
390 >>> class X(object):
391 >>> def meth(self): #@
392 >>> pass
394 The function object 'meth' will be extracted.
396 Expressions:
397 To extract arbitrary expressions, surround them with the fake
398 function call __(...). After parsing, the surrounded expression
399 will be returned and the whole AST (accessible via the returned
400 node's parent attribute) will look like the function call was
401 never there in the first place.
403 Examples:
404 >>> a = __(1)
406 The const node will be extracted.
408 >>> def x(d=__(foo.bar)): pass
410 The node containing the default argument will be extracted.
412 >>> def foo(a, b):
413 >>> return 0 < __(len(a)) < b
415 The node containing the function call 'len' will be extracted.
417 If no statements or expressions are selected, the last toplevel
418 statement will be returned.
420 If the selected statement is a discard statement, (i.e. an expression
421 turned into a statement), the wrapped expression is returned instead.
423 For convenience, singleton lists are unpacked.
425 :param str code: A piece of Python code that is parsed as
426 a module. Will be passed through textwrap.dedent first.
427 :param str module_name: The name of the module.
428 :returns: The designated node from the parse tree, or a list of nodes.
429 """
431 def _extract(node: nodes.NodeNG | None) -> nodes.NodeNG | None:
432 if isinstance(node, nodes.Expr):
433 return node.value
435 return node
437 requested_lines: list[int] = []
438 for idx, line in enumerate(code.splitlines()):
439 if line.strip().endswith(_STATEMENT_SELECTOR):
440 requested_lines.append(idx + 1)
442 tree = parse(code, module_name=module_name)
443 if not tree.body:
444 raise ValueError("Empty tree, cannot extract from it")
446 extracted: list[nodes.NodeNG | None] = []
447 if requested_lines:
448 extracted = [_find_statement_by_line(tree, line) for line in requested_lines]
450 # Modifies the tree.
451 extracted.extend(_extract_expressions(tree))
453 if not extracted:
454 extracted.append(tree.body[-1])
456 extracted = [_extract(node) for node in extracted]
457 extracted_without_none = [node for node in extracted if node is not None]
458 if len(extracted_without_none) == 1:
459 return extracted_without_none[0]
460 return extracted_without_none
463def _extract_single_node(code: str, module_name: str = "") -> nodes.NodeNG:
464 """Call extract_node while making sure that only one value is returned."""
465 ret = extract_node(code, module_name)
466 if isinstance(ret, list):
467 return ret[0]
468 return ret
471def _parse_string(
472 data: str, type_comments: bool = True, modname: str | None = None
473) -> tuple[ast.Module, ParserModule]:
474 parser_module = get_parser_module(type_comments=type_comments)
475 try:
476 parsed = parser_module.parse(
477 data + "\n", type_comments=type_comments, filename=modname
478 )
479 except SyntaxError as exc:
480 # If the type annotations are misplaced for some reason, we do not want
481 # to fail the entire parsing of the file, so we need to retry the
482 # parsing without type comment support. We use a heuristic for
483 # determining if the error is due to type annotations.
484 type_annot_related = re.search(r"#\s+type:", exc.text or "")
485 if not (type_annot_related and type_comments):
486 raise
488 parser_module = get_parser_module(type_comments=False)
489 parsed = parser_module.parse(data + "\n", type_comments=False)
490 return parsed, parser_module