Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/astroid/builder.py: 53%
222 statements
« prev ^ index » next coverage.py v7.2.7, created at 2023-06-07 06:53 +0000
1# Licensed under the LGPL: https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html
2# For details: https://github.com/pylint-dev/astroid/blob/main/LICENSE
3# Copyright (c) https://github.com/pylint-dev/astroid/blob/main/CONTRIBUTORS.txt
5"""The AstroidBuilder makes astroid from living object and / or from _ast.
7The builder is not thread safe and can't be used to parse different sources
8at the same time.
9"""
11from __future__ import annotations
13import ast
14import os
15import textwrap
16import types
17from collections.abc import Iterator, Sequence
18from io import TextIOWrapper
19from tokenize import detect_encoding
21from astroid import bases, modutils, nodes, raw_building, rebuilder, util
22from astroid._ast import ParserModule, get_parser_module
23from astroid.exceptions import AstroidBuildingError, AstroidSyntaxError, InferenceError
24from astroid.manager import AstroidManager
26# The name of the transient function that is used to
27# wrap expressions to be extracted when calling
28# extract_node.
29_TRANSIENT_FUNCTION = "__"
31# The comment used to select a statement to be extracted
32# when calling extract_node.
33_STATEMENT_SELECTOR = "#@"
34MISPLACED_TYPE_ANNOTATION_ERROR = "misplaced type annotation"
37def open_source_file(filename: str) -> tuple[TextIOWrapper, str, str]:
38 # pylint: disable=consider-using-with
39 with open(filename, "rb") as byte_stream:
40 encoding = detect_encoding(byte_stream.readline)[0]
41 stream = open(filename, newline=None, encoding=encoding)
42 data = stream.read()
43 return stream, encoding, data
46def _can_assign_attr(node: nodes.ClassDef, attrname: str | None) -> bool:
47 try:
48 slots = node.slots()
49 except NotImplementedError:
50 pass
51 else:
52 if slots and attrname not in {slot.value for slot in slots}:
53 return False
54 return node.qname() != "builtins.object"
57class AstroidBuilder(raw_building.InspectBuilder):
58 """Class for building an astroid tree from source code or from a live module.
60 The param *manager* specifies the manager class which should be used.
61 If no manager is given, then the default one will be used. The
62 param *apply_transforms* determines if the transforms should be
63 applied after the tree was built from source or from a live object,
64 by default being True.
65 """
67 def __init__(
68 self, manager: AstroidManager | None = None, apply_transforms: bool = True
69 ) -> None:
70 super().__init__(manager)
71 self._apply_transforms = apply_transforms
73 def module_build(
74 self, module: types.ModuleType, modname: str | None = None
75 ) -> nodes.Module:
76 """Build an astroid from a living module instance."""
77 node = None
78 path = getattr(module, "__file__", None)
79 loader = getattr(module, "__loader__", None)
80 # Prefer the loader to get the source rather than assuming we have a
81 # filesystem to read the source file from ourselves.
82 if loader:
83 modname = modname or module.__name__
84 source = loader.get_source(modname)
85 if source:
86 node = self.string_build(source, modname, path=path)
87 if node is None and path is not None:
88 path_, ext = os.path.splitext(modutils._path_from_filename(path))
89 if ext in {".py", ".pyc", ".pyo"} and os.path.exists(path_ + ".py"):
90 node = self.file_build(path_ + ".py", modname)
91 if node is None:
92 # this is a built-in module
93 # get a partial representation by introspection
94 node = self.inspect_build(module, modname=modname, path=path)
95 if self._apply_transforms:
96 # We have to handle transformation by ourselves since the
97 # rebuilder isn't called for builtin nodes
98 node = self._manager.visit_transforms(node)
99 assert isinstance(node, nodes.Module)
100 return node
102 def file_build(self, path: str, modname: str | None = None) -> nodes.Module:
103 """Build astroid from a source code file (i.e. from an ast).
105 *path* is expected to be a python source file
106 """
107 try:
108 stream, encoding, data = open_source_file(path)
109 except OSError as exc:
110 raise AstroidBuildingError(
111 "Unable to load file {path}:\n{error}",
112 modname=modname,
113 path=path,
114 error=exc,
115 ) from exc
116 except (SyntaxError, LookupError) as exc:
117 raise AstroidSyntaxError(
118 "Python 3 encoding specification error or unknown encoding:\n"
119 "{error}",
120 modname=modname,
121 path=path,
122 error=exc,
123 ) from exc
124 except UnicodeError as exc: # wrong encoding
125 # detect_encoding returns utf-8 if no encoding specified
126 raise AstroidBuildingError(
127 "Wrong or no encoding specified for {filename}.", filename=path
128 ) from exc
129 with stream:
130 # get module name if necessary
131 if modname is None:
132 try:
133 modname = ".".join(modutils.modpath_from_file(path))
134 except ImportError:
135 modname = os.path.splitext(os.path.basename(path))[0]
136 # build astroid representation
137 module, builder = self._data_build(data, modname, path)
138 return self._post_build(module, builder, encoding)
140 def string_build(
141 self, data: str, modname: str = "", path: str | None = None
142 ) -> nodes.Module:
143 """Build astroid from source code string."""
144 module, builder = self._data_build(data, modname, path)
145 module.file_bytes = data.encode("utf-8")
146 return self._post_build(module, builder, "utf-8")
148 def _post_build(
149 self, module: nodes.Module, builder: rebuilder.TreeRebuilder, encoding: str
150 ) -> nodes.Module:
151 """Handles encoding and delayed nodes after a module has been built."""
152 module.file_encoding = encoding
153 self._manager.cache_module(module)
154 # post tree building steps after we stored the module in the cache:
155 for from_node in builder._import_from_nodes:
156 if from_node.modname == "__future__":
157 for symbol, _ in from_node.names:
158 module.future_imports.add(symbol)
159 self.add_from_names_to_locals(from_node)
160 # handle delayed assattr nodes
161 for delayed in builder._delayed_assattr:
162 self.delayed_assattr(delayed)
164 # Visit the transforms
165 if self._apply_transforms:
166 module = self._manager.visit_transforms(module)
167 return module
169 def _data_build(
170 self, data: str, modname: str, path: str | None
171 ) -> tuple[nodes.Module, rebuilder.TreeRebuilder]:
172 """Build tree node from data and add some informations."""
173 try:
174 node, parser_module = _parse_string(data, type_comments=True)
175 except (TypeError, ValueError, SyntaxError) as exc:
176 raise AstroidSyntaxError(
177 "Parsing Python code failed:\n{error}",
178 source=data,
179 modname=modname,
180 path=path,
181 error=exc,
182 ) from exc
184 if path is not None:
185 node_file = os.path.abspath(path)
186 else:
187 node_file = "<?>"
188 if modname.endswith(".__init__"):
189 modname = modname[:-9]
190 package = True
191 else:
192 package = (
193 path is not None
194 and os.path.splitext(os.path.basename(path))[0] == "__init__"
195 )
196 builder = rebuilder.TreeRebuilder(self._manager, parser_module, data)
197 module = builder.visit_module(node, modname, node_file, package)
198 return module, builder
200 def add_from_names_to_locals(self, node: nodes.ImportFrom) -> None:
201 """Store imported names to the locals.
203 Resort the locals if coming from a delayed node
204 """
206 def _key_func(node: nodes.NodeNG) -> int:
207 return node.fromlineno or 0
209 def sort_locals(my_list: list[nodes.NodeNG]) -> None:
210 my_list.sort(key=_key_func)
212 assert node.parent # It should always default to the module
213 for name, asname in node.names:
214 if name == "*":
215 try:
216 imported = node.do_import_module()
217 except AstroidBuildingError:
218 continue
219 for name in imported.public_names():
220 node.parent.set_local(name, node)
221 sort_locals(node.parent.scope().locals[name]) # type: ignore[arg-type]
222 else:
223 node.parent.set_local(asname or name, node)
224 sort_locals(node.parent.scope().locals[asname or name]) # type: ignore[arg-type]
226 def delayed_assattr(self, node: nodes.AssignAttr) -> None:
227 """Visit a AssAttr node.
229 This adds name to locals and handle members definition.
230 """
231 from astroid import objects # pylint: disable=import-outside-toplevel
233 try:
234 frame = node.frame(future=True)
235 for inferred in node.expr.infer():
236 if isinstance(inferred, util.UninferableBase):
237 continue
238 try:
239 # pylint: disable=unidiomatic-typecheck # We want a narrow check on the
240 # parent type, not all of its subclasses
241 if (
242 type(inferred) == bases.Instance
243 or type(inferred) == objects.ExceptionInstance
244 ):
245 inferred = inferred._proxied
246 iattrs = inferred.instance_attrs
247 if not _can_assign_attr(inferred, node.attrname):
248 continue
249 elif isinstance(inferred, bases.Instance):
250 # Const, Tuple or other containers that inherit from
251 # `Instance`
252 continue
253 elif isinstance(inferred, (bases.Proxy, util.UninferableBase)):
254 continue
255 elif inferred.is_function:
256 iattrs = inferred.instance_attrs
257 else:
258 iattrs = inferred.locals
259 except AttributeError:
260 # XXX log error
261 continue
262 values = iattrs.setdefault(node.attrname, [])
263 if node in values:
264 continue
265 # get assign in __init__ first XXX useful ?
266 if (
267 frame.name == "__init__"
268 and values
269 and values[0].frame(future=True).name != "__init__"
270 ):
271 values.insert(0, node)
272 else:
273 values.append(node)
274 except InferenceError:
275 pass
278def build_namespace_package_module(name: str, path: Sequence[str]) -> nodes.Module:
279 module = nodes.Module(name, path=path, package=True)
280 module.postinit(body=[], doc_node=None)
281 return module
284def parse(
285 code: str,
286 module_name: str = "",
287 path: str | None = None,
288 apply_transforms: bool = True,
289) -> nodes.Module:
290 """Parses a source string in order to obtain an astroid AST from it.
292 :param str code: The code for the module.
293 :param str module_name: The name for the module, if any
294 :param str path: The path for the module
295 :param bool apply_transforms:
296 Apply the transforms for the give code. Use it if you
297 don't want the default transforms to be applied.
298 """
299 code = textwrap.dedent(code)
300 builder = AstroidBuilder(
301 manager=AstroidManager(), apply_transforms=apply_transforms
302 )
303 return builder.string_build(code, modname=module_name, path=path)
306def _extract_expressions(node: nodes.NodeNG) -> Iterator[nodes.NodeNG]:
307 """Find expressions in a call to _TRANSIENT_FUNCTION and extract them.
309 The function walks the AST recursively to search for expressions that
310 are wrapped into a call to _TRANSIENT_FUNCTION. If it finds such an
311 expression, it completely removes the function call node from the tree,
312 replacing it by the wrapped expression inside the parent.
314 :param node: An astroid node.
315 :type node: astroid.bases.NodeNG
316 :yields: The sequence of wrapped expressions on the modified tree
317 expression can be found.
318 """
319 if (
320 isinstance(node, nodes.Call)
321 and isinstance(node.func, nodes.Name)
322 and node.func.name == _TRANSIENT_FUNCTION
323 ):
324 real_expr = node.args[0]
325 assert node.parent
326 real_expr.parent = node.parent
327 # Search for node in all _astng_fields (the fields checked when
328 # get_children is called) of its parent. Some of those fields may
329 # be lists or tuples, in which case the elements need to be checked.
330 # When we find it, replace it by real_expr, so that the AST looks
331 # like no call to _TRANSIENT_FUNCTION ever took place.
332 for name in node.parent._astroid_fields:
333 child = getattr(node.parent, name)
334 if isinstance(child, list):
335 for idx, compound_child in enumerate(child):
336 if compound_child is node:
337 child[idx] = real_expr
338 elif child is node:
339 setattr(node.parent, name, real_expr)
340 yield real_expr
341 else:
342 for child in node.get_children():
343 yield from _extract_expressions(child)
346def _find_statement_by_line(node: nodes.NodeNG, line: int) -> nodes.NodeNG | None:
347 """Extracts the statement on a specific line from an AST.
349 If the line number of node matches line, it will be returned;
350 otherwise its children are iterated and the function is called
351 recursively.
353 :param node: An astroid node.
354 :type node: astroid.bases.NodeNG
355 :param line: The line number of the statement to extract.
356 :type line: int
357 :returns: The statement on the line, or None if no statement for the line
358 can be found.
359 :rtype: astroid.bases.NodeNG or None
360 """
361 if isinstance(node, (nodes.ClassDef, nodes.FunctionDef, nodes.MatchCase)):
362 # This is an inaccuracy in the AST: the nodes that can be
363 # decorated do not carry explicit information on which line
364 # the actual definition (class/def), but .fromline seems to
365 # be close enough.
366 node_line = node.fromlineno
367 else:
368 node_line = node.lineno
370 if node_line == line:
371 return node
373 for child in node.get_children():
374 result = _find_statement_by_line(child, line)
375 if result:
376 return result
378 return None
381def extract_node(code: str, module_name: str = "") -> nodes.NodeNG | list[nodes.NodeNG]:
382 """Parses some Python code as a module and extracts a designated AST node.
384 Statements:
385 To extract one or more statement nodes, append #@ to the end of the line
387 Examples:
388 >>> def x():
389 >>> def y():
390 >>> return 1 #@
392 The return statement will be extracted.
394 >>> class X(object):
395 >>> def meth(self): #@
396 >>> pass
398 The function object 'meth' will be extracted.
400 Expressions:
401 To extract arbitrary expressions, surround them with the fake
402 function call __(...). After parsing, the surrounded expression
403 will be returned and the whole AST (accessible via the returned
404 node's parent attribute) will look like the function call was
405 never there in the first place.
407 Examples:
408 >>> a = __(1)
410 The const node will be extracted.
412 >>> def x(d=__(foo.bar)): pass
414 The node containing the default argument will be extracted.
416 >>> def foo(a, b):
417 >>> return 0 < __(len(a)) < b
419 The node containing the function call 'len' will be extracted.
421 If no statements or expressions are selected, the last toplevel
422 statement will be returned.
424 If the selected statement is a discard statement, (i.e. an expression
425 turned into a statement), the wrapped expression is returned instead.
427 For convenience, singleton lists are unpacked.
429 :param str code: A piece of Python code that is parsed as
430 a module. Will be passed through textwrap.dedent first.
431 :param str module_name: The name of the module.
432 :returns: The designated node from the parse tree, or a list of nodes.
433 """
435 def _extract(node: nodes.NodeNG | None) -> nodes.NodeNG | None:
436 if isinstance(node, nodes.Expr):
437 return node.value
439 return node
441 requested_lines: list[int] = []
442 for idx, line in enumerate(code.splitlines()):
443 if line.strip().endswith(_STATEMENT_SELECTOR):
444 requested_lines.append(idx + 1)
446 tree = parse(code, module_name=module_name)
447 if not tree.body:
448 raise ValueError("Empty tree, cannot extract from it")
450 extracted: list[nodes.NodeNG | None] = []
451 if requested_lines:
452 extracted = [_find_statement_by_line(tree, line) for line in requested_lines]
454 # Modifies the tree.
455 extracted.extend(_extract_expressions(tree))
457 if not extracted:
458 extracted.append(tree.body[-1])
460 extracted = [_extract(node) for node in extracted]
461 extracted_without_none = [node for node in extracted if node is not None]
462 if len(extracted_without_none) == 1:
463 return extracted_without_none[0]
464 return extracted_without_none
467def _extract_single_node(code: str, module_name: str = "") -> nodes.NodeNG:
468 """Call extract_node while making sure that only one value is returned."""
469 ret = extract_node(code, module_name)
470 if isinstance(ret, list):
471 return ret[0]
472 return ret
475def _parse_string(
476 data: str, type_comments: bool = True
477) -> tuple[ast.Module, ParserModule]:
478 parser_module = get_parser_module(type_comments=type_comments)
479 try:
480 parsed = parser_module.parse(data + "\n", type_comments=type_comments)
481 except SyntaxError as exc:
482 # If the type annotations are misplaced for some reason, we do not want
483 # to fail the entire parsing of the file, so we need to retry the parsing without
484 # type comment support.
485 if exc.args[0] != MISPLACED_TYPE_ANNOTATION_ERROR or not type_comments:
486 raise
488 parser_module = get_parser_module(type_comments=False)
489 parsed = parser_module.parse(data + "\n", type_comments=False)
490 return parsed, parser_module