Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/libcst/metadata/scope_provider.py: 30%
620 statements
« prev ^ index » next coverage.py v7.3.1, created at 2023-09-25 06:43 +0000
1# Copyright (c) Meta Platforms, Inc. and affiliates.
2#
3# This source code is licensed under the MIT license found in the
4# LICENSE file in the root directory of this source tree.
7import abc
8import builtins
9from collections import defaultdict
10from contextlib import contextmanager
11from dataclasses import dataclass
12from enum import auto, Enum
13from typing import (
14 Collection,
15 Dict,
16 Iterator,
17 List,
18 Mapping,
19 MutableMapping,
20 Optional,
21 Set,
22 Tuple,
23 Type,
24 Union,
25)
27import libcst as cst
28from libcst import ensure_type
29from libcst._add_slots import add_slots
30from libcst.helpers import get_full_name_for_node
31from libcst.metadata.base_provider import BatchableMetadataProvider
32from libcst.metadata.expression_context_provider import (
33 ExpressionContext,
34 ExpressionContextProvider,
35)
37# Comprehensions are handled separately in _visit_comp_alike due to
38# the complexity of the semantics
39_ASSIGNMENT_LIKE_NODES = (
40 cst.AnnAssign,
41 cst.AsName,
42 cst.Assign,
43 cst.AugAssign,
44 cst.ClassDef,
45 cst.CompFor,
46 cst.FunctionDef,
47 cst.Global,
48 cst.Import,
49 cst.ImportFrom,
50 cst.NamedExpr,
51 cst.Nonlocal,
52 cst.Parameters,
53 cst.WithItem,
54)
57@add_slots
58@dataclass(frozen=False)
59class Access:
60 """
61 An Access records an access of an assignment.
63 .. note::
64 This scope analysis only analyzes access via a :class:`~libcst.Name` or a :class:`~libcst.Name`
65 node embedded in other node like :class:`~libcst.Call` or :class:`~libcst.Attribute`.
66 It doesn't support type annontation using :class:`~libcst.SimpleString` literal for forward
67 references. E.g. in this example, the ``"Tree"`` isn't parsed as an access::
69 class Tree:
70 def __new__(cls) -> "Tree":
71 ...
72 """
74 #: The node of the access. A name is an access when the expression context is
75 #: :attr:`ExpressionContext.LOAD`. This is usually the name node representing the
76 #: access, except for: 1) dotted imports, when it might be the attribute that
77 #: represents the most specific part of the imported symbol; and 2) string
78 #: annotations, when it is the entire string literal
79 node: Union[cst.Name, cst.Attribute, cst.BaseString]
81 #: The scope of the access. Note that a access could be in a child scope of its
82 #: assignment.
83 scope: "Scope"
85 is_annotation: bool
87 is_type_hint: bool
89 __assignments: Set["BaseAssignment"]
90 __index: int
92 def __init__(
93 self, node: cst.Name, scope: "Scope", is_annotation: bool, is_type_hint: bool
94 ) -> None:
95 self.node = node
96 self.scope = scope
97 self.is_annotation = is_annotation
98 self.is_type_hint = is_type_hint
99 self.__assignments = set()
100 self.__index = scope._assignment_count
102 def __hash__(self) -> int:
103 return id(self)
105 @property
106 def referents(self) -> Collection["BaseAssignment"]:
107 """Return all assignments of the access."""
108 return self.__assignments
110 @property
111 def _index(self) -> int:
112 return self.__index
114 def record_assignment(self, assignment: "BaseAssignment") -> None:
115 if assignment.scope != self.scope or assignment._index < self.__index:
116 self.__assignments.add(assignment)
118 def record_assignments(self, name: str) -> None:
119 assignments = self.scope[name]
120 # filter out assignments that happened later than this access
121 previous_assignments = {
122 assignment
123 for assignment in assignments
124 if assignment.scope != self.scope or assignment._index < self.__index
125 }
126 if not previous_assignments and assignments and self.scope.parent != self.scope:
127 previous_assignments = self.scope.parent[name]
128 self.__assignments |= previous_assignments
131class QualifiedNameSource(Enum):
132 IMPORT = auto()
133 BUILTIN = auto()
134 LOCAL = auto()
137@add_slots
138@dataclass(frozen=True)
139class QualifiedName:
140 #: Qualified name, e.g. ``a.b.c`` or ``fn.<locals>.var``.
141 name: str
143 #: Source of the name, either :attr:`QualifiedNameSource.IMPORT`, :attr:`QualifiedNameSource.BUILTIN`
144 #: or :attr:`QualifiedNameSource.LOCAL`.
145 source: QualifiedNameSource
148class BaseAssignment(abc.ABC):
149 """Abstract base class of :class:`Assignment` and :class:`BuitinAssignment`."""
151 #: The name of assignment.
152 name: str
154 #: The scope associates to assignment.
155 scope: "Scope"
156 __accesses: Set[Access]
158 def __init__(self, name: str, scope: "Scope") -> None:
159 self.name = name
160 self.scope = scope
161 self.__accesses = set()
163 def record_access(self, access: Access) -> None:
164 if access.scope != self.scope or self._index < access._index:
165 self.__accesses.add(access)
167 def record_accesses(self, accesses: Set[Access]) -> None:
168 later_accesses = {
169 access
170 for access in accesses
171 if access.scope != self.scope or self._index < access._index
172 }
173 self.__accesses |= later_accesses
174 earlier_accesses = accesses - later_accesses
175 if earlier_accesses and self.scope.parent != self.scope:
176 # Accesses "earlier" than the relevant assignment should be attached
177 # to assignments of the same name in the parent
178 for shadowed_assignment in self.scope.parent[self.name]:
179 shadowed_assignment.record_accesses(earlier_accesses)
181 @property
182 def references(self) -> Collection[Access]:
183 """Return all accesses of the assignment."""
184 # we don't want to publicly expose the mutable version of this
185 return self.__accesses
187 def __hash__(self) -> int:
188 return id(self)
190 @property
191 def _index(self) -> int:
192 """Return an integer that represents the order of assignments in `scope`"""
193 return -1
195 @abc.abstractmethod
196 def get_qualified_names_for(self, full_name: str) -> Set[QualifiedName]:
197 ...
200class Assignment(BaseAssignment):
201 """An assignment records the name, CSTNode and its accesses."""
203 #: The node of assignment, it could be a :class:`~libcst.Import`, :class:`~libcst.ImportFrom`,
204 #: :class:`~libcst.Name`, :class:`~libcst.FunctionDef`, or :class:`~libcst.ClassDef`.
205 node: cst.CSTNode
206 __index: int
208 def __init__(
209 self, name: str, scope: "Scope", node: cst.CSTNode, index: int
210 ) -> None:
211 self.node = node
212 self.__index = index
213 super().__init__(name, scope)
215 @property
216 def _index(self) -> int:
217 return self.__index
219 def get_qualified_names_for(self, full_name: str) -> Set[QualifiedName]:
220 return {
221 QualifiedName(
222 f"{self.scope._name_prefix}.{full_name}"
223 if self.scope._name_prefix
224 else full_name,
225 QualifiedNameSource.LOCAL,
226 )
227 }
230# even though we don't override the constructor.
231class BuiltinAssignment(BaseAssignment):
232 """
233 A BuiltinAssignment represents an value provide by Python as a builtin, including
234 `functions <https://docs.python.org/3/library/functions.html>`_,
235 `constants <https://docs.python.org/3/library/constants.html>`_, and
236 `types <https://docs.python.org/3/library/stdtypes.html>`_.
237 """
239 def get_qualified_names_for(self, full_name: str) -> Set[QualifiedName]:
240 return {QualifiedName(f"builtins.{self.name}", QualifiedNameSource.BUILTIN)}
243class ImportAssignment(Assignment):
244 """An assignment records the import node and it's alias"""
246 as_name: cst.CSTNode
248 def __init__(
249 self,
250 name: str,
251 scope: "Scope",
252 node: cst.CSTNode,
253 index: int,
254 as_name: cst.CSTNode,
255 ) -> None:
256 super().__init__(name, scope, node, index)
257 self.as_name = as_name
259 def get_module_name_for_import(self) -> str:
260 module = ""
261 if isinstance(self.node, cst.ImportFrom):
262 module_attr = self.node.module
263 relative = self.node.relative
264 if module_attr:
265 module = get_full_name_for_node(module_attr) or ""
266 if relative:
267 module = "." * len(relative) + module
268 return module
270 def get_qualified_names_for(self, full_name: str) -> Set[QualifiedName]:
271 module = self.get_module_name_for_import()
272 results = set()
273 assert isinstance(self.node, (cst.ImportFrom, cst.Import))
274 import_names = self.node.names
275 if not isinstance(import_names, cst.ImportStar):
276 for name in import_names:
277 real_name = get_full_name_for_node(name.name)
278 if not real_name:
279 continue
280 # real_name can contain `.` for dotted imports
281 # for these we want to find the longest prefix that matches full_name
282 parts = real_name.split(".")
283 real_names = [".".join(parts[:i]) for i in range(len(parts), 0, -1)]
284 for real_name in real_names:
285 as_name = real_name
286 if module and module.endswith("."):
287 # from . import a
288 # real_name should be ".a"
289 real_name = f"{module}{real_name}"
290 elif module:
291 real_name = f"{module}.{real_name}"
292 if name and name.asname:
293 eval_alias = name.evaluated_alias
294 if eval_alias is not None:
295 as_name = eval_alias
296 if full_name.startswith(as_name):
297 remaining_name = full_name.split(as_name, 1)[1]
298 if remaining_name and not remaining_name.startswith("."):
299 continue
300 remaining_name = remaining_name.lstrip(".")
301 results.add(
302 QualifiedName(
303 f"{real_name}.{remaining_name}"
304 if remaining_name
305 else real_name,
306 QualifiedNameSource.IMPORT,
307 )
308 )
309 break
310 return results
313class Assignments:
314 """A container to provide all assignments in a scope."""
316 def __init__(self, assignments: Mapping[str, Collection[BaseAssignment]]) -> None:
317 self._assignments = assignments
319 def __iter__(self) -> Iterator[BaseAssignment]:
320 """Iterate through all assignments by ``for i in scope.assignments``."""
321 for assignments in self._assignments.values():
322 for assignment in assignments:
323 yield assignment
325 def __getitem__(self, node: Union[str, cst.CSTNode]) -> Collection[BaseAssignment]:
326 """Get assignments given a name str or :class:`~libcst.CSTNode` by ``scope.assignments[node]``"""
327 name = _NameUtil.get_name_for(node)
328 return set(self._assignments[name]) if name in self._assignments else set()
330 def __contains__(self, node: Union[str, cst.CSTNode]) -> bool:
331 """Check if a name str or :class:`~libcst.CSTNode` has any assignment by ``node in scope.assignments``"""
332 return len(self[node]) > 0
335class Accesses:
336 """A container to provide all accesses in a scope."""
338 def __init__(self, accesses: Mapping[str, Collection[Access]]) -> None:
339 self._accesses = accesses
341 def __iter__(self) -> Iterator[Access]:
342 """Iterate through all accesses by ``for i in scope.accesses``."""
343 for accesses in self._accesses.values():
344 for access in accesses:
345 yield access
347 def __getitem__(self, node: Union[str, cst.CSTNode]) -> Collection[Access]:
348 """Get accesses given a name str or :class:`~libcst.CSTNode` by ``scope.accesses[node]``"""
349 name = _NameUtil.get_name_for(node)
350 return self._accesses[name] if name in self._accesses else set()
352 def __contains__(self, node: Union[str, cst.CSTNode]) -> bool:
353 """Check if a name str or :class:`~libcst.CSTNode` has any access by ``node in scope.accesses``"""
354 return len(self[node]) > 0
357class _NameUtil:
358 @staticmethod
359 def get_name_for(node: Union[str, cst.CSTNode]) -> Optional[str]:
360 """A helper function to retrieve simple name str from a CSTNode or str"""
361 if isinstance(node, cst.Name):
362 return node.value
363 elif isinstance(node, str):
364 return node
365 elif isinstance(node, cst.Call):
366 return _NameUtil.get_name_for(node.func)
367 elif isinstance(node, cst.Subscript):
368 return _NameUtil.get_name_for(node.value)
369 elif isinstance(node, (cst.FunctionDef, cst.ClassDef)):
370 return _NameUtil.get_name_for(node.name)
371 return None
374class Scope(abc.ABC):
375 """
376 Base class of all scope classes. Scope object stores assignments from imports,
377 variable assignments, function definition or class definition.
378 A scope has a parent scope which represents the inheritance relationship. That means
379 an assignment in parent scope is viewable to the child scope and the child scope may
380 overwrites the assignment by using the same name.
382 Use ``name in scope`` to check whether a name is viewable in the scope.
383 Use ``scope[name]`` to retrieve all viewable assignments in the scope.
385 .. note::
386 This scope analysis module only analyzes local variable names and it doesn't handle
387 attribute names; for example, given ``a.b.c = 1``, local variable name ``a`` is recorded
388 as an assignment instead of ``c`` or ``a.b.c``. To analyze the assignment/access of
389 arbitrary object attributes, we leave the job to type inference metadata provider
390 coming in the future.
391 """
393 #: Parent scope. Note the parent scope of a GlobalScope is itself.
394 parent: "Scope"
396 #: Refers to the GlobalScope.
397 globals: "GlobalScope"
398 _assignments: MutableMapping[str, Set[BaseAssignment]]
399 _assignment_count: int
400 _accesses_by_name: MutableMapping[str, Set[Access]]
401 _accesses_by_node: MutableMapping[cst.CSTNode, Set[Access]]
402 _name_prefix: str
404 def __init__(self, parent: "Scope") -> None:
405 super().__init__()
406 self.parent = parent
407 self.globals = parent.globals
408 self._assignments = defaultdict(set)
409 self._assignment_count = 0
410 self._accesses_by_name = defaultdict(set)
411 self._accesses_by_node = defaultdict(set)
412 self._name_prefix = ""
414 def record_assignment(self, name: str, node: cst.CSTNode) -> None:
415 target = self._find_assignment_target(name)
416 target._assignments[name].add(
417 Assignment(
418 name=name, scope=target, node=node, index=target._assignment_count
419 )
420 )
422 def record_import_assignment(
423 self, name: str, node: cst.CSTNode, as_name: cst.CSTNode
424 ) -> None:
425 target = self._find_assignment_target(name)
426 target._assignments[name].add(
427 ImportAssignment(
428 name=name,
429 scope=target,
430 node=node,
431 as_name=as_name,
432 index=target._assignment_count,
433 )
434 )
436 def _find_assignment_target(self, name: str) -> "Scope":
437 return self
439 def record_access(self, name: str, access: Access) -> None:
440 self._accesses_by_name[name].add(access)
441 self._accesses_by_node[access.node].add(access)
443 def _is_visible_from_children(self) -> bool:
444 """Returns if the assignments in this scope can be accessed from children.
446 This is normally True, except for class scopes::
448 def outer_fn():
449 v = ... # outer_fn's declaration
450 class InnerCls:
451 v = ... # shadows outer_fn's declaration
452 class InnerInnerCls:
453 v = ... # shadows all previous declarations of v
454 def inner_fn():
455 nonlocal v
456 v = ... # this refers to outer_fn's declaration
457 # and not to any of the inner classes' as those are
458 # hidden from their children.
459 """
460 return True
462 def _next_visible_parent(self, first: Optional["Scope"] = None) -> "Scope":
463 parent = first if first is not None else self.parent
464 while not parent._is_visible_from_children():
465 parent = parent.parent
466 return parent
468 @abc.abstractmethod
469 def __contains__(self, name: str) -> bool:
470 """Check if the name str exist in current scope by ``name in scope``."""
471 ...
473 @abc.abstractmethod
474 def __getitem__(self, name: str) -> Set[BaseAssignment]:
475 """
476 Get assignments given a name str by ``scope[name]``.
478 .. note::
479 *Why does it return a list of assignments given a name instead of just one assignment?*
481 Many programming languages differentiate variable declaration and assignment.
482 Further, those programming languages often disallow duplicate declarations within
483 the same scope, and will often hoist the declaration (without its assignment) to
484 the top of the scope. These design decisions make static analysis much easier,
485 because it's possible to match a name against its single declaration for a given scope.
487 As an example, the following code would be valid in JavaScript::
489 function fn() {
490 console.log(value); // value is defined here, because the declaration is hoisted, but is currently 'undefined'.
491 var value = 5; // A function-scoped declaration.
492 }
493 fn(); // prints 'undefined'.
495 In contrast, Python's declaration and assignment are identical and are not hoisted::
497 if conditional_value:
498 value = 5
499 elif other_conditional_value:
500 value = 10
501 print(value) # possibly valid, depending on conditional execution
503 This code may throw a ``NameError`` if both conditional values are falsy.
504 It also means that depending on the codepath taken, the original declaration
505 could come from either ``value = ...`` assignment node.
506 As a result, instead of returning a single declaration,
507 we're forced to return a collection of all of the assignments we think could have
508 defined a given name by the time a piece of code is executed.
509 For the above example, value would resolve to a set of both assignments.
510 """
511 ...
513 def __hash__(self) -> int:
514 return id(self)
516 @abc.abstractmethod
517 def record_global_overwrite(self, name: str) -> None:
518 ...
520 @abc.abstractmethod
521 def record_nonlocal_overwrite(self, name: str) -> None:
522 ...
524 def get_qualified_names_for(
525 self, node: Union[str, cst.CSTNode]
526 ) -> Collection[QualifiedName]:
527 """Get all :class:`~libcst.metadata.QualifiedName` in current scope given a
528 :class:`~libcst.CSTNode`.
529 The source of a qualified name can be either :attr:`QualifiedNameSource.IMPORT`,
530 :attr:`QualifiedNameSource.BUILTIN` or :attr:`QualifiedNameSource.LOCAL`.
531 Given the following example, ``c`` has qualified name ``a.b.c`` with source ``IMPORT``,
532 ``f`` has qualified name ``Cls.f`` with source ``LOCAL``, ``a`` has qualified name
533 ``Cls.f.<locals>.a``, ``i`` has qualified name ``Cls.f.<locals>.<comprehension>.i``,
534 and the builtin ``int`` has qualified name ``builtins.int`` with source ``BUILTIN``::
536 from a.b import c
537 class Cls:
538 def f(self) -> "c":
539 c()
540 a = int("1")
541 [i for i in c()]
543 We extends `PEP-3155 <https://www.python.org/dev/peps/pep-3155/>`_
544 (defines ``__qualname__`` for class and function only; function namespace is followed
545 by a ``<locals>``) to provide qualified name for all :class:`~libcst.CSTNode`
546 recorded by :class:`~libcst.metadata.Assignment` and :class:`~libcst.metadata.Access`.
547 The namespace of a comprehension (:class:`~libcst.ListComp`, :class:`~libcst.SetComp`,
548 :class:`~libcst.DictComp`) is represented with ``<comprehension>``.
550 An imported name may be used for type annotation with :class:`~libcst.SimpleString` and
551 currently resolving the qualified given :class:`~libcst.SimpleString` is not supported
552 considering it could be a complex type annotation in the string which is hard to
553 resolve, e.g. ``List[Union[int, str]]``.
554 """
555 # if this node is an access we know the assignment and we can use that name
556 node_accesses = (
557 self._accesses_by_node.get(node) if isinstance(node, cst.CSTNode) else None
558 )
559 if node_accesses:
560 return {
561 qname
562 for access in node_accesses
563 for referent in access.referents
564 for qname in referent.get_qualified_names_for(referent.name)
565 }
567 full_name = get_full_name_for_node(node)
568 if full_name is None:
569 return set()
571 assignments = set()
572 prefix = full_name
573 while prefix:
574 if prefix in self:
575 assignments = self[prefix]
576 break
577 idx = prefix.rfind(".")
578 prefix = None if idx == -1 else prefix[:idx]
580 if not isinstance(node, str):
581 for assignment in assignments:
582 if isinstance(assignment, Assignment) and _is_assignment(
583 node, assignment.node
584 ):
585 return assignment.get_qualified_names_for(full_name)
587 results = set()
588 for assignment in assignments:
589 results |= assignment.get_qualified_names_for(full_name)
590 return results
592 @property
593 def assignments(self) -> Assignments:
594 """Return an :class:`~libcst.metadata.Assignments` contains all assignmens in current scope."""
595 return Assignments(self._assignments)
597 @property
598 def accesses(self) -> Accesses:
599 """Return an :class:`~libcst.metadata.Accesses` contains all accesses in current scope."""
600 return Accesses(self._accesses_by_name)
603class BuiltinScope(Scope):
604 """
605 A BuiltinScope represents python builtin declarations. See https://docs.python.org/3/library/builtins.html
606 """
608 def __init__(self, globals: Scope) -> None:
609 self.globals: Scope = globals # must be defined before Scope.__init__ is called
610 super().__init__(parent=self)
612 def __contains__(self, name: str) -> bool:
613 return hasattr(builtins, name)
615 def __getitem__(self, name: str) -> Set[BaseAssignment]:
616 if name in self._assignments:
617 return self._assignments[name]
618 if hasattr(builtins, name):
619 # note - we only see the builtin assignments during the deferred
620 # access resolution. unfortunately that means we have to create the
621 # assignment here, which can cause the set to mutate during iteration
622 self._assignments[name].add(BuiltinAssignment(name, self))
623 return self._assignments[name]
624 return set()
626 def record_global_overwrite(self, name: str) -> None:
627 raise NotImplementedError("global overwrite in builtin scope are not allowed")
629 def record_nonlocal_overwrite(self, name: str) -> None:
630 raise NotImplementedError("declarations in builtin scope are not allowed")
632 def _find_assignment_target(self, name: str) -> "Scope":
633 raise NotImplementedError("assignments in builtin scope are not allowed")
636class GlobalScope(Scope):
637 """
638 A GlobalScope is the scope of module. All module level assignments are recorded in GlobalScope.
639 """
641 def __init__(self) -> None:
642 super().__init__(parent=BuiltinScope(self))
644 def __contains__(self, name: str) -> bool:
645 if name in self._assignments:
646 return len(self._assignments[name]) > 0
647 return name in self._next_visible_parent()
649 def __getitem__(self, name: str) -> Set[BaseAssignment]:
650 if name in self._assignments:
651 return self._assignments[name]
653 parent = self._next_visible_parent()
654 return parent[name]
656 def record_global_overwrite(self, name: str) -> None:
657 pass
659 def record_nonlocal_overwrite(self, name: str) -> None:
660 raise NotImplementedError("nonlocal declaration not allowed at module level")
663class LocalScope(Scope, abc.ABC):
664 _scope_overwrites: Dict[str, Scope]
666 #: Name of function. Used as qualified name.
667 name: Optional[str]
669 #: The :class:`~libcst.CSTNode` node defines the current scope.
670 node: cst.CSTNode
672 def __init__(
673 self, parent: Scope, node: cst.CSTNode, name: Optional[str] = None
674 ) -> None:
675 super().__init__(parent)
676 self.name = name
677 self.node = node
678 self._scope_overwrites = {}
679 # pyre-fixme[4]: Attribute `_name_prefix` of class `LocalScope` has type `str` but no type is specified.
680 self._name_prefix = self._make_name_prefix()
682 def record_global_overwrite(self, name: str) -> None:
683 self._scope_overwrites[name] = self.globals
685 def record_nonlocal_overwrite(self, name: str) -> None:
686 self._scope_overwrites[name] = self.parent
688 def _find_assignment_target(self, name: str) -> "Scope":
689 if name in self._scope_overwrites:
690 scope = self._scope_overwrites[name]
691 return self._next_visible_parent(scope)._find_assignment_target(name)
692 else:
693 return super()._find_assignment_target(name)
695 def __contains__(self, name: str) -> bool:
696 if name in self._scope_overwrites:
697 return name in self._scope_overwrites[name]
698 if name in self._assignments:
699 return len(self._assignments[name]) > 0
700 return name in self._next_visible_parent()
702 def __getitem__(self, name: str) -> Set[BaseAssignment]:
703 if name in self._scope_overwrites:
704 scope = self._scope_overwrites[name]
705 return self._next_visible_parent(scope)[name]
706 if name in self._assignments:
707 return self._assignments[name]
708 else:
709 return self._next_visible_parent()[name]
711 def _make_name_prefix(self) -> str:
712 # filter falsey strings out
713 return ".".join(filter(None, [self.parent._name_prefix, self.name, "<locals>"]))
716# even though we don't override the constructor.
717class FunctionScope(LocalScope):
718 """
719 When a function is defined, it creates a FunctionScope.
720 """
722 pass
725# even though we don't override the constructor.
726class ClassScope(LocalScope):
727 """
728 When a class is defined, it creates a ClassScope.
729 """
731 def _is_visible_from_children(self) -> bool:
732 return False
734 def _make_name_prefix(self) -> str:
735 # filter falsey strings out
736 return ".".join(filter(None, [self.parent._name_prefix, self.name]))
739# even though we don't override the constructor.
740class ComprehensionScope(LocalScope):
741 """
742 Comprehensions and generator expressions create their own scope. For example, in
744 [i for i in range(10)]
746 The variable ``i`` is only viewable within the ComprehensionScope.
747 """
749 # TODO: Assignment expressions (Python 3.8) will complicate ComprehensionScopes,
750 # and will require us to handle such assignments as non-local.
751 # https://www.python.org/dev/peps/pep-0572/#scope-of-the-target
753 def _make_name_prefix(self) -> str:
754 # filter falsey strings out
755 return ".".join(filter(None, [self.parent._name_prefix, "<comprehension>"]))
758# Generates dotted names from an Attribute or Name node:
759# Attribute(value=Name(value="a"), attr=Name(value="b")) -> ("a.b", "a")
760# each string has the corresponding CSTNode attached to it
761def _gen_dotted_names(
762 node: Union[cst.Attribute, cst.Name]
763) -> Iterator[Tuple[str, Union[cst.Attribute, cst.Name]]]:
764 if isinstance(node, cst.Name):
765 yield node.value, node
766 else:
767 value = node.value
768 if isinstance(value, cst.Call):
769 value = value.func
770 if isinstance(value, (cst.Attribute, cst.Name)):
771 name_values = _gen_dotted_names(value)
772 try:
773 next_name, next_node = next(name_values)
774 except StopIteration:
775 return
776 else:
777 yield next_name, next_node
778 yield from name_values
779 elif isinstance(value, (cst.Attribute, cst.Name)):
780 name_values = _gen_dotted_names(value)
781 try:
782 next_name, next_node = next(name_values)
783 except StopIteration:
784 return
785 else:
786 yield f"{next_name}.{node.attr.value}", node
787 yield next_name, next_node
788 yield from name_values
791def _is_assignment(node: cst.CSTNode, assignment_node: cst.CSTNode) -> bool:
792 """
793 Returns true if ``node`` is part of the assignment at ``assignment_node``.
795 Normally this is just a simple identity check, except for imports where the
796 assignment is attached to the entire import statement but we are interested in
797 ``Name`` nodes inside the statement.
798 """
799 if node is assignment_node:
800 return True
801 if isinstance(assignment_node, (cst.Import, cst.ImportFrom)):
802 aliases = assignment_node.names
803 if isinstance(aliases, cst.ImportStar):
804 return False
805 for alias in aliases:
806 if alias.name is node:
807 return True
808 asname = alias.asname
809 if asname is not None:
810 if asname.name is node:
811 return True
812 return False
815@dataclass(frozen=True)
816class DeferredAccess:
817 access: Access
818 enclosing_attribute: Optional[cst.Attribute]
819 enclosing_string_annotation: Optional[cst.BaseString]
822class ScopeVisitor(cst.CSTVisitor):
823 # since it's probably not useful. That can makes this visitor cleaner.
824 def __init__(self, provider: "ScopeProvider") -> None:
825 self.provider: ScopeProvider = provider
826 self.scope: Scope = GlobalScope()
827 self.__deferred_accesses: List[DeferredAccess] = []
828 self.__top_level_attribute_stack: List[Optional[cst.Attribute]] = [None]
829 self.__in_annotation_stack: List[bool] = [False]
830 self.__in_type_hint_stack: List[bool] = [False]
831 self.__in_ignored_subscript: Set[cst.Subscript] = set()
832 self.__last_string_annotation: Optional[cst.BaseString] = None
833 self.__ignore_annotation: int = 0
835 @contextmanager
836 def _new_scope(
837 self, kind: Type[LocalScope], node: cst.CSTNode, name: Optional[str] = None
838 ) -> Iterator[None]:
839 parent_scope = self.scope
840 self.scope = kind(parent_scope, node, name)
841 try:
842 yield
843 finally:
844 self.scope = parent_scope
846 @contextmanager
847 def _switch_scope(self, scope: Scope) -> Iterator[None]:
848 current_scope = self.scope
849 self.scope = scope
850 try:
851 yield
852 finally:
853 self.scope = current_scope
855 def _visit_import_alike(self, node: Union[cst.Import, cst.ImportFrom]) -> bool:
856 names = node.names
857 if isinstance(names, cst.ImportStar):
858 return False
860 # make sure node.names is Sequence[ImportAlias]
861 for name in names:
862 self.provider.set_metadata(name, self.scope)
863 asname = name.asname
864 if asname is not None:
865 name_values = _gen_dotted_names(cst.ensure_type(asname.name, cst.Name))
866 import_node_asname = asname.name
867 else:
868 name_values = _gen_dotted_names(name.name)
869 import_node_asname = name.name
871 for name_value, _ in name_values:
872 self.scope.record_import_assignment(
873 name_value, node, import_node_asname
874 )
875 return False
877 def visit_Import(self, node: cst.Import) -> Optional[bool]:
878 return self._visit_import_alike(node)
880 def visit_ImportFrom(self, node: cst.ImportFrom) -> Optional[bool]:
881 return self._visit_import_alike(node)
883 def visit_Attribute(self, node: cst.Attribute) -> Optional[bool]:
884 if self.__top_level_attribute_stack[-1] is None:
885 self.__top_level_attribute_stack[-1] = node
886 node.value.visit(self) # explicitly not visiting attr
887 if self.__top_level_attribute_stack[-1] is node:
888 self.__top_level_attribute_stack[-1] = None
889 return False
891 def visit_Call(self, node: cst.Call) -> Optional[bool]:
892 self.__top_level_attribute_stack.append(None)
893 self.__in_type_hint_stack.append(False)
894 qnames = {qn.name for qn in self.scope.get_qualified_names_for(node)}
895 if "typing.NewType" in qnames or "typing.TypeVar" in qnames:
896 node.func.visit(self)
897 self.__in_type_hint_stack[-1] = True
898 for arg in node.args[1:]:
899 arg.visit(self)
900 return False
901 if "typing.cast" in qnames:
902 node.func.visit(self)
903 if len(node.args) > 0:
904 self.__in_type_hint_stack.append(True)
905 node.args[0].visit(self)
906 self.__in_type_hint_stack.pop()
907 for arg in node.args[1:]:
908 arg.visit(self)
909 return False
910 return True
912 def leave_Call(self, original_node: cst.Call) -> None:
913 self.__top_level_attribute_stack.pop()
914 self.__in_type_hint_stack.pop()
916 def visit_Annotation(self, node: cst.Annotation) -> Optional[bool]:
917 self.__in_annotation_stack.append(True)
919 def leave_Annotation(self, original_node: cst.Annotation) -> None:
920 self.__in_annotation_stack.pop()
922 def visit_SimpleString(self, node: cst.SimpleString) -> Optional[bool]:
923 self._handle_string_annotation(node)
924 return False
926 def visit_ConcatenatedString(self, node: cst.ConcatenatedString) -> Optional[bool]:
927 return not self._handle_string_annotation(node)
929 def _handle_string_annotation(
930 self, node: Union[cst.SimpleString, cst.ConcatenatedString]
931 ) -> bool:
932 """Returns whether it successfully handled the string annotation"""
933 if (
934 self.__in_type_hint_stack[-1] or self.__in_annotation_stack[-1]
935 ) and not self.__in_ignored_subscript:
936 value = node.evaluated_value
937 if value:
938 top_level_annotation = self.__last_string_annotation is None
939 if top_level_annotation:
940 self.__last_string_annotation = node
941 try:
942 mod = cst.parse_module(value)
943 mod.visit(self)
944 except cst.ParserSyntaxError:
945 # swallow string annotation parsing errors
946 # this is the same behavior as cPython
947 pass
948 if top_level_annotation:
949 self.__last_string_annotation = None
950 return True
951 return False
953 def visit_Subscript(self, node: cst.Subscript) -> Optional[bool]:
954 in_type_hint = False
955 if isinstance(node.value, cst.Name):
956 qnames = {qn.name for qn in self.scope.get_qualified_names_for(node.value)}
957 if any(qn.startswith(("typing.", "typing_extensions.")) for qn in qnames):
958 in_type_hint = True
959 if "typing.Literal" in qnames or "typing_extensions.Literal" in qnames:
960 self.__in_ignored_subscript.add(node)
962 self.__in_type_hint_stack.append(in_type_hint)
963 return True
965 def leave_Subscript(self, original_node: cst.Subscript) -> None:
966 self.__in_type_hint_stack.pop()
967 self.__in_ignored_subscript.discard(original_node)
969 def visit_Name(self, node: cst.Name) -> Optional[bool]:
970 # not all Name have ExpressionContext
971 context = self.provider.get_metadata(ExpressionContextProvider, node, None)
972 if context == ExpressionContext.STORE:
973 self.scope.record_assignment(node.value, node)
974 elif context in (ExpressionContext.LOAD, ExpressionContext.DEL, None):
975 access = Access(
976 node,
977 self.scope,
978 is_annotation=bool(
979 self.__in_annotation_stack[-1] and not self.__ignore_annotation
980 ),
981 is_type_hint=bool(self.__in_type_hint_stack[-1]),
982 )
983 self.__deferred_accesses.append(
984 DeferredAccess(
985 access=access,
986 enclosing_attribute=self.__top_level_attribute_stack[-1],
987 enclosing_string_annotation=self.__last_string_annotation,
988 )
989 )
991 def visit_FunctionDef(self, node: cst.FunctionDef) -> Optional[bool]:
992 self.scope.record_assignment(node.name.value, node)
993 self.provider.set_metadata(node.name, self.scope)
995 with self._new_scope(FunctionScope, node, get_full_name_for_node(node.name)):
996 node.params.visit(self)
997 node.body.visit(self)
999 for decorator in node.decorators:
1000 decorator.visit(self)
1001 returns = node.returns
1002 if returns:
1003 returns.visit(self)
1005 return False
1007 def visit_Lambda(self, node: cst.Lambda) -> Optional[bool]:
1008 with self._new_scope(FunctionScope, node):
1009 node.params.visit(self)
1010 node.body.visit(self)
1011 return False
1013 def visit_Param(self, node: cst.Param) -> Optional[bool]:
1014 self.scope.record_assignment(node.name.value, node)
1015 self.provider.set_metadata(node.name, self.scope)
1016 with self._switch_scope(self.scope.parent):
1017 for field in [node.default, node.annotation]:
1018 if field:
1019 field.visit(self)
1021 return False
1023 def visit_Arg(self, node: cst.Arg) -> bool:
1024 # The keyword of Arg is neither an Assignment nor an Access and we explicitly don't visit it.
1025 value = node.value
1026 if value:
1027 value.visit(self)
1028 return False
1030 def visit_ClassDef(self, node: cst.ClassDef) -> Optional[bool]:
1031 self.scope.record_assignment(node.name.value, node)
1032 self.provider.set_metadata(node.name, self.scope)
1033 for decorator in node.decorators:
1034 decorator.visit(self)
1035 for base in node.bases:
1036 base.visit(self)
1037 for keyword in node.keywords:
1038 keyword.visit(self)
1040 with self._new_scope(ClassScope, node, get_full_name_for_node(node.name)):
1041 for statement in node.body.body:
1042 statement.visit(self)
1043 return False
1045 def visit_ClassDef_bases(self, node: cst.ClassDef) -> None:
1046 self.__ignore_annotation += 1
1048 def leave_ClassDef_bases(self, node: cst.ClassDef) -> None:
1049 self.__ignore_annotation -= 1
1051 def visit_Global(self, node: cst.Global) -> Optional[bool]:
1052 for name_item in node.names:
1053 self.scope.record_global_overwrite(name_item.name.value)
1054 return False
1056 def visit_Nonlocal(self, node: cst.Nonlocal) -> Optional[bool]:
1057 for name_item in node.names:
1058 self.scope.record_nonlocal_overwrite(name_item.name.value)
1059 return False
1061 def visit_ListComp(self, node: cst.ListComp) -> Optional[bool]:
1062 return self._visit_comp_alike(node)
1064 def visit_SetComp(self, node: cst.SetComp) -> Optional[bool]:
1065 return self._visit_comp_alike(node)
1067 def visit_DictComp(self, node: cst.DictComp) -> Optional[bool]:
1068 return self._visit_comp_alike(node)
1070 def visit_GeneratorExp(self, node: cst.GeneratorExp) -> Optional[bool]:
1071 return self._visit_comp_alike(node)
1073 def _visit_comp_alike(
1074 self, node: Union[cst.ListComp, cst.SetComp, cst.DictComp, cst.GeneratorExp]
1075 ) -> bool:
1076 """
1077 Cheat sheet: `[elt for target in iter if ifs]`
1079 Terminology:
1080 target: The variable or pattern we're storing each element of the iter in.
1081 iter: The thing we're iterating over.
1082 ifs: A list of conditions provided
1083 elt: The value that will be computed and "yielded" each time the loop
1084 iterates. For most comprehensions, this is just the `node.elt`, but
1085 DictComp has `key` and `value`, which behave like `node.elt` would.
1088 Nested Comprehension: ``[a for b in c for a in b]`` is a "nested" ListComp.
1089 The outer iterator is in ``node.for_in`` and the inner iterator is in
1090 ``node.for_in.inner_for_in``.
1093 The first comprehension object's iter in generators is evaluated
1094 outside of the ComprehensionScope. Every other comprehension's iter is
1095 evaluated inside the ComprehensionScope. Even though that doesn't seem very sane,
1096 but that appears to be how it works.
1098 non_flat = [ [1,2,3], [4,5,6], [7,8]
1099 flat = [y for x in non_flat for y in x] # this works fine
1101 # This will give a "NameError: name 'x' is not defined":
1102 flat = [y for x in x for y in x]
1103 # x isn't defined, because the first iter is evaluted outside the scope.
1105 # This will give an UnboundLocalError, indicating that the second
1106 # comprehension's iter value is evaluated inside the scope as its elt.
1107 # UnboundLocalError: local variable 'y' referenced before assignment
1108 flat = [y for x in non_flat for y in y]
1109 """
1110 for_in = node.for_in
1111 for_in.iter.visit(self)
1112 self.provider.set_metadata(for_in, self.scope)
1113 with self._new_scope(ComprehensionScope, node):
1114 for_in.target.visit(self)
1115 # Things from here on can refer to the target.
1116 self.scope._assignment_count += 1
1117 for condition in for_in.ifs:
1118 condition.visit(self)
1119 inner_for_in = for_in.inner_for_in
1120 if inner_for_in:
1121 inner_for_in.visit(self)
1122 if isinstance(node, cst.DictComp):
1123 node.key.visit(self)
1124 node.value.visit(self)
1125 else:
1126 node.elt.visit(self)
1127 return False
1129 def visit_For(self, node: cst.For) -> Optional[bool]:
1130 node.target.visit(self)
1131 self.scope._assignment_count += 1
1132 for child in [node.iter, node.body, node.orelse, node.asynchronous]:
1133 if child is not None:
1134 child.visit(self)
1135 return False
1137 def infer_accesses(self) -> None:
1138 # Aggregate access with the same name and batch add with set union as an optimization.
1139 # In worst case, all accesses (m) and assignments (n) refer to the same name,
1140 # the time complexity is O(m x n), this optimizes it as O(m + n).
1141 scope_name_accesses = defaultdict(set)
1142 for def_access in self.__deferred_accesses:
1143 access, enclosing_attribute, enclosing_string_annotation = (
1144 def_access.access,
1145 def_access.enclosing_attribute,
1146 def_access.enclosing_string_annotation,
1147 )
1148 name = ensure_type(access.node, cst.Name).value
1149 if enclosing_attribute is not None:
1150 # if _gen_dotted_names doesn't generate any values, fall back to
1151 # the original name node above
1152 for attr_name, node in _gen_dotted_names(enclosing_attribute):
1153 if attr_name in access.scope:
1154 access.node = node
1155 name = attr_name
1156 break
1158 if enclosing_string_annotation is not None:
1159 access.node = enclosing_string_annotation
1161 scope_name_accesses[(access.scope, name)].add(access)
1162 access.record_assignments(name)
1163 access.scope.record_access(name, access)
1165 for (scope, name), accesses in scope_name_accesses.items():
1166 for assignment in scope[name]:
1167 assignment.record_accesses(accesses)
1169 self.__deferred_accesses = []
1171 def on_leave(self, original_node: cst.CSTNode) -> None:
1172 self.provider.set_metadata(original_node, self.scope)
1173 if isinstance(original_node, _ASSIGNMENT_LIKE_NODES):
1174 self.scope._assignment_count += 1
1175 super().on_leave(original_node)
1178class ScopeProvider(BatchableMetadataProvider[Optional[Scope]]):
1179 """
1180 :class:`ScopeProvider` traverses the entire module and creates the scope inheritance
1181 structure. It provides the scope of name assignment and accesses. It is useful for
1182 more advanced static analysis. E.g. given a :class:`~libcst.FunctionDef`
1183 node, we can check the type of its Scope to figure out whether it is a class method
1184 (:class:`ClassScope`) or a regular function (:class:`GlobalScope`).
1186 Scope metadata is available for most node types other than formatting information nodes
1187 (whitespace, parentheses, etc.).
1188 """
1190 METADATA_DEPENDENCIES = (ExpressionContextProvider,)
1192 def visit_Module(self, node: cst.Module) -> Optional[bool]:
1193 visitor = ScopeVisitor(self)
1194 node.visit(visitor)
1195 visitor.infer_accesses()