Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/libcst/_nodes/base.py: 60%
177 statements
« prev ^ index » next coverage.py v7.3.1, created at 2023-09-25 06:43 +0000
1# Copyright (c) Meta Platforms, Inc. and affiliates.
2#
3# This source code is licensed under the MIT license found in the
4# LICENSE file in the root directory of this source tree.
6from abc import ABC, abstractmethod
7from copy import deepcopy
8from dataclasses import dataclass, field, fields, replace
9from typing import Any, cast, ClassVar, Dict, List, Mapping, Sequence, TypeVar, Union
11from libcst._flatten_sentinel import FlattenSentinel
12from libcst._nodes.internal import CodegenState
13from libcst._removal_sentinel import RemovalSentinel
14from libcst._type_enforce import is_value_of_type
15from libcst._types import CSTNodeT
16from libcst._visitors import CSTTransformer, CSTVisitor, CSTVisitorT
18_CSTNodeSelfT = TypeVar("_CSTNodeSelfT", bound="CSTNode")
19_EMPTY_SEQUENCE: Sequence["CSTNode"] = ()
22class CSTValidationError(SyntaxError):
23 pass
26class CSTCodegenError(SyntaxError):
27 pass
30class _ChildrenCollectionVisitor(CSTVisitor):
31 def __init__(self) -> None:
32 self.children: List[CSTNode] = []
34 def on_visit(self, node: "CSTNode") -> bool:
35 self.children.append(node)
36 return False # Don't include transitive children
39class _ChildReplacementTransformer(CSTTransformer):
40 def __init__(
41 self, old_node: "CSTNode", new_node: Union["CSTNode", RemovalSentinel]
42 ) -> None:
43 self.old_node = old_node
44 self.new_node = new_node
46 def on_visit(self, node: "CSTNode") -> bool:
47 # If the node is one we are about to replace, we shouldn't
48 # recurse down it, that would be a waste of time.
49 return node is not self.old_node
51 def on_leave(
52 self, original_node: "CSTNode", updated_node: "CSTNode"
53 ) -> Union["CSTNode", RemovalSentinel]:
54 if original_node is self.old_node:
55 return self.new_node
56 return updated_node
59class _ChildWithChangesTransformer(CSTTransformer):
60 def __init__(self, old_node: "CSTNode", changes: Mapping[str, Any]) -> None:
61 self.old_node = old_node
62 self.changes = changes
64 def on_visit(self, node: "CSTNode") -> bool:
65 # If the node is one we are about to replace, we shouldn't
66 # recurse down it, that would be a waste of time.
67 return node is not self.old_node
69 def on_leave(self, original_node: "CSTNode", updated_node: "CSTNode") -> "CSTNode":
70 if original_node is self.old_node:
71 return updated_node.with_changes(**self.changes)
72 return updated_node
75class _NOOPVisitor(CSTTransformer):
76 pass
79def _pretty_repr(value: object) -> str:
80 if not isinstance(value, str) and isinstance(value, Sequence):
81 return _pretty_repr_sequence(value)
82 else:
83 return repr(value)
86def _pretty_repr_sequence(seq: Sequence[object]) -> str:
87 if len(seq) == 0:
88 return "[]"
89 else:
90 return "\n".join(["[", *[f"{_indent(repr(el))}," for el in seq], "]"])
93def _indent(value: str) -> str:
94 return "\n".join(f" {line}" for line in value.split("\n"))
97def _clone(val: object) -> object:
98 # We can't use isinstance(val, CSTNode) here due to poor performance
99 # of isinstance checks against ABC direct subclasses. What we're trying
100 # to do here is recursively call this functionality on subclasses, but
101 # if the attribute isn't a CSTNode, fall back to copy.deepcopy.
102 try:
103 # pyre-ignore We know this might not exist, that's the point of the
104 # attribute error and try block.
105 return val.deep_clone()
106 except AttributeError:
107 return deepcopy(val)
110@dataclass(frozen=True)
111class CSTNode(ABC):
112 __slots__: ClassVar[Sequence[str]] = ()
114 def __post_init__(self) -> None:
115 # PERF: It might make more sense to move validation work into the visitor, which
116 # would allow us to avoid validating the tree when parsing a file.
117 self._validate()
119 @classmethod
120 def __init_subclass__(cls, **kwargs: Any) -> None:
121 """
122 HACK: Add our implementation of `__repr__`, `__hash__`, and `__eq__` to the
123 class's __dict__ to prevent dataclass from generating it's own `__repr__`,
124 `__hash__`, and `__eq__`.
126 The alternative is to require each implementation of a node to remember to add
127 `repr=False, eq=False`, which is more error-prone.
128 """
129 super().__init_subclass__(**kwargs)
131 if "__repr__" not in cls.__dict__:
132 cls.__repr__ = CSTNode.__repr__
133 if "__eq__" not in cls.__dict__:
134 cls.__eq__ = CSTNode.__eq__
135 if "__hash__" not in cls.__dict__:
136 cls.__hash__ = CSTNode.__hash__
138 def _validate(self) -> None:
139 """
140 Override this to perform runtime validation of a newly created node.
142 The function is called during `__init__`. It should check for possible mistakes
143 that wouldn't be caught by a static type checker.
145 If you can't use a static type checker, and want to perform a runtime validation
146 of this node's types, use `validate_types` instead.
147 """
148 pass
150 def validate_types_shallow(self) -> None:
151 """
152 Compares the type annotations on a node's fields with those field's actual
153 values at runtime. Raises a TypeError is a mismatch is found.
155 Only validates the current node, not any of it's children. For a recursive
156 version, see :func:`validate_types_deep`.
158 If you're using a static type checker (highly recommended), this is useless.
159 However, if your code doesn't use a static type checker, or if you're unable to
160 statically type your code for some reason, you can use this method to help
161 validate your tree.
163 Some (non-typing) validation is done unconditionally during the construction of
164 a node. That validation does not overlap with the work that
165 :func:`validate_types_deep` does.
166 """
167 for f in fields(self):
168 value = getattr(self, f.name)
169 if not is_value_of_type(value, f.type):
170 raise TypeError(
171 f"Expected an instance of {f.type!r} on "
172 + f"{type(self).__name__}'s '{f.name}' field, but instead got "
173 + f"an instance of {type(value)!r}"
174 )
176 def validate_types_deep(self) -> None:
177 """
178 Like :func:`validate_types_shallow`, but recursively validates the whole tree.
179 """
180 self.validate_types_shallow()
181 for ch in self.children:
182 ch.validate_types_deep()
184 @property
185 def children(self) -> Sequence["CSTNode"]:
186 """
187 The immediate (not transitive) child CSTNodes of the current node. Various
188 properties on the nodes, such as string values, will not be visited if they are
189 not a subclass of CSTNode.
191 Iterable properties of the node (e.g. an IndentedBlock's body) will be flattened
192 into the children's sequence.
194 The children will always be returned in the same order that they appear
195 lexically in the code.
196 """
198 # We're hooking into _visit_and_replace_children, which means that our current
199 # implementation is slow. We may need to rethink and/or cache this if it becomes
200 # a frequently accessed property.
201 #
202 # This probably won't be called frequently, because most child access will
203 # probably through visit, or directly through named property access, not through
204 # children.
206 visitor = _ChildrenCollectionVisitor()
207 self._visit_and_replace_children(visitor)
208 return visitor.children
210 def visit(
211 self: _CSTNodeSelfT, visitor: CSTVisitorT
212 ) -> Union[_CSTNodeSelfT, RemovalSentinel, FlattenSentinel[_CSTNodeSelfT]]:
213 """
214 Visits the current node, its children, and all transitive children using
215 the given visitor's callbacks.
216 """
217 # visit self
218 should_visit_children = visitor.on_visit(self)
220 # TODO: provide traversal where children are not replaced
221 # visit children (optionally)
222 if should_visit_children:
223 # It's not possible to define `_visit_and_replace_children` with the correct
224 # return type in any sane way, so we're using this cast. See the
225 # explanation above the declaration of `_visit_and_replace_children`.
226 with_updated_children = cast(
227 _CSTNodeSelfT, self._visit_and_replace_children(visitor)
228 )
229 else:
230 with_updated_children = self
232 if isinstance(visitor, CSTVisitor):
233 visitor.on_leave(self)
234 leave_result = self
235 else:
236 leave_result = visitor.on_leave(self, with_updated_children)
238 # validate return type of the user-defined `visitor.on_leave` method
239 if not isinstance(leave_result, (CSTNode, RemovalSentinel, FlattenSentinel)):
240 raise Exception(
241 "Expected a node of type CSTNode or a RemovalSentinel, "
242 + f"but got a return value of {type(leave_result).__name__}"
243 )
245 # TODO: Run runtime typechecks against updated nodes
247 return leave_result
249 # The return type of `_visit_and_replace_children` is `CSTNode`, not
250 # `_CSTNodeSelfT`. This is because pyre currently doesn't have a way to annotate
251 # classes as final. https://mypy.readthedocs.io/en/latest/final_attrs.html
252 #
253 # The issue is that any reasonable implementation of `_visit_and_replace_children`
254 # needs to refer to the class' own constructor:
255 #
256 # class While(CSTNode):
257 # def _visit_and_replace_children(self, visitor: CSTVisitorT) -> While:
258 # return While(...)
259 #
260 # You'll notice that because this implementation needs to call the `While`
261 # constructor, the return type is also `While`. This function is a valid subtype of
262 # `Callable[[CSTVisitorT], CSTNode]`.
263 #
264 # It is not a valid subtype of `Callable[[CSTVisitorT], _CSTNodeSelfT]`. That's
265 # because the return type of this function wouldn't be valid for any subclasses.
266 # In practice, that's not an issue, because we don't have any subclasses of `While`,
267 # but there's no way to tell pyre that without a `@final` annotation.
268 #
269 # Instead, we're just relying on an unchecked call to `cast()` in the `visit`
270 # method.
271 @abstractmethod
272 def _visit_and_replace_children(self, visitor: CSTVisitorT) -> "CSTNode":
273 """
274 Intended to be overridden by subclasses to provide a low-level hook for the
275 visitor API.
277 Don't call this directly. Instead, use `visitor.visit_and_replace_node` or
278 `visitor.visit_and_replace_module`. If you need list of children, access the
279 `children` property instead.
281 The general expectation is that children should be visited in the order in which
282 they appear lexically.
283 """
284 ...
286 def _is_removable(self) -> bool:
287 """
288 Intended to be overridden by nodes that will be iterated over inside
289 Module and IndentedBlock. Returning true signifies that this node is
290 essentially useless and can be dropped when doing a visit across it.
291 """
292 return False
294 @abstractmethod
295 def _codegen_impl(self, state: CodegenState) -> None:
296 ...
298 def _codegen(self, state: CodegenState, **kwargs: Any) -> None:
299 state.before_codegen(self)
300 self._codegen_impl(state, **kwargs)
301 state.after_codegen(self)
303 def with_changes(self: _CSTNodeSelfT, **changes: Any) -> _CSTNodeSelfT:
304 """
305 A convenience method for performing mutation-like operations on immutable nodes.
306 Creates a new object of the same type, replacing fields with values from the
307 supplied keyword arguments.
309 For example, to update the test of an if conditional, you could do::
311 def leave_If(self, original_node: cst.If, updated_node: cst.If) -> cst.If:
312 new_node = updated_node.with_changes(test=new_conditional)
313 return new_node
315 ``new_node`` will have the same ``body``, ``orelse``, and whitespace fields as
316 ``updated_node``, but with the updated ``test`` field.
318 The accepted arguments match the arguments given to ``__init__``, however there
319 are no required or positional arguments.
321 TODO: This API is untyped. There's probably no sane way to type it using pyre's
322 current feature-set, but we should still think about ways to type this or a
323 similar API in the future.
324 """
325 return replace(self, **changes)
327 def deep_clone(self: _CSTNodeSelfT) -> _CSTNodeSelfT:
328 """
329 Recursively clone the entire tree. The created tree is a new tree has the same
330 representation but different identity.
332 >>> tree = cst.parse_expression("1+2")
334 >>> tree.deep_clone() == tree
335 False
337 >>> tree == tree
338 True
340 >>> tree.deep_equals(tree.deep_clone())
341 True
342 """
343 cloned_fields: Dict[str, object] = {}
344 for field in fields(self):
345 key = field.name
346 if key[0] == "_":
347 continue
348 val = getattr(self, key)
350 # Much like the comment on _clone itself, we are allergic to instance
351 # checks against Sequence because of speed issues with ABC classes. So,
352 # instead, first handle sequence types that we do not want to iterate on
353 # and then just try to iterate and clone.
354 if isinstance(val, (str, bytes)):
355 cloned_fields[key] = _clone(val)
356 else:
357 try:
358 cloned_fields[key] = tuple(_clone(v) for v in val)
359 except TypeError:
360 cloned_fields[key] = _clone(val)
362 return type(self)(**cloned_fields)
364 def deep_equals(self, other: "CSTNode") -> bool:
365 """
366 Recursively inspects the entire tree under ``self`` and ``other`` to determine if
367 the two trees are equal by representation instead of identity (``==``).
368 """
369 from libcst._nodes.deep_equals import deep_equals as deep_equals_impl
371 return deep_equals_impl(self, other)
373 def deep_replace(
374 self: _CSTNodeSelfT, old_node: "CSTNode", new_node: CSTNodeT
375 ) -> Union[_CSTNodeSelfT, CSTNodeT]:
376 """
377 Recursively replaces any instance of ``old_node`` with ``new_node`` by identity.
378 Use this to avoid nested ``with_changes`` blocks when you are replacing one of
379 a node's deep children with a new node. Note that if you have previously
380 modified the tree in a way that ``old_node`` appears more than once as a deep
381 child, all instances will be replaced.
382 """
383 new_tree = self.visit(_ChildReplacementTransformer(old_node, new_node))
384 if isinstance(new_tree, (FlattenSentinel, RemovalSentinel)):
385 # The above transform never returns *Sentinel, so this isn't possible
386 raise Exception("Logic error, cannot get a *Sentinel here!")
387 return new_tree
389 def deep_remove(
390 self: _CSTNodeSelfT, old_node: "CSTNode"
391 ) -> Union[_CSTNodeSelfT, RemovalSentinel]:
392 """
393 Recursively removes any instance of ``old_node`` by identity. Note that if you
394 have previously modified the tree in a way that ``old_node`` appears more than
395 once as a deep child, all instances will be removed.
396 """
397 new_tree = self.visit(
398 _ChildReplacementTransformer(old_node, RemovalSentinel.REMOVE)
399 )
401 if isinstance(new_tree, FlattenSentinel):
402 # The above transform never returns FlattenSentinel, so this isn't possible
403 raise Exception("Logic error, cannot get a FlattenSentinel here!")
405 return new_tree
407 def with_deep_changes(
408 self: _CSTNodeSelfT, old_node: "CSTNode", **changes: Any
409 ) -> _CSTNodeSelfT:
410 """
411 A convenience method for applying :attr:`with_changes` to a child node. Use
412 this to avoid chains of :attr:`with_changes` or combinations of
413 :attr:`deep_replace` and :attr:`with_changes`.
415 The accepted arguments match the arguments given to the child node's
416 ``__init__``.
418 TODO: This API is untyped. There's probably no sane way to type it using pyre's
419 current feature-set, but we should still think about ways to type this or a
420 similar API in the future.
421 """
422 new_tree = self.visit(_ChildWithChangesTransformer(old_node, changes))
423 if isinstance(new_tree, (FlattenSentinel, RemovalSentinel)):
424 # This is impossible with the above transform.
425 raise Exception("Logic error, cannot get a *Sentinel here!")
426 return new_tree
428 def __eq__(self: _CSTNodeSelfT, other: object) -> bool:
429 """
430 CSTNodes are only treated as equal by identity. This matches the behavior of
431 CPython's AST nodes.
433 If you actually want to compare the value instead of the identity of the current
434 node with another, use `node.deep_equals`. Because `deep_equals` must traverse
435 the entire tree, it can have an unexpectedly large time complexity.
437 We're not exposing value equality as the default behavior because of
438 `deep_equals`'s large time complexity.
439 """
440 return self is other
442 def __hash__(self) -> int:
443 # Equality of nodes is based on identity, so the hash should be too.
444 return id(self)
446 def __repr__(self) -> str:
447 if len(fields(self)) == 0:
448 return f"{type(self).__name__}()"
450 lines = [f"{type(self).__name__}("]
451 for f in fields(self):
452 key = f.name
453 if key[0] != "_":
454 value = getattr(self, key)
455 lines.append(_indent(f"{key}={_pretty_repr(value)},"))
456 lines.append(")")
457 return "\n".join(lines)
459 @classmethod
460 # pyre-fixme[3]: Return annotation cannot be `Any`.
461 def field(cls, *args: object, **kwargs: object) -> Any:
462 """
463 A helper that allows us to easily use CSTNodes in dataclass constructor
464 defaults without accidentally aliasing nodes by identity across multiple
465 instances.
466 """
467 # pyre-ignore Pyre is complaining about CSTNode not being instantiable,
468 # but we're only going to call this from concrete subclasses.
469 return field(default_factory=lambda: cls(*args, **kwargs))
472class BaseLeaf(CSTNode, ABC):
473 __slots__ = ()
475 @property
476 def children(self) -> Sequence[CSTNode]:
477 # override this with an optimized implementation
478 return _EMPTY_SEQUENCE
480 def _visit_and_replace_children(
481 self: _CSTNodeSelfT, visitor: CSTVisitorT
482 ) -> _CSTNodeSelfT:
483 return self
486class BaseValueToken(BaseLeaf, ABC):
487 """
488 Represents the subset of nodes that only contain a value. Not all tokens from the
489 tokenizer will exist as BaseValueTokens. In places where the token is always a
490 constant value (e.g. a COLON token), the token's value will be implicitly folded
491 into the parent CSTNode, and hard-coded into the implementation of _codegen.
492 """
494 __slots__ = ()
496 value: str
498 def _codegen_impl(self, state: CodegenState) -> None:
499 state.add_token(self.value)