Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# Licensed under the LGPL: https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html
2# For details: https://github.com/pylint-dev/astroid/blob/main/LICENSE
3# Copyright (c) https://github.com/pylint-dev/astroid/blob/main/CONTRIBUTORS.txt
5"""Astroid hooks for the Python standard library."""
7from __future__ import annotations
9import functools
10import keyword
11import unicodedata
12from collections.abc import Iterator
13from textwrap import dedent
14from typing import Final
16from astroid import arguments, bases, nodes, util
17from astroid.builder import AstroidBuilder, _extract_single_node, extract_node
18from astroid.context import InferenceContext
19from astroid.exceptions import (
20 AstroidError,
21 AstroidTypeError,
22 AstroidValueError,
23 InferenceError,
24 MroError,
25 UseInferenceDefault,
26)
27from astroid.inference_tip import inference_tip
28from astroid.manager import AstroidManager
29from astroid.nodes.scoped_nodes.scoped_nodes import SYNTHETIC_ROOT
31ENUM_QNAME: Final[str] = "enum.Enum"
32TYPING_NAMEDTUPLE_QUALIFIED: Final = {
33 "typing.NamedTuple",
34 "typing_extensions.NamedTuple",
35}
36TYPING_NAMEDTUPLE_BASENAMES: Final = {
37 "NamedTuple",
38 "typing.NamedTuple",
39 "typing_extensions.NamedTuple",
40}
43def _infer_first(node, context):
44 if isinstance(node, util.UninferableBase):
45 raise UseInferenceDefault
46 try:
47 value = next(node.infer(context=context))
48 except StopIteration as exc:
49 raise InferenceError from exc
50 if isinstance(value, util.UninferableBase):
51 raise UseInferenceDefault()
52 return value
55def _find_func_form_arguments(node, context):
56 def _extract_namedtuple_arg_or_keyword( # pylint: disable=inconsistent-return-statements
57 position, key_name=None
58 ):
59 if len(args) > position:
60 return _infer_first(args[position], context)
61 if key_name and key_name in found_keywords:
62 return _infer_first(found_keywords[key_name], context)
64 args = node.args
65 keywords = node.keywords
66 found_keywords = (
67 {keyword.arg: keyword.value for keyword in keywords} if keywords else {}
68 )
70 name = _extract_namedtuple_arg_or_keyword(position=0, key_name="typename")
71 names = _extract_namedtuple_arg_or_keyword(position=1, key_name="field_names")
72 if name and names:
73 return name.value, names
75 raise UseInferenceDefault()
78def infer_func_form(
79 node: nodes.Call,
80 base_type: nodes.NodeNG,
81 *,
82 parent: nodes.NodeNG,
83 context: InferenceContext | None = None,
84 enum: bool = False,
85) -> tuple[nodes.ClassDef, str, list[str]]:
86 """Specific inference function for namedtuple or Python 3 enum."""
87 # node is a Call node, class name as first argument and generated class
88 # attributes as second argument
90 # namedtuple or enums list of attributes can be a list of strings or a
91 # whitespace-separate string
92 try:
93 name, names = _find_func_form_arguments(node, context)
94 try:
95 attributes: list[str] = names.value.replace(",", " ").split()
96 except (AttributeError, TypeError) as exc:
97 # ``names`` is not a whitespace-separated string: it is either a
98 # container node, which has no ``value`` (AttributeError), or a
99 # constant of another type such as bytes (TypeError).
101 # Handle attributes of NamedTuples
102 if not enum:
103 attributes = []
104 fields = _get_namedtuple_fields(node)
105 if fields:
106 fields_node = extract_node(fields)
107 attributes = [
108 _infer_first(const, context).value for const in fields_node.elts
109 ]
111 # Handle attributes of Enums
112 else:
113 # Enums supports either iterator of (name, value) pairs
114 # or mappings.
115 if hasattr(names, "items") and isinstance(names.items, list):
116 attributes = [
117 _infer_first(const[0], context).value
118 for const in names.items
119 if isinstance(const[0], nodes.Const)
120 ]
121 elif hasattr(names, "elts"):
122 # Enums can support either ["a", "b", "c"]
123 # or [("a", 1), ("b", 2), ...], but they can't
124 # be mixed.
125 if all(isinstance(const, nodes.Tuple) for const in names.elts):
126 attributes = [
127 _infer_first(const.elts[0], context).value
128 for const in names.elts
129 if isinstance(const, nodes.Tuple)
130 ]
131 else:
132 attributes = [
133 _infer_first(const, context).value for const in names.elts
134 ]
135 else:
136 raise AttributeError from exc
137 if not attributes:
138 raise AttributeError from exc
139 except (AttributeError, InferenceError) as exc:
140 raise UseInferenceDefault from exc
142 if not enum:
143 # namedtuple maps sys.intern(str()) over over field_names
144 attributes = [str(attr) for attr in attributes]
145 # XXX this should succeed *unless* __str__/__repr__ is incorrect or throws
146 # in which case we should not have inferred these values and raised earlier
147 if any(not isinstance(attr, str) for attr in attributes):
148 # Enum members must be named by strings; a non-string attribute (e.g.
149 # ``Enum("e", (1,))``) means the definition is invalid, so fall back to
150 # the default inference instead of crashing.
151 raise UseInferenceDefault
152 if enum and not isinstance(name, str):
153 # Enum class names must be strings; inferring a class with any other
154 # name can make consumers crash when they perform string operations.
155 raise UseInferenceDefault
156 attributes = [attr for attr in attributes if " " not in attr]
158 # If we can't infer the name of the class, don't crash, up to this point
159 # we know it is a namedtuple anyway.
160 name = name or "Uninferable"
161 # we want to return a Class node instance with proper attributes set
162 class_node = nodes.ClassDef(
163 name,
164 lineno=node.lineno,
165 col_offset=node.col_offset,
166 end_lineno=node.end_lineno,
167 end_col_offset=node.end_col_offset,
168 parent=parent,
169 )
170 class_node.postinit(
171 bases=[base_type],
172 body=[],
173 decorators=None,
174 )
175 # XXX add __init__(*attributes) method
176 for attr in attributes:
177 fake_node = nodes.EmptyNode()
178 fake_node.parent = class_node
179 fake_node.attrname = attr
180 class_node.instance_attrs[attr] = [fake_node]
181 return class_node, name, attributes
184def _has_namedtuple_base(node):
185 """Predicate for class inference tip.
187 :type node: ClassDef
188 :rtype: bool
189 """
190 return set(node.basenames) & TYPING_NAMEDTUPLE_BASENAMES
193def _looks_like(node, name) -> bool:
194 func = node.func
195 if isinstance(func, nodes.Attribute):
196 return func.attrname == name
197 if isinstance(func, nodes.Name):
198 return func.name == name
199 return False
202_looks_like_namedtuple = functools.partial(_looks_like, name="namedtuple")
203_looks_like_enum = functools.partial(_looks_like, name="Enum")
204_looks_like_typing_namedtuple = functools.partial(_looks_like, name="NamedTuple")
207def infer_named_tuple(
208 node: nodes.Call, context: InferenceContext | None = None
209) -> Iterator[nodes.ClassDef]:
210 """Specific inference function for namedtuple Call node."""
211 tuple_base: nodes.Name = _extract_single_node("tuple")
212 class_node, name, attributes = infer_func_form(
213 node, tuple_base, parent=SYNTHETIC_ROOT, context=context
214 )
216 call_site = arguments.CallSite.from_call(node, context=context)
217 func = util.safe_infer(
218 _extract_single_node("import collections; collections.namedtuple")
219 )
220 assert isinstance(func, nodes.NodeNG)
221 try:
222 rename_arg_bool_value = next(
223 call_site.infer_argument(func, "rename", context or InferenceContext())
224 ).bool_value()
225 rename = rename_arg_bool_value is True
226 except (InferenceError, StopIteration):
227 rename = False
229 try:
230 attributes = _check_namedtuple_attributes(name, attributes, rename)
231 except AstroidTypeError as exc:
232 raise UseInferenceDefault("TypeError: " + str(exc)) from exc
233 except AstroidValueError as exc:
234 raise UseInferenceDefault("ValueError: " + str(exc)) from exc
236 if tuple(class_node.instance_attrs) != tuple(attributes):
237 # ``infer_func_form`` recorded the field names as written, but ``rename=True``
238 # replaces invalid, keyword or duplicate names with ``_N``, so the instance
239 # would otherwise carry names the namedtuple does not actually have (and lose
240 # the duplicated ones entirely). Rebuild them from the renamed fields.
241 class_node.instance_attrs.clear()
242 for attr in attributes:
243 fake_node = nodes.EmptyNode()
244 fake_node.parent = class_node
245 fake_node.attrname = attr
246 class_node.instance_attrs[attr] = [fake_node]
248 replace_args = ", ".join(f"{arg}=None" for arg in attributes)
249 field_def = (
250 " {name} = property(lambda self: self[{index:d}], "
251 "doc='Alias for field number {index:d}')"
252 )
253 field_defs = "\n".join(
254 field_def.format(name=name, index=index)
255 for index, name in enumerate(attributes)
256 )
257 fake = AstroidBuilder(AstroidManager()).string_build(f"""
258class {name}(tuple):
259 __slots__ = ()
260 _fields = {attributes!r}
261 def _asdict(self):
262 return self.__dict__
263 @classmethod
264 def _make(cls, iterable, new=tuple.__new__, len=len):
265 return new(cls, iterable)
266 def _replace(self, {replace_args}):
267 return self
268 def __getnewargs__(self):
269 return tuple(self)
270{field_defs}
271 """)
272 class_node.locals["_asdict"] = fake.body[0].locals["_asdict"]
273 class_node.locals["_make"] = fake.body[0].locals["_make"]
274 class_node.locals["_replace"] = fake.body[0].locals["_replace"]
275 class_node.locals["_fields"] = fake.body[0].locals["_fields"]
276 for attr in attributes:
277 # Python normalises identifiers to NFKC, so a field named "\u00b5" (MICRO SIGN)
278 # is stored by the parser as "\u03bc" (GREEK SMALL LETTER MU). Looking the
279 # attribute up under the name as written raises KeyError on a definition
280 # namedtuple itself accepts, so use the name the parser actually used.
281 normalized = unicodedata.normalize("NFKC", attr)
282 class_node.locals[normalized] = fake.body[0].locals[normalized]
283 # we use UseInferenceDefault, we can't be a generator so return an iterator
284 return iter([class_node])
287def _get_renamed_namedtuple_attributes(field_names):
288 names = list(field_names)
289 seen = set()
290 for i, name in enumerate(field_names):
291 # pylint: disable = too-many-boolean-expressions
292 if (
293 not all(c.isalnum() or c == "_" for c in name)
294 or keyword.iskeyword(name)
295 or not name
296 or name[0].isdigit()
297 or name.startswith("_")
298 or name in seen
299 ):
300 names[i] = f"_{i}"
301 seen.add(name)
302 return tuple(names)
305def _check_namedtuple_attributes(typename, attributes, rename=False):
306 attributes = tuple(attributes)
307 if rename:
308 attributes = _get_renamed_namedtuple_attributes(attributes)
310 # The following snippet is derived from the CPython Lib/collections/__init__.py sources
311 # <snippet>
312 for name in (typename, *attributes):
313 if not isinstance(name, str):
314 raise AstroidTypeError(
315 f"Type names and field names must be strings, not {type(name)!r}"
316 )
317 if not name.isidentifier():
318 raise AstroidValueError(
319 "Type names and field names must be valid" + f"identifiers: {name!r}"
320 )
321 if keyword.iskeyword(name):
322 raise AstroidValueError(
323 f"Type names and field names cannot be a keyword: {name!r}"
324 )
326 seen = set()
327 for name in attributes:
328 if name.startswith("_") and not rename:
329 raise AstroidValueError(
330 f"Field names cannot start with an underscore: {name!r}"
331 )
332 if name in seen:
333 raise AstroidValueError(f"Encountered duplicate field name: {name!r}")
334 seen.add(name)
335 # </snippet>
337 return attributes
340def infer_enum(
341 node: nodes.Call, context: InferenceContext | None = None
342) -> Iterator[bases.Instance]:
343 """Specific inference function for enum Call node."""
344 # Raise `UseInferenceDefault` if `node` is a call to a a user-defined Enum.
345 try:
346 inferred = node.func.infer(context)
347 except (InferenceError, StopIteration) as exc:
348 raise UseInferenceDefault from exc
350 if not any(
351 isinstance(item, nodes.ClassDef) and item.qname() == ENUM_QNAME
352 for item in inferred
353 ):
354 raise UseInferenceDefault
356 enum_meta = _extract_single_node("""
357 class EnumMeta(object):
358 'docstring'
359 def __call__(self, node):
360 class EnumAttribute(object):
361 name = ''
362 value = 0
363 return EnumAttribute()
364 def __iter__(self):
365 class EnumAttribute(object):
366 name = ''
367 value = 0
368 return [EnumAttribute()]
369 def __reversed__(self):
370 class EnumAttribute(object):
371 name = ''
372 value = 0
373 return (EnumAttribute, )
374 def __next__(self):
375 return next(iter(self))
376 def __getitem__(self, attr):
377 class Value(object):
378 @property
379 def name(self):
380 return ''
381 @property
382 def value(self):
383 return attr
385 return Value()
386 __members__ = ['']
387 """)
389 # FIXME arguably, the base here shouldn't be the EnumMeta class definition
390 # itself, but a reference (Name) to it. Otherwise, the invariant that all
391 # children of a node have that node as their parent is broken.
392 class_node = infer_func_form(
393 node,
394 enum_meta,
395 parent=SYNTHETIC_ROOT,
396 context=context,
397 enum=True,
398 )[0]
399 return iter([class_node.instantiate_class()])
402INT_FLAG_ADDITION_METHODS = """
403 def __or__(self, other):
404 return {name}(self.value | other.value)
405 def __and__(self, other):
406 return {name}(self.value & other.value)
407 def __xor__(self, other):
408 return {name}(self.value ^ other.value)
409 def __add__(self, other):
410 return {name}(self.value + other.value)
411 def __div__(self, other):
412 return {name}(self.value / other.value)
413 def __invert__(self):
414 return {name}(~self.value)
415 def __mul__(self, other):
416 return {name}(self.value * other.value)
417"""
420def infer_enum_class(node: nodes.ClassDef) -> nodes.ClassDef:
421 """Specific inference for enums."""
422 try:
423 mro = node.mro()
424 except MroError:
425 # A malformed class hierarchy (e.g. duplicate bases) has no resolvable
426 # MRO; leave the node untransformed rather than crashing.
427 return node
428 for basename in (b for cls in mro for b in cls.basenames):
429 if node.root().name == "enum":
430 # Skip if the class is directly from enum module.
431 break
432 dunder_members = {}
433 target_names = set()
434 for local, values in node.locals.items():
435 if (
436 any(not isinstance(value, nodes.AssignName) for value in values)
437 or local == "_ignore_"
438 ):
439 continue
441 stmt = values[0].statement()
442 if isinstance(stmt, nodes.Assign):
443 if isinstance(stmt.targets[0], nodes.Tuple):
444 targets = stmt.targets[0].itered()
445 else:
446 targets = stmt.targets
447 elif isinstance(stmt, nodes.AnnAssign):
448 targets = [stmt.target]
449 else:
450 continue
452 inferred_return_value = None
453 if stmt.value is not None:
454 if isinstance(stmt.value, nodes.Const):
455 if isinstance(stmt.value.value, str):
456 inferred_return_value = repr(stmt.value.value)
457 else:
458 inferred_return_value = stmt.value.value
459 else:
460 inferred_return_value = stmt.value.as_string()
462 new_targets = []
463 for target in targets:
464 if isinstance(target, nodes.Starred):
465 continue
466 target_names.add(target.name)
467 # Replace all the assignments with our mocked class.
468 classdef = dedent(
469 """
470 class {name}({types}):
471 @property
472 def value(self):
473 return {return_value}
474 @property
475 def _value_(self):
476 return {return_value}
477 @property
478 def name(self):
479 return "{name}"
480 @property
481 def _name_(self):
482 return "{name}"
483 """.format(
484 name=target.name,
485 types=", ".join(node.basenames),
486 return_value=inferred_return_value,
487 )
488 )
489 if "IntFlag" in basename:
490 # Alright, we need to add some additional methods.
491 # Unfortunately we still can't infer the resulting objects as
492 # Enum members, but once we'll be able to do that, the following
493 # should result in some nice symbolic execution
494 classdef += INT_FLAG_ADDITION_METHODS.format(name=target.name)
496 fake = AstroidBuilder(
497 AstroidManager(), apply_transforms=False
498 ).string_build(classdef)[target.name]
499 fake.parent = target.parent
500 for method in node.mymethods():
501 fake.locals[method.name] = [method]
502 new_targets.append(fake.instantiate_class())
503 if stmt.value is None:
504 continue
505 dunder_members[local] = fake
506 node.locals[local] = new_targets
508 # The undocumented `_value2member_map_` member:
509 node.locals["_value2member_map_"] = [
510 nodes.Dict(
511 parent=node,
512 lineno=node.lineno,
513 col_offset=node.col_offset,
514 end_lineno=node.end_lineno,
515 end_col_offset=node.end_col_offset,
516 )
517 ]
519 members = nodes.Dict(
520 parent=node,
521 lineno=node.lineno,
522 col_offset=node.col_offset,
523 end_lineno=node.end_lineno,
524 end_col_offset=node.end_col_offset,
525 )
526 members.postinit(
527 [
528 (
529 nodes.Const(k, parent=members),
530 nodes.Name(
531 v.name,
532 parent=members,
533 lineno=v.lineno,
534 col_offset=v.col_offset,
535 end_lineno=v.end_lineno,
536 end_col_offset=v.end_col_offset,
537 ),
538 )
539 for k, v in dunder_members.items()
540 ]
541 )
542 node.locals["__members__"] = [members]
543 # The enum.Enum class itself defines two @DynamicClassAttribute data-descriptors
544 # "name" and "value" (which we override in the mocked class for each enum member
545 # above). When dealing with inference of an arbitrary instance of the enum
546 # class, e.g. in a method defined in the class body like:
547 # class SomeEnum(enum.Enum):
548 # def method(self):
549 # self.name # <- here
550 # In the absence of an enum member called "name" or "value", these attributes
551 # should resolve to the descriptor on that particular instance, i.e. enum member.
552 # For "value", we have no idea what that should be, but for "name", we at least
553 # know that it should be a string, so infer that as a guess.
554 if "name" not in target_names:
555 code = dedent('''
556 @property
557 def name(self):
558 """The name of the Enum member.
560 This is a reconstruction by astroid: enums are too dynamic to understand, but we at least
561 know 'name' should be a string, so this is astroid's best guess.
562 """
563 return ''
564 ''')
565 name_dynamicclassattr = AstroidBuilder(AstroidManager()).string_build(code)[
566 "name"
567 ]
568 node.locals["name"] = [name_dynamicclassattr]
569 break
570 return node
573def infer_typing_namedtuple_class(class_node, context: InferenceContext | None = None):
574 """Infer a subclass of typing.NamedTuple."""
575 # Check if it has the corresponding bases
576 annassigns_fields = [
577 annassign.target.name
578 for annassign in class_node.body
579 if isinstance(annassign, nodes.AnnAssign)
580 ]
581 code = dedent("""
582 from collections import namedtuple
583 namedtuple({typename!r}, {fields!r})
584 """).format(typename=class_node.name, fields=",".join(annassigns_fields))
585 node = extract_node(code)
586 try:
587 generated_class_node = next(infer_named_tuple(node, context))
588 except StopIteration as e:
589 raise InferenceError(node=node, context=context) from e
590 for method in class_node.mymethods():
591 generated_class_node.locals[method.name] = [method]
593 for body_node in class_node.body:
594 if isinstance(body_node, nodes.Assign):
595 for target in body_node.targets:
596 # A target is not necessarily a single name: ``cat.color = ...``
597 # and ``basket[0] = ...`` define no class attribute at all, while
598 # ``apple, banana = ...`` defines one per unpacked element.
599 for assign_name in target.nodes_of_class(nodes.AssignName):
600 attr = assign_name.name
601 generated_class_node.locals[attr] = class_node.locals[attr]
602 elif isinstance(body_node, nodes.ClassDef):
603 generated_class_node.locals[body_node.name] = [body_node]
605 return iter((generated_class_node,))
608def infer_typing_namedtuple_function(node, context: InferenceContext | None = None):
609 """
610 Starting with python3.9, NamedTuple is a function of the typing module.
611 The class NamedTuple is build dynamically through a call to `type` during
612 initialization of the `_NamedTuple` variable.
613 """
614 klass = extract_node("""
615 from typing import _NamedTuple
616 _NamedTuple
617 """)
618 return klass.infer(context)
621def infer_typing_namedtuple(
622 node: nodes.Call, context: InferenceContext | None = None
623) -> Iterator[nodes.ClassDef]:
624 """Infer a typing.NamedTuple(...) call."""
625 # This is essentially a namedtuple with different arguments
626 # so we extract the args and infer a named tuple.
627 try:
628 func = next(node.func.infer())
629 except (InferenceError, StopIteration) as exc:
630 raise UseInferenceDefault from exc
632 if func.qname() not in TYPING_NAMEDTUPLE_QUALIFIED:
633 raise UseInferenceDefault
635 if len(node.args) != 2:
636 raise UseInferenceDefault
638 if not isinstance(node.args[1], (nodes.List, nodes.Tuple)):
639 raise UseInferenceDefault
641 return infer_named_tuple(node, context)
644def _get_namedtuple_fields(node: nodes.Call) -> str:
645 """Get and return fields of a NamedTuple in code-as-a-string.
647 Because the fields are represented in their code form we can
648 extract a node from them later on.
649 """
650 names = []
651 container = None
652 try:
653 container = next(node.args[1].infer())
654 except (InferenceError, StopIteration) as exc:
655 raise UseInferenceDefault from exc
656 # We pass on IndexError as we'll try to infer 'field_names' from the keywords
657 except IndexError:
658 pass
659 if not container:
660 for keyword_node in node.keywords:
661 if keyword_node.arg == "field_names":
662 try:
663 container = next(keyword_node.value.infer())
664 except (InferenceError, StopIteration) as exc:
665 raise UseInferenceDefault from exc
666 break
667 if not isinstance(container, nodes.BaseContainer):
668 raise UseInferenceDefault
669 for elt in container.elts:
670 if isinstance(elt, nodes.Const):
671 names.append(elt.as_string())
672 continue
673 if not isinstance(elt, (nodes.List, nodes.Tuple)):
674 raise UseInferenceDefault
675 if len(elt.elts) != 2:
676 raise UseInferenceDefault
677 names.append(elt.elts[0].as_string())
679 if names:
680 field_names = f"({','.join(names)},)"
681 else:
682 field_names = ""
683 return field_names
686def _is_enum_subclass(cls: nodes.ClassDef) -> bool:
687 """Return whether cls is a subclass of an Enum."""
688 try:
689 return cls.is_subtype_of("enum.Enum")
690 except AstroidError:
691 return False
694def register(manager: AstroidManager) -> None:
695 manager.register_transform(
696 nodes.Call, inference_tip(infer_named_tuple), _looks_like_namedtuple
697 )
698 manager.register_transform(nodes.Call, inference_tip(infer_enum), _looks_like_enum)
699 manager.register_transform(
700 nodes.ClassDef, infer_enum_class, predicate=_is_enum_subclass
701 )
702 manager.register_transform(
703 nodes.ClassDef,
704 inference_tip(infer_typing_namedtuple_class),
705 _has_namedtuple_base,
706 )
707 manager.register_transform(
708 nodes.FunctionDef,
709 inference_tip(infer_typing_namedtuple_function),
710 lambda node: node.name == "NamedTuple"
711 and getattr(node.root(), "name", None) == "typing",
712 )
713 manager.register_transform(
714 nodes.Call,
715 inference_tip(infer_typing_namedtuple),
716 _looks_like_typing_namedtuple,
717 )