1# Licensed under the LGPL: https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html
2# For details: https://github.com/pylint-dev/astroid/blob/main/LICENSE
3# Copyright (c) https://github.com/pylint-dev/astroid/blob/main/CONTRIBUTORS.txt
4
5"""Astroid hooks for the Python standard library."""
6
7from __future__ import annotations
8
9import functools
10import keyword
11import unicodedata
12from collections.abc import Iterator
13from textwrap import dedent
14from typing import Final
15
16from astroid import arguments, bases, nodes, util
17from astroid.builder import AstroidBuilder, _extract_single_node, extract_node
18from astroid.context import InferenceContext
19from astroid.exceptions import (
20 AstroidError,
21 AstroidTypeError,
22 AstroidValueError,
23 InferenceError,
24 MroError,
25 UseInferenceDefault,
26)
27from astroid.inference_tip import inference_tip
28from astroid.manager import AstroidManager
29from astroid.nodes.scoped_nodes.scoped_nodes import SYNTHETIC_ROOT
30
31ENUM_QNAME: Final[str] = "enum.Enum"
32TYPING_NAMEDTUPLE_QUALIFIED: Final = {
33 "typing.NamedTuple",
34 "typing_extensions.NamedTuple",
35}
36TYPING_NAMEDTUPLE_BASENAMES: Final = {
37 "NamedTuple",
38 "typing.NamedTuple",
39 "typing_extensions.NamedTuple",
40}
41
42
43def _infer_first(node, context):
44 if isinstance(node, util.UninferableBase):
45 raise UseInferenceDefault
46 try:
47 value = next(node.infer(context=context))
48 except StopIteration as exc:
49 raise InferenceError from exc
50 if isinstance(value, util.UninferableBase):
51 raise UseInferenceDefault()
52 return value
53
54
55def _find_func_form_arguments(node, context):
56 def _extract_namedtuple_arg_or_keyword( # pylint: disable=inconsistent-return-statements
57 position, key_name=None
58 ):
59 if len(args) > position:
60 return _infer_first(args[position], context)
61 if key_name and key_name in found_keywords:
62 return _infer_first(found_keywords[key_name], context)
63
64 args = node.args
65 keywords = node.keywords
66 found_keywords = (
67 {keyword.arg: keyword.value for keyword in keywords} if keywords else {}
68 )
69
70 name = _extract_namedtuple_arg_or_keyword(position=0, key_name="typename")
71 names = _extract_namedtuple_arg_or_keyword(position=1, key_name="field_names")
72 if name and names:
73 return name.value, names
74
75 raise UseInferenceDefault()
76
77
78def infer_func_form(
79 node: nodes.Call,
80 base_type: nodes.NodeNG,
81 *,
82 parent: nodes.NodeNG,
83 context: InferenceContext | None = None,
84 enum: bool = False,
85) -> tuple[nodes.ClassDef, str, list[str]]:
86 """Specific inference function for namedtuple or Python 3 enum."""
87 # node is a Call node, class name as first argument and generated class
88 # attributes as second argument
89
90 # namedtuple or enums list of attributes can be a list of strings or a
91 # whitespace-separate string
92 try:
93 name, names = _find_func_form_arguments(node, context)
94 try:
95 attributes: list[str] = names.value.replace(",", " ").split()
96 except (AttributeError, TypeError) as exc:
97 # ``names`` is not a whitespace-separated string: it is either a
98 # container node, which has no ``value`` (AttributeError), or a
99 # constant of another type such as bytes (TypeError).
100
101 # Handle attributes of NamedTuples
102 if not enum:
103 attributes = []
104 fields = _get_namedtuple_fields(node)
105 if fields:
106 fields_node = extract_node(fields)
107 attributes = [
108 _infer_first(const, context).value for const in fields_node.elts
109 ]
110
111 # Handle attributes of Enums
112 else:
113 # Enums supports either iterator of (name, value) pairs
114 # or mappings.
115 if hasattr(names, "items") and isinstance(names.items, list):
116 attributes = [
117 _infer_first(const[0], context).value
118 for const in names.items
119 if isinstance(const[0], nodes.Const)
120 ]
121 elif hasattr(names, "elts"):
122 # Enums can support either ["a", "b", "c"]
123 # or [("a", 1), ("b", 2), ...], but they can't
124 # be mixed.
125 if all(isinstance(const, nodes.Tuple) for const in names.elts):
126 attributes = [
127 _infer_first(const.elts[0], context).value
128 for const in names.elts
129 if isinstance(const, nodes.Tuple)
130 ]
131 else:
132 attributes = [
133 _infer_first(const, context).value for const in names.elts
134 ]
135 else:
136 raise AttributeError from exc
137 if not attributes:
138 raise AttributeError from exc
139 except (AttributeError, InferenceError) as exc:
140 raise UseInferenceDefault from exc
141
142 if not enum:
143 # namedtuple maps sys.intern(str()) over over field_names
144 attributes = [str(attr) for attr in attributes]
145 # XXX this should succeed *unless* __str__/__repr__ is incorrect or throws
146 # in which case we should not have inferred these values and raised earlier
147 if any(not isinstance(attr, str) for attr in attributes):
148 # Enum members must be named by strings; a non-string attribute (e.g.
149 # ``Enum("e", (1,))``) means the definition is invalid, so fall back to
150 # the default inference instead of crashing.
151 raise UseInferenceDefault
152 if enum and not isinstance(name, str):
153 # Enum class names must be strings; inferring a class with any other
154 # name can make consumers crash when they perform string operations.
155 raise UseInferenceDefault
156 attributes = [attr for attr in attributes if " " not in attr]
157
158 # If we can't infer the name of the class, don't crash, up to this point
159 # we know it is a namedtuple anyway.
160 name = name or "Uninferable"
161 # we want to return a Class node instance with proper attributes set
162 class_node = nodes.ClassDef(
163 name,
164 lineno=node.lineno,
165 col_offset=node.col_offset,
166 end_lineno=node.end_lineno,
167 end_col_offset=node.end_col_offset,
168 parent=parent,
169 )
170 class_node.postinit(
171 bases=[base_type],
172 body=[],
173 decorators=None,
174 )
175 # XXX add __init__(*attributes) method
176 for attr in attributes:
177 fake_node = nodes.EmptyNode()
178 fake_node.parent = class_node
179 fake_node.attrname = attr
180 class_node.instance_attrs[attr] = [fake_node]
181 return class_node, name, attributes
182
183
184def _has_namedtuple_base(node: nodes.ClassDef) -> bool:
185 """Predicate for class inference tip."""
186 return bool(set(node.basenames) & TYPING_NAMEDTUPLE_BASENAMES)
187
188
189def _looks_like(node, name) -> bool:
190 func = node.func
191 if isinstance(func, nodes.Attribute):
192 return func.attrname == name
193 if isinstance(func, nodes.Name):
194 return func.name == name
195 return False
196
197
198_looks_like_namedtuple = functools.partial(_looks_like, name="namedtuple")
199_looks_like_enum = functools.partial(_looks_like, name="Enum")
200_looks_like_typing_namedtuple = functools.partial(_looks_like, name="NamedTuple")
201
202
203def infer_named_tuple(
204 node: nodes.Call, context: InferenceContext | None = None
205) -> Iterator[nodes.ClassDef]:
206 """Specific inference function for namedtuple Call node."""
207 tuple_base: nodes.Name = _extract_single_node("tuple")
208 class_node, name, attributes = infer_func_form(
209 node, tuple_base, parent=SYNTHETIC_ROOT, context=context
210 )
211
212 call_site = arguments.CallSite.from_call(node, context=context)
213 func = util.safe_infer(
214 _extract_single_node("import collections; collections.namedtuple")
215 )
216 assert isinstance(func, nodes.NodeNG)
217 try:
218 rename_arg_bool_value = next(
219 call_site.infer_argument(func, "rename", context or InferenceContext())
220 ).bool_value()
221 rename = rename_arg_bool_value is True
222 except (InferenceError, StopIteration):
223 rename = False
224
225 try:
226 attributes = _check_namedtuple_attributes(name, attributes, rename)
227 except AstroidTypeError as exc:
228 raise UseInferenceDefault("TypeError: " + str(exc)) from exc
229 except AstroidValueError as exc:
230 raise UseInferenceDefault("ValueError: " + str(exc)) from exc
231
232 if tuple(class_node.instance_attrs) != tuple(attributes):
233 # ``infer_func_form`` recorded the field names as written, but ``rename=True``
234 # replaces invalid, keyword or duplicate names with ``_N``, so the instance
235 # would otherwise carry names the namedtuple does not actually have (and lose
236 # the duplicated ones entirely). Rebuild them from the renamed fields.
237 class_node.instance_attrs.clear()
238 for attr in attributes:
239 fake_node = nodes.EmptyNode()
240 fake_node.parent = class_node
241 fake_node.attrname = attr
242 class_node.instance_attrs[attr] = [fake_node]
243
244 replace_args = ", ".join(f"{arg}=None" for arg in attributes)
245 field_def = (
246 " {name} = property(lambda self: self[{index:d}], "
247 "doc='Alias for field number {index:d}')"
248 )
249 field_defs = "\n".join(
250 field_def.format(name=name, index=index)
251 for index, name in enumerate(attributes)
252 )
253 fake = AstroidBuilder(AstroidManager()).string_build(f"""
254class {name}(tuple):
255 __slots__ = ()
256 _fields = {attributes!r}
257 def _asdict(self):
258 return self.__dict__
259 @classmethod
260 def _make(cls, iterable, new=tuple.__new__, len=len):
261 return new(cls, iterable)
262 def _replace(self, {replace_args}):
263 return self
264 def __getnewargs__(self):
265 return tuple(self)
266{field_defs}
267 """)
268 class_node.locals["_asdict"] = fake.body[0].locals["_asdict"]
269 class_node.locals["_make"] = fake.body[0].locals["_make"]
270 class_node.locals["_replace"] = fake.body[0].locals["_replace"]
271 class_node.locals["_fields"] = fake.body[0].locals["_fields"]
272 for attr in attributes:
273 # Python normalises identifiers to NFKC, so a field named "\u00b5" (MICRO SIGN)
274 # is stored by the parser as "\u03bc" (GREEK SMALL LETTER MU). Looking the
275 # attribute up under the name as written raises KeyError on a definition
276 # namedtuple itself accepts, so use the name the parser actually used.
277 normalized = unicodedata.normalize("NFKC", attr)
278 class_node.locals[normalized] = fake.body[0].locals[normalized]
279 # we use UseInferenceDefault, we can't be a generator so return an iterator
280 return iter([class_node])
281
282
283def _get_renamed_namedtuple_attributes(field_names):
284 names = list(field_names)
285 seen = set()
286 for i, name in enumerate(field_names):
287 # pylint: disable = too-many-boolean-expressions
288 if (
289 not all(c.isalnum() or c == "_" for c in name)
290 or keyword.iskeyword(name)
291 or not name
292 or name[0].isdigit()
293 or name.startswith("_")
294 or name in seen
295 ):
296 names[i] = f"_{i}"
297 seen.add(name)
298 return tuple(names)
299
300
301def _check_namedtuple_attributes(typename, attributes, rename=False):
302 attributes = tuple(attributes)
303 if rename:
304 attributes = _get_renamed_namedtuple_attributes(attributes)
305
306 # The following snippet is derived from the CPython Lib/collections/__init__.py sources
307 # <snippet>
308 for name in (typename, *attributes):
309 if not isinstance(name, str):
310 raise AstroidTypeError(
311 f"Type names and field names must be strings, not {type(name)!r}"
312 )
313 if not name.isidentifier():
314 raise AstroidValueError(
315 "Type names and field names must be valid" + f"identifiers: {name!r}"
316 )
317 if keyword.iskeyword(name):
318 raise AstroidValueError(
319 f"Type names and field names cannot be a keyword: {name!r}"
320 )
321
322 seen = set()
323 for name in attributes:
324 if name.startswith("_") and not rename:
325 raise AstroidValueError(
326 f"Field names cannot start with an underscore: {name!r}"
327 )
328 if name in seen:
329 raise AstroidValueError(f"Encountered duplicate field name: {name!r}")
330 seen.add(name)
331 # </snippet>
332
333 return attributes
334
335
336def infer_enum(
337 node: nodes.Call, context: InferenceContext | None = None
338) -> Iterator[bases.Instance]:
339 """Specific inference function for enum Call node."""
340 # Raise `UseInferenceDefault` if `node` is a call to a a user-defined Enum.
341 try:
342 inferred = node.func.infer(context)
343 except (InferenceError, StopIteration) as exc:
344 raise UseInferenceDefault from exc
345
346 if not any(
347 isinstance(item, nodes.ClassDef) and item.qname() == ENUM_QNAME
348 for item in inferred
349 ):
350 raise UseInferenceDefault
351
352 enum_meta = _extract_single_node("""
353 class EnumMeta(object):
354 'docstring'
355 def __call__(self, node):
356 class EnumAttribute(object):
357 name = ''
358 value = 0
359 return EnumAttribute()
360 def __iter__(self):
361 class EnumAttribute(object):
362 name = ''
363 value = 0
364 return [EnumAttribute()]
365 def __reversed__(self):
366 class EnumAttribute(object):
367 name = ''
368 value = 0
369 return (EnumAttribute, )
370 def __next__(self):
371 return next(iter(self))
372 def __getitem__(self, attr):
373 class Value(object):
374 @property
375 def name(self):
376 return ''
377 @property
378 def value(self):
379 return attr
380
381 return Value()
382 __members__ = ['']
383 """)
384
385 # FIXME arguably, the base here shouldn't be the EnumMeta class definition
386 # itself, but a reference (Name) to it. Otherwise, the invariant that all
387 # children of a node have that node as their parent is broken.
388 class_node = infer_func_form(
389 node,
390 enum_meta,
391 parent=SYNTHETIC_ROOT,
392 context=context,
393 enum=True,
394 )[0]
395 return iter([class_node.instantiate_class()])
396
397
398INT_FLAG_ADDITION_METHODS = """
399 def __or__(self, other):
400 return {name}(self.value | other.value)
401 def __and__(self, other):
402 return {name}(self.value & other.value)
403 def __xor__(self, other):
404 return {name}(self.value ^ other.value)
405 def __add__(self, other):
406 return {name}(self.value + other.value)
407 def __div__(self, other):
408 return {name}(self.value / other.value)
409 def __invert__(self):
410 return {name}(~self.value)
411 def __mul__(self, other):
412 return {name}(self.value * other.value)
413"""
414
415
416def infer_enum_class(node: nodes.ClassDef) -> nodes.ClassDef:
417 """Specific inference for enums."""
418 try:
419 mro = node.mro()
420 except MroError:
421 # A malformed class hierarchy (e.g. duplicate bases) has no resolvable
422 # MRO; leave the node untransformed rather than crashing.
423 return node
424 for basename in (b for cls in mro for b in cls.basenames):
425 if node.root().name == "enum":
426 # Skip if the class is directly from enum module.
427 break
428 dunder_members = {}
429 target_names = set()
430 for local, values in node.locals.items():
431 if (
432 any(not isinstance(value, nodes.AssignName) for value in values)
433 or local == "_ignore_"
434 ):
435 continue
436
437 stmt = values[0].statement()
438 if isinstance(stmt, nodes.Assign):
439 if isinstance(stmt.targets[0], nodes.Tuple):
440 targets = stmt.targets[0].itered()
441 else:
442 targets = stmt.targets
443 elif isinstance(stmt, nodes.AnnAssign):
444 targets = [stmt.target]
445 else:
446 continue
447
448 inferred_return_value = None
449 if stmt.value is not None:
450 if isinstance(stmt.value, nodes.Const):
451 if isinstance(stmt.value.value, str):
452 inferred_return_value = repr(stmt.value.value)
453 else:
454 inferred_return_value = stmt.value.value
455 else:
456 inferred_return_value = stmt.value.as_string()
457
458 new_targets = []
459 for target in targets:
460 if isinstance(target, nodes.Starred):
461 continue
462 target_names.add(target.name)
463 # Replace all the assignments with our mocked class.
464 classdef = dedent(
465 """
466 class {name}({types}):
467 @property
468 def value(self):
469 return {return_value}
470 @property
471 def _value_(self):
472 return {return_value}
473 @property
474 def name(self):
475 return "{name}"
476 @property
477 def _name_(self):
478 return "{name}"
479 """.format(
480 name=target.name,
481 types=", ".join(node.basenames),
482 return_value=inferred_return_value,
483 )
484 )
485 if "IntFlag" in basename:
486 # Alright, we need to add some additional methods.
487 # Unfortunately we still can't infer the resulting objects as
488 # Enum members, but once we'll be able to do that, the following
489 # should result in some nice symbolic execution
490 classdef += INT_FLAG_ADDITION_METHODS.format(name=target.name)
491
492 fake = AstroidBuilder(
493 AstroidManager(), apply_transforms=False
494 ).string_build(classdef)[target.name]
495 fake.parent = target.parent
496 for method in node.mymethods():
497 fake.locals[method.name] = [method]
498 new_targets.append(fake.instantiate_class())
499 if stmt.value is None:
500 continue
501 dunder_members[local] = fake
502 node.locals[local] = new_targets
503
504 # The undocumented `_value2member_map_` member:
505 node.locals["_value2member_map_"] = [
506 nodes.Dict(
507 parent=node,
508 lineno=node.lineno,
509 col_offset=node.col_offset,
510 end_lineno=node.end_lineno,
511 end_col_offset=node.end_col_offset,
512 )
513 ]
514
515 members = nodes.Dict(
516 parent=node,
517 lineno=node.lineno,
518 col_offset=node.col_offset,
519 end_lineno=node.end_lineno,
520 end_col_offset=node.end_col_offset,
521 )
522 members.postinit(
523 [
524 (
525 nodes.Const(k, parent=members),
526 nodes.Name(
527 v.name,
528 parent=members,
529 lineno=v.lineno,
530 col_offset=v.col_offset,
531 end_lineno=v.end_lineno,
532 end_col_offset=v.end_col_offset,
533 ),
534 )
535 for k, v in dunder_members.items()
536 ]
537 )
538 node.locals["__members__"] = [members]
539 # The enum.Enum class itself defines two @DynamicClassAttribute data-descriptors
540 # "name" and "value" (which we override in the mocked class for each enum member
541 # above). When dealing with inference of an arbitrary instance of the enum
542 # class, e.g. in a method defined in the class body like:
543 # class SomeEnum(enum.Enum):
544 # def method(self):
545 # self.name # <- here
546 # In the absence of an enum member called "name" or "value", these attributes
547 # should resolve to the descriptor on that particular instance, i.e. enum member.
548 # For "value", we have no idea what that should be, but for "name", we at least
549 # know that it should be a string, so infer that as a guess.
550 if "name" not in target_names:
551 code = dedent('''
552 @property
553 def name(self):
554 """The name of the Enum member.
555
556 This is a reconstruction by astroid: enums are too dynamic to understand, but we at least
557 know 'name' should be a string, so this is astroid's best guess.
558 """
559 return ''
560 ''')
561 name_dynamicclassattr = AstroidBuilder(AstroidManager()).string_build(code)[
562 "name"
563 ]
564 node.locals["name"] = [name_dynamicclassattr]
565 break
566 return node
567
568
569def infer_typing_namedtuple_class(class_node, context: InferenceContext | None = None):
570 """Infer a subclass of typing.NamedTuple."""
571 # Check if it has the corresponding bases
572 annassigns_fields = [
573 annassign.target.name
574 for annassign in class_node.body
575 if isinstance(annassign, nodes.AnnAssign)
576 ]
577 code = dedent("""
578 from collections import namedtuple
579 namedtuple({typename!r}, {fields!r})
580 """).format(typename=class_node.name, fields=",".join(annassigns_fields))
581 node = extract_node(code)
582 try:
583 generated_class_node = next(infer_named_tuple(node, context))
584 except StopIteration as e:
585 raise InferenceError(node=node, context=context) from e
586 for method in class_node.mymethods():
587 generated_class_node.locals[method.name] = [method]
588
589 for body_node in class_node.body:
590 if isinstance(body_node, nodes.Assign):
591 for target in body_node.targets:
592 # A target is not necessarily a single name: ``cat.color = ...``
593 # and ``basket[0] = ...`` define no class attribute at all, while
594 # ``apple, banana = ...`` defines one per unpacked element.
595 for assign_name in target.nodes_of_class(nodes.AssignName):
596 attr = assign_name.name
597 generated_class_node.locals[attr] = class_node.locals[attr]
598 elif isinstance(body_node, nodes.ClassDef):
599 generated_class_node.locals[body_node.name] = [body_node]
600
601 return iter((generated_class_node,))
602
603
604def infer_typing_namedtuple_function(node, context: InferenceContext | None = None):
605 """
606 Starting with python3.9, NamedTuple is a function of the typing module.
607 The class NamedTuple is build dynamically through a call to `type` during
608 initialization of the `_NamedTuple` variable.
609 """
610 klass = extract_node("""
611 from typing import _NamedTuple
612 _NamedTuple
613 """)
614 return klass.infer(context)
615
616
617def infer_typing_namedtuple(
618 node: nodes.Call, context: InferenceContext | None = None
619) -> Iterator[nodes.ClassDef]:
620 """Infer a typing.NamedTuple(...) call."""
621 # This is essentially a namedtuple with different arguments
622 # so we extract the args and infer a named tuple.
623 try:
624 func = next(node.func.infer())
625 except (InferenceError, StopIteration) as exc:
626 raise UseInferenceDefault from exc
627
628 if func.qname() not in TYPING_NAMEDTUPLE_QUALIFIED:
629 raise UseInferenceDefault
630
631 if len(node.args) != 2:
632 raise UseInferenceDefault
633
634 if not isinstance(node.args[1], (nodes.List, nodes.Tuple)):
635 raise UseInferenceDefault
636
637 return infer_named_tuple(node, context)
638
639
640def _get_namedtuple_fields(node: nodes.Call) -> str:
641 """Get and return fields of a NamedTuple in code-as-a-string.
642
643 Because the fields are represented in their code form we can
644 extract a node from them later on.
645 """
646 names = []
647 container = None
648 try:
649 container = next(node.args[1].infer())
650 except (InferenceError, StopIteration) as exc:
651 raise UseInferenceDefault from exc
652 # We pass on IndexError as we'll try to infer 'field_names' from the keywords
653 except IndexError:
654 pass
655 if not container:
656 for keyword_node in node.keywords:
657 if keyword_node.arg == "field_names":
658 try:
659 container = next(keyword_node.value.infer())
660 except (InferenceError, StopIteration) as exc:
661 raise UseInferenceDefault from exc
662 break
663 if not isinstance(container, nodes.BaseContainer):
664 raise UseInferenceDefault
665 for elt in container.elts:
666 if isinstance(elt, nodes.Const):
667 names.append(elt.as_string())
668 continue
669 if not isinstance(elt, (nodes.List, nodes.Tuple)):
670 raise UseInferenceDefault
671 if len(elt.elts) != 2:
672 raise UseInferenceDefault
673 names.append(elt.elts[0].as_string())
674
675 if names:
676 field_names = f"({','.join(names)},)"
677 else:
678 field_names = ""
679 return field_names
680
681
682def _is_enum_subclass(cls: nodes.ClassDef) -> bool:
683 """Return whether cls is a subclass of an Enum."""
684 try:
685 return cls.is_subtype_of("enum.Enum")
686 except AstroidError:
687 return False
688
689
690def register(manager: AstroidManager) -> None:
691 manager.register_transform(
692 nodes.Call, inference_tip(infer_named_tuple), _looks_like_namedtuple
693 )
694 manager.register_transform(nodes.Call, inference_tip(infer_enum), _looks_like_enum)
695 manager.register_transform(
696 nodes.ClassDef, infer_enum_class, predicate=_is_enum_subclass
697 )
698 manager.register_transform(
699 nodes.ClassDef,
700 inference_tip(infer_typing_namedtuple_class),
701 _has_namedtuple_base,
702 )
703 manager.register_transform(
704 nodes.FunctionDef,
705 inference_tip(infer_typing_namedtuple_function),
706 lambda node: node.name == "NamedTuple"
707 and getattr(node.root(), "name", None) == "typing",
708 )
709 manager.register_transform(
710 nodes.Call,
711 inference_tip(infer_typing_namedtuple),
712 _looks_like_typing_namedtuple,
713 )