Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# Licensed under the LGPL: https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html
2# For details: https://github.com/pylint-dev/astroid/blob/main/LICENSE
3# Copyright (c) https://github.com/pylint-dev/astroid/blob/main/CONTRIBUTORS.txt
5"""Astroid hooks for the Python standard library."""
7from __future__ import annotations
9import functools
10import keyword
11from collections.abc import Iterator
12from textwrap import dedent
13from typing import Final
15from astroid import arguments, bases, nodes, util
16from astroid.builder import AstroidBuilder, _extract_single_node, extract_node
17from astroid.context import InferenceContext
18from astroid.exceptions import (
19 AstroidTypeError,
20 AstroidValueError,
21 InferenceError,
22 MroError,
23 UseInferenceDefault,
24)
25from astroid.inference_tip import inference_tip
26from astroid.manager import AstroidManager
27from astroid.nodes.scoped_nodes.scoped_nodes import SYNTHETIC_ROOT
29ENUM_QNAME: Final[str] = "enum.Enum"
30TYPING_NAMEDTUPLE_QUALIFIED: Final = {
31 "typing.NamedTuple",
32 "typing_extensions.NamedTuple",
33}
34TYPING_NAMEDTUPLE_BASENAMES: Final = {
35 "NamedTuple",
36 "typing.NamedTuple",
37 "typing_extensions.NamedTuple",
38}
41def _infer_first(node, context):
42 if isinstance(node, util.UninferableBase):
43 raise UseInferenceDefault
44 try:
45 value = next(node.infer(context=context))
46 except StopIteration as exc:
47 raise InferenceError from exc
48 if isinstance(value, util.UninferableBase):
49 raise UseInferenceDefault()
50 return value
53def _find_func_form_arguments(node, context):
54 def _extract_namedtuple_arg_or_keyword( # pylint: disable=inconsistent-return-statements
55 position, key_name=None
56 ):
57 if len(args) > position:
58 return _infer_first(args[position], context)
59 if key_name and key_name in found_keywords:
60 return _infer_first(found_keywords[key_name], context)
62 args = node.args
63 keywords = node.keywords
64 found_keywords = (
65 {keyword.arg: keyword.value for keyword in keywords} if keywords else {}
66 )
68 name = _extract_namedtuple_arg_or_keyword(position=0, key_name="typename")
69 names = _extract_namedtuple_arg_or_keyword(position=1, key_name="field_names")
70 if name and names:
71 return name.value, names
73 raise UseInferenceDefault()
76def infer_func_form(
77 node: nodes.Call,
78 base_type: nodes.NodeNG,
79 *,
80 parent: nodes.NodeNG,
81 context: InferenceContext | None = None,
82 enum: bool = False,
83) -> tuple[nodes.ClassDef, str, list[str]]:
84 """Specific inference function for namedtuple or Python 3 enum."""
85 # node is a Call node, class name as first argument and generated class
86 # attributes as second argument
88 # namedtuple or enums list of attributes can be a list of strings or a
89 # whitespace-separate string
90 try:
91 name, names = _find_func_form_arguments(node, context)
92 try:
93 attributes: list[str] = names.value.replace(",", " ").split()
94 except AttributeError as exc:
95 # Handle attributes of NamedTuples
96 if not enum:
97 attributes = []
98 fields = _get_namedtuple_fields(node)
99 if fields:
100 fields_node = extract_node(fields)
101 attributes = [
102 _infer_first(const, context).value for const in fields_node.elts
103 ]
105 # Handle attributes of Enums
106 else:
107 # Enums supports either iterator of (name, value) pairs
108 # or mappings.
109 if hasattr(names, "items") and isinstance(names.items, list):
110 attributes = [
111 _infer_first(const[0], context).value
112 for const in names.items
113 if isinstance(const[0], nodes.Const)
114 ]
115 elif hasattr(names, "elts"):
116 # Enums can support either ["a", "b", "c"]
117 # or [("a", 1), ("b", 2), ...], but they can't
118 # be mixed.
119 if all(isinstance(const, nodes.Tuple) for const in names.elts):
120 attributes = [
121 _infer_first(const.elts[0], context).value
122 for const in names.elts
123 if isinstance(const, nodes.Tuple)
124 ]
125 else:
126 attributes = [
127 _infer_first(const, context).value for const in names.elts
128 ]
129 else:
130 raise AttributeError from exc
131 if not attributes:
132 raise AttributeError from exc
133 except (AttributeError, InferenceError) as exc:
134 raise UseInferenceDefault from exc
136 if not enum:
137 # namedtuple maps sys.intern(str()) over over field_names
138 attributes = [str(attr) for attr in attributes]
139 # XXX this should succeed *unless* __str__/__repr__ is incorrect or throws
140 # in which case we should not have inferred these values and raised earlier
141 if any(not isinstance(attr, str) for attr in attributes):
142 # Enum members must be named by strings; a non-string attribute (e.g.
143 # ``Enum("e", (1,))``) means the definition is invalid, so fall back to
144 # the default inference instead of crashing.
145 raise UseInferenceDefault
146 attributes = [attr for attr in attributes if " " not in attr]
148 # If we can't infer the name of the class, don't crash, up to this point
149 # we know it is a namedtuple anyway.
150 name = name or "Uninferable"
151 # we want to return a Class node instance with proper attributes set
152 class_node = nodes.ClassDef(
153 name,
154 lineno=node.lineno,
155 col_offset=node.col_offset,
156 end_lineno=node.end_lineno,
157 end_col_offset=node.end_col_offset,
158 parent=parent,
159 )
160 class_node.postinit(
161 bases=[base_type],
162 body=[],
163 decorators=None,
164 )
165 # XXX add __init__(*attributes) method
166 for attr in attributes:
167 fake_node = nodes.EmptyNode()
168 fake_node.parent = class_node
169 fake_node.attrname = attr
170 class_node.instance_attrs[attr] = [fake_node]
171 return class_node, name, attributes
174def _has_namedtuple_base(node):
175 """Predicate for class inference tip.
177 :type node: ClassDef
178 :rtype: bool
179 """
180 return set(node.basenames) & TYPING_NAMEDTUPLE_BASENAMES
183def _looks_like(node, name) -> bool:
184 func = node.func
185 if isinstance(func, nodes.Attribute):
186 return func.attrname == name
187 if isinstance(func, nodes.Name):
188 return func.name == name
189 return False
192_looks_like_namedtuple = functools.partial(_looks_like, name="namedtuple")
193_looks_like_enum = functools.partial(_looks_like, name="Enum")
194_looks_like_typing_namedtuple = functools.partial(_looks_like, name="NamedTuple")
197def infer_named_tuple(
198 node: nodes.Call, context: InferenceContext | None = None
199) -> Iterator[nodes.ClassDef]:
200 """Specific inference function for namedtuple Call node."""
201 tuple_base: nodes.Name = _extract_single_node("tuple")
202 class_node, name, attributes = infer_func_form(
203 node, tuple_base, parent=SYNTHETIC_ROOT, context=context
204 )
206 call_site = arguments.CallSite.from_call(node, context=context)
207 func = util.safe_infer(
208 _extract_single_node("import collections; collections.namedtuple")
209 )
210 assert isinstance(func, nodes.NodeNG)
211 try:
212 rename_arg_bool_value = next(
213 call_site.infer_argument(func, "rename", context or InferenceContext())
214 ).bool_value()
215 rename = rename_arg_bool_value is True
216 except (InferenceError, StopIteration):
217 rename = False
219 try:
220 attributes = _check_namedtuple_attributes(name, attributes, rename)
221 except AstroidTypeError as exc:
222 raise UseInferenceDefault("TypeError: " + str(exc)) from exc
223 except AstroidValueError as exc:
224 raise UseInferenceDefault("ValueError: " + str(exc)) from exc
226 replace_args = ", ".join(f"{arg}=None" for arg in attributes)
227 field_def = (
228 " {name} = property(lambda self: self[{index:d}], "
229 "doc='Alias for field number {index:d}')"
230 )
231 field_defs = "\n".join(
232 field_def.format(name=name, index=index)
233 for index, name in enumerate(attributes)
234 )
235 fake = AstroidBuilder(AstroidManager()).string_build(f"""
236class {name}(tuple):
237 __slots__ = ()
238 _fields = {attributes!r}
239 def _asdict(self):
240 return self.__dict__
241 @classmethod
242 def _make(cls, iterable, new=tuple.__new__, len=len):
243 return new(cls, iterable)
244 def _replace(self, {replace_args}):
245 return self
246 def __getnewargs__(self):
247 return tuple(self)
248{field_defs}
249 """)
250 class_node.locals["_asdict"] = fake.body[0].locals["_asdict"]
251 class_node.locals["_make"] = fake.body[0].locals["_make"]
252 class_node.locals["_replace"] = fake.body[0].locals["_replace"]
253 class_node.locals["_fields"] = fake.body[0].locals["_fields"]
254 for attr in attributes:
255 class_node.locals[attr] = fake.body[0].locals[attr]
256 # we use UseInferenceDefault, we can't be a generator so return an iterator
257 return iter([class_node])
260def _get_renamed_namedtuple_attributes(field_names):
261 names = list(field_names)
262 seen = set()
263 for i, name in enumerate(field_names):
264 # pylint: disable = too-many-boolean-expressions
265 if (
266 not all(c.isalnum() or c == "_" for c in name)
267 or keyword.iskeyword(name)
268 or not name
269 or name[0].isdigit()
270 or name.startswith("_")
271 or name in seen
272 ):
273 names[i] = f"_{i}"
274 seen.add(name)
275 return tuple(names)
278def _check_namedtuple_attributes(typename, attributes, rename=False):
279 attributes = tuple(attributes)
280 if rename:
281 attributes = _get_renamed_namedtuple_attributes(attributes)
283 # The following snippet is derived from the CPython Lib/collections/__init__.py sources
284 # <snippet>
285 for name in (typename, *attributes):
286 if not isinstance(name, str):
287 raise AstroidTypeError(
288 f"Type names and field names must be strings, not {type(name)!r}"
289 )
290 if not name.isidentifier():
291 raise AstroidValueError(
292 "Type names and field names must be valid" + f"identifiers: {name!r}"
293 )
294 if keyword.iskeyword(name):
295 raise AstroidValueError(
296 f"Type names and field names cannot be a keyword: {name!r}"
297 )
299 seen = set()
300 for name in attributes:
301 if name.startswith("_") and not rename:
302 raise AstroidValueError(
303 f"Field names cannot start with an underscore: {name!r}"
304 )
305 if name in seen:
306 raise AstroidValueError(f"Encountered duplicate field name: {name!r}")
307 seen.add(name)
308 # </snippet>
310 return attributes
313def infer_enum(
314 node: nodes.Call, context: InferenceContext | None = None
315) -> Iterator[bases.Instance]:
316 """Specific inference function for enum Call node."""
317 # Raise `UseInferenceDefault` if `node` is a call to a a user-defined Enum.
318 try:
319 inferred = node.func.infer(context)
320 except (InferenceError, StopIteration) as exc:
321 raise UseInferenceDefault from exc
323 if not any(
324 isinstance(item, nodes.ClassDef) and item.qname() == ENUM_QNAME
325 for item in inferred
326 ):
327 raise UseInferenceDefault
329 enum_meta = _extract_single_node("""
330 class EnumMeta(object):
331 'docstring'
332 def __call__(self, node):
333 class EnumAttribute(object):
334 name = ''
335 value = 0
336 return EnumAttribute()
337 def __iter__(self):
338 class EnumAttribute(object):
339 name = ''
340 value = 0
341 return [EnumAttribute()]
342 def __reversed__(self):
343 class EnumAttribute(object):
344 name = ''
345 value = 0
346 return (EnumAttribute, )
347 def __next__(self):
348 return next(iter(self))
349 def __getitem__(self, attr):
350 class Value(object):
351 @property
352 def name(self):
353 return ''
354 @property
355 def value(self):
356 return attr
358 return Value()
359 __members__ = ['']
360 """)
362 # FIXME arguably, the base here shouldn't be the EnumMeta class definition
363 # itself, but a reference (Name) to it. Otherwise, the invariant that all
364 # children of a node have that node as their parent is broken.
365 class_node = infer_func_form(
366 node,
367 enum_meta,
368 parent=SYNTHETIC_ROOT,
369 context=context,
370 enum=True,
371 )[0]
372 return iter([class_node.instantiate_class()])
375INT_FLAG_ADDITION_METHODS = """
376 def __or__(self, other):
377 return {name}(self.value | other.value)
378 def __and__(self, other):
379 return {name}(self.value & other.value)
380 def __xor__(self, other):
381 return {name}(self.value ^ other.value)
382 def __add__(self, other):
383 return {name}(self.value + other.value)
384 def __div__(self, other):
385 return {name}(self.value / other.value)
386 def __invert__(self):
387 return {name}(~self.value)
388 def __mul__(self, other):
389 return {name}(self.value * other.value)
390"""
393def infer_enum_class(node: nodes.ClassDef) -> nodes.ClassDef:
394 """Specific inference for enums."""
395 try:
396 mro = node.mro()
397 except MroError:
398 # A malformed class hierarchy (e.g. duplicate bases) has no resolvable
399 # MRO; leave the node untransformed rather than crashing.
400 return node
401 for basename in (b for cls in mro for b in cls.basenames):
402 if node.root().name == "enum":
403 # Skip if the class is directly from enum module.
404 break
405 dunder_members = {}
406 target_names = set()
407 for local, values in node.locals.items():
408 if (
409 any(not isinstance(value, nodes.AssignName) for value in values)
410 or local == "_ignore_"
411 ):
412 continue
414 stmt = values[0].statement()
415 if isinstance(stmt, nodes.Assign):
416 if isinstance(stmt.targets[0], nodes.Tuple):
417 targets = stmt.targets[0].itered()
418 else:
419 targets = stmt.targets
420 elif isinstance(stmt, nodes.AnnAssign):
421 targets = [stmt.target]
422 else:
423 continue
425 inferred_return_value = None
426 if stmt.value is not None:
427 if isinstance(stmt.value, nodes.Const):
428 if isinstance(stmt.value.value, str):
429 inferred_return_value = repr(stmt.value.value)
430 else:
431 inferred_return_value = stmt.value.value
432 else:
433 inferred_return_value = stmt.value.as_string()
435 new_targets = []
436 for target in targets:
437 if isinstance(target, nodes.Starred):
438 continue
439 target_names.add(target.name)
440 # Replace all the assignments with our mocked class.
441 classdef = dedent(
442 """
443 class {name}({types}):
444 @property
445 def value(self):
446 return {return_value}
447 @property
448 def _value_(self):
449 return {return_value}
450 @property
451 def name(self):
452 return "{name}"
453 @property
454 def _name_(self):
455 return "{name}"
456 """.format(
457 name=target.name,
458 types=", ".join(node.basenames),
459 return_value=inferred_return_value,
460 )
461 )
462 if "IntFlag" in basename:
463 # Alright, we need to add some additional methods.
464 # Unfortunately we still can't infer the resulting objects as
465 # Enum members, but once we'll be able to do that, the following
466 # should result in some nice symbolic execution
467 classdef += INT_FLAG_ADDITION_METHODS.format(name=target.name)
469 fake = AstroidBuilder(
470 AstroidManager(), apply_transforms=False
471 ).string_build(classdef)[target.name]
472 fake.parent = target.parent
473 for method in node.mymethods():
474 fake.locals[method.name] = [method]
475 new_targets.append(fake.instantiate_class())
476 if stmt.value is None:
477 continue
478 dunder_members[local] = fake
479 node.locals[local] = new_targets
481 # The undocumented `_value2member_map_` member:
482 node.locals["_value2member_map_"] = [
483 nodes.Dict(
484 parent=node,
485 lineno=node.lineno,
486 col_offset=node.col_offset,
487 end_lineno=node.end_lineno,
488 end_col_offset=node.end_col_offset,
489 )
490 ]
492 members = nodes.Dict(
493 parent=node,
494 lineno=node.lineno,
495 col_offset=node.col_offset,
496 end_lineno=node.end_lineno,
497 end_col_offset=node.end_col_offset,
498 )
499 members.postinit(
500 [
501 (
502 nodes.Const(k, parent=members),
503 nodes.Name(
504 v.name,
505 parent=members,
506 lineno=v.lineno,
507 col_offset=v.col_offset,
508 end_lineno=v.end_lineno,
509 end_col_offset=v.end_col_offset,
510 ),
511 )
512 for k, v in dunder_members.items()
513 ]
514 )
515 node.locals["__members__"] = [members]
516 # The enum.Enum class itself defines two @DynamicClassAttribute data-descriptors
517 # "name" and "value" (which we override in the mocked class for each enum member
518 # above). When dealing with inference of an arbitrary instance of the enum
519 # class, e.g. in a method defined in the class body like:
520 # class SomeEnum(enum.Enum):
521 # def method(self):
522 # self.name # <- here
523 # In the absence of an enum member called "name" or "value", these attributes
524 # should resolve to the descriptor on that particular instance, i.e. enum member.
525 # For "value", we have no idea what that should be, but for "name", we at least
526 # know that it should be a string, so infer that as a guess.
527 if "name" not in target_names:
528 code = dedent('''
529 @property
530 def name(self):
531 """The name of the Enum member.
533 This is a reconstruction by astroid: enums are too dynamic to understand, but we at least
534 know 'name' should be a string, so this is astroid's best guess.
535 """
536 return ''
537 ''')
538 name_dynamicclassattr = AstroidBuilder(AstroidManager()).string_build(code)[
539 "name"
540 ]
541 node.locals["name"] = [name_dynamicclassattr]
542 break
543 return node
546def infer_typing_namedtuple_class(class_node, context: InferenceContext | None = None):
547 """Infer a subclass of typing.NamedTuple."""
548 # Check if it has the corresponding bases
549 annassigns_fields = [
550 annassign.target.name
551 for annassign in class_node.body
552 if isinstance(annassign, nodes.AnnAssign)
553 ]
554 code = dedent("""
555 from collections import namedtuple
556 namedtuple({typename!r}, {fields!r})
557 """).format(typename=class_node.name, fields=",".join(annassigns_fields))
558 node = extract_node(code)
559 try:
560 generated_class_node = next(infer_named_tuple(node, context))
561 except StopIteration as e:
562 raise InferenceError(node=node, context=context) from e
563 for method in class_node.mymethods():
564 generated_class_node.locals[method.name] = [method]
566 for body_node in class_node.body:
567 if isinstance(body_node, nodes.Assign):
568 for target in body_node.targets:
569 attr = target.name
570 generated_class_node.locals[attr] = class_node.locals[attr]
571 elif isinstance(body_node, nodes.ClassDef):
572 generated_class_node.locals[body_node.name] = [body_node]
574 return iter((generated_class_node,))
577def infer_typing_namedtuple_function(node, context: InferenceContext | None = None):
578 """
579 Starting with python3.9, NamedTuple is a function of the typing module.
580 The class NamedTuple is build dynamically through a call to `type` during
581 initialization of the `_NamedTuple` variable.
582 """
583 klass = extract_node("""
584 from typing import _NamedTuple
585 _NamedTuple
586 """)
587 return klass.infer(context)
590def infer_typing_namedtuple(
591 node: nodes.Call, context: InferenceContext | None = None
592) -> Iterator[nodes.ClassDef]:
593 """Infer a typing.NamedTuple(...) call."""
594 # This is essentially a namedtuple with different arguments
595 # so we extract the args and infer a named tuple.
596 try:
597 func = next(node.func.infer())
598 except (InferenceError, StopIteration) as exc:
599 raise UseInferenceDefault from exc
601 if func.qname() not in TYPING_NAMEDTUPLE_QUALIFIED:
602 raise UseInferenceDefault
604 if len(node.args) != 2:
605 raise UseInferenceDefault
607 if not isinstance(node.args[1], (nodes.List, nodes.Tuple)):
608 raise UseInferenceDefault
610 return infer_named_tuple(node, context)
613def _get_namedtuple_fields(node: nodes.Call) -> str:
614 """Get and return fields of a NamedTuple in code-as-a-string.
616 Because the fields are represented in their code form we can
617 extract a node from them later on.
618 """
619 names = []
620 container = None
621 try:
622 container = next(node.args[1].infer())
623 except (InferenceError, StopIteration) as exc:
624 raise UseInferenceDefault from exc
625 # We pass on IndexError as we'll try to infer 'field_names' from the keywords
626 except IndexError:
627 pass
628 if not container:
629 for keyword_node in node.keywords:
630 if keyword_node.arg == "field_names":
631 try:
632 container = next(keyword_node.value.infer())
633 except (InferenceError, StopIteration) as exc:
634 raise UseInferenceDefault from exc
635 break
636 if not isinstance(container, nodes.BaseContainer):
637 raise UseInferenceDefault
638 for elt in container.elts:
639 if isinstance(elt, nodes.Const):
640 names.append(elt.as_string())
641 continue
642 if not isinstance(elt, (nodes.List, nodes.Tuple)):
643 raise UseInferenceDefault
644 if len(elt.elts) != 2:
645 raise UseInferenceDefault
646 names.append(elt.elts[0].as_string())
648 if names:
649 field_names = f"({','.join(names)},)"
650 else:
651 field_names = ""
652 return field_names
655def _is_enum_subclass(cls: nodes.ClassDef) -> bool:
656 """Return whether cls is a subclass of an Enum."""
657 return cls.is_subtype_of("enum.Enum")
660def register(manager: AstroidManager) -> None:
661 manager.register_transform(
662 nodes.Call, inference_tip(infer_named_tuple), _looks_like_namedtuple
663 )
664 manager.register_transform(nodes.Call, inference_tip(infer_enum), _looks_like_enum)
665 manager.register_transform(
666 nodes.ClassDef, infer_enum_class, predicate=_is_enum_subclass
667 )
668 manager.register_transform(
669 nodes.ClassDef,
670 inference_tip(infer_typing_namedtuple_class),
671 _has_namedtuple_base,
672 )
673 manager.register_transform(
674 nodes.FunctionDef,
675 inference_tip(infer_typing_namedtuple_function),
676 lambda node: node.name == "NamedTuple"
677 and getattr(node.root(), "name", None) == "typing",
678 )
679 manager.register_transform(
680 nodes.Call,
681 inference_tip(infer_typing_namedtuple),
682 _looks_like_typing_namedtuple,
683 )