Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# Licensed under the LGPL: https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html
2# For details: https://github.com/pylint-dev/astroid/blob/main/LICENSE
3# Copyright (c) https://github.com/pylint-dev/astroid/blob/main/CONTRIBUTORS.txt
5"""Astroid hooks for the Python standard library."""
7from __future__ import annotations
9import functools
10import keyword
11from collections.abc import Iterator
12from textwrap import dedent
13from typing import Final
15import astroid
16from astroid import arguments, bases, nodes, util
17from astroid.builder import AstroidBuilder, _extract_single_node, extract_node
18from astroid.context import InferenceContext
19from astroid.exceptions import (
20 AstroidTypeError,
21 AstroidValueError,
22 InferenceError,
23 UseInferenceDefault,
24)
25from astroid.inference_tip import inference_tip
26from astroid.manager import AstroidManager
27from astroid.nodes.scoped_nodes.scoped_nodes import SYNTHETIC_ROOT
29ENUM_QNAME: Final[str] = "enum.Enum"
30TYPING_NAMEDTUPLE_QUALIFIED: Final = {
31 "typing.NamedTuple",
32 "typing_extensions.NamedTuple",
33}
34TYPING_NAMEDTUPLE_BASENAMES: Final = {
35 "NamedTuple",
36 "typing.NamedTuple",
37 "typing_extensions.NamedTuple",
38}
41def _infer_first(node, context):
42 if isinstance(node, util.UninferableBase):
43 raise UseInferenceDefault
44 try:
45 value = next(node.infer(context=context))
46 except StopIteration as exc:
47 raise InferenceError from exc
48 if isinstance(value, util.UninferableBase):
49 raise UseInferenceDefault()
50 return value
53def _find_func_form_arguments(node, context):
54 def _extract_namedtuple_arg_or_keyword( # pylint: disable=inconsistent-return-statements
55 position, key_name=None
56 ):
57 if len(args) > position:
58 return _infer_first(args[position], context)
59 if key_name and key_name in found_keywords:
60 return _infer_first(found_keywords[key_name], context)
62 args = node.args
63 keywords = node.keywords
64 found_keywords = (
65 {keyword.arg: keyword.value for keyword in keywords} if keywords else {}
66 )
68 name = _extract_namedtuple_arg_or_keyword(position=0, key_name="typename")
69 names = _extract_namedtuple_arg_or_keyword(position=1, key_name="field_names")
70 if name and names:
71 return name.value, names
73 raise UseInferenceDefault()
76def infer_func_form(
77 node: nodes.Call,
78 base_type: nodes.NodeNG,
79 *,
80 parent: nodes.NodeNG,
81 context: InferenceContext | None = None,
82 enum: bool = False,
83) -> tuple[nodes.ClassDef, str, list[str]]:
84 """Specific inference function for namedtuple or Python 3 enum."""
85 # node is a Call node, class name as first argument and generated class
86 # attributes as second argument
88 # namedtuple or enums list of attributes can be a list of strings or a
89 # whitespace-separate string
90 try:
91 name, names = _find_func_form_arguments(node, context)
92 try:
93 attributes: list[str] = names.value.replace(",", " ").split()
94 except AttributeError as exc:
95 # Handle attributes of NamedTuples
96 if not enum:
97 attributes = []
98 fields = _get_namedtuple_fields(node)
99 if fields:
100 fields_node = extract_node(fields)
101 attributes = [
102 _infer_first(const, context).value for const in fields_node.elts
103 ]
105 # Handle attributes of Enums
106 else:
107 # Enums supports either iterator of (name, value) pairs
108 # or mappings.
109 if hasattr(names, "items") and isinstance(names.items, list):
110 attributes = [
111 _infer_first(const[0], context).value
112 for const in names.items
113 if isinstance(const[0], nodes.Const)
114 ]
115 elif hasattr(names, "elts"):
116 # Enums can support either ["a", "b", "c"]
117 # or [("a", 1), ("b", 2), ...], but they can't
118 # be mixed.
119 if all(isinstance(const, nodes.Tuple) for const in names.elts):
120 attributes = [
121 _infer_first(const.elts[0], context).value
122 for const in names.elts
123 if isinstance(const, nodes.Tuple)
124 ]
125 else:
126 attributes = [
127 _infer_first(const, context).value for const in names.elts
128 ]
129 else:
130 raise AttributeError from exc
131 if not attributes:
132 raise AttributeError from exc
133 except (AttributeError, InferenceError) as exc:
134 raise UseInferenceDefault from exc
136 if not enum:
137 # namedtuple maps sys.intern(str()) over over field_names
138 attributes = [str(attr) for attr in attributes]
139 # XXX this should succeed *unless* __str__/__repr__ is incorrect or throws
140 # in which case we should not have inferred these values and raised earlier
141 attributes = [attr for attr in attributes if " " not in attr]
143 # If we can't infer the name of the class, don't crash, up to this point
144 # we know it is a namedtuple anyway.
145 name = name or "Uninferable"
146 # we want to return a Class node instance with proper attributes set
147 class_node = nodes.ClassDef(
148 name,
149 lineno=node.lineno,
150 col_offset=node.col_offset,
151 end_lineno=node.end_lineno,
152 end_col_offset=node.end_col_offset,
153 parent=parent,
154 )
155 class_node.postinit(
156 bases=[base_type],
157 body=[],
158 decorators=None,
159 )
160 # XXX add __init__(*attributes) method
161 for attr in attributes:
162 fake_node = nodes.EmptyNode()
163 fake_node.parent = class_node
164 fake_node.attrname = attr
165 class_node.instance_attrs[attr] = [fake_node]
166 return class_node, name, attributes
169def _has_namedtuple_base(node):
170 """Predicate for class inference tip.
172 :type node: ClassDef
173 :rtype: bool
174 """
175 return set(node.basenames) & TYPING_NAMEDTUPLE_BASENAMES
178def _looks_like(node, name) -> bool:
179 func = node.func
180 if isinstance(func, nodes.Attribute):
181 return func.attrname == name
182 if isinstance(func, nodes.Name):
183 return func.name == name
184 return False
187_looks_like_namedtuple = functools.partial(_looks_like, name="namedtuple")
188_looks_like_enum = functools.partial(_looks_like, name="Enum")
189_looks_like_typing_namedtuple = functools.partial(_looks_like, name="NamedTuple")
192def infer_named_tuple(
193 node: nodes.Call, context: InferenceContext | None = None
194) -> Iterator[nodes.ClassDef]:
195 """Specific inference function for namedtuple Call node."""
196 tuple_base: nodes.Name = _extract_single_node("tuple")
197 class_node, name, attributes = infer_func_form(
198 node, tuple_base, parent=SYNTHETIC_ROOT, context=context
199 )
201 call_site = arguments.CallSite.from_call(node, context=context)
202 func = util.safe_infer(
203 _extract_single_node("import collections; collections.namedtuple")
204 )
205 assert isinstance(func, nodes.NodeNG)
206 try:
207 rename = next(
208 call_site.infer_argument(func, "rename", context or InferenceContext())
209 ).bool_value()
210 except (InferenceError, StopIteration):
211 rename = False
213 try:
214 attributes = _check_namedtuple_attributes(name, attributes, rename)
215 except AstroidTypeError as exc:
216 raise UseInferenceDefault("TypeError: " + str(exc)) from exc
217 except AstroidValueError as exc:
218 raise UseInferenceDefault("ValueError: " + str(exc)) from exc
220 replace_args = ", ".join(f"{arg}=None" for arg in attributes)
221 field_def = (
222 " {name} = property(lambda self: self[{index:d}], "
223 "doc='Alias for field number {index:d}')"
224 )
225 field_defs = "\n".join(
226 field_def.format(name=name, index=index)
227 for index, name in enumerate(attributes)
228 )
229 fake = AstroidBuilder(AstroidManager()).string_build(
230 f"""
231class {name}(tuple):
232 __slots__ = ()
233 _fields = {attributes!r}
234 def _asdict(self):
235 return self.__dict__
236 @classmethod
237 def _make(cls, iterable, new=tuple.__new__, len=len):
238 return new(cls, iterable)
239 def _replace(self, {replace_args}):
240 return self
241 def __getnewargs__(self):
242 return tuple(self)
243{field_defs}
244 """
245 )
246 class_node.locals["_asdict"] = fake.body[0].locals["_asdict"]
247 class_node.locals["_make"] = fake.body[0].locals["_make"]
248 class_node.locals["_replace"] = fake.body[0].locals["_replace"]
249 class_node.locals["_fields"] = fake.body[0].locals["_fields"]
250 for attr in attributes:
251 class_node.locals[attr] = fake.body[0].locals[attr]
252 # we use UseInferenceDefault, we can't be a generator so return an iterator
253 return iter([class_node])
256def _get_renamed_namedtuple_attributes(field_names):
257 names = list(field_names)
258 seen = set()
259 for i, name in enumerate(field_names):
260 # pylint: disable = too-many-boolean-expressions
261 if (
262 not all(c.isalnum() or c == "_" for c in name)
263 or keyword.iskeyword(name)
264 or not name
265 or name[0].isdigit()
266 or name.startswith("_")
267 or name in seen
268 ):
269 names[i] = "_%d" % i
270 seen.add(name)
271 return tuple(names)
274def _check_namedtuple_attributes(typename, attributes, rename=False):
275 attributes = tuple(attributes)
276 if rename:
277 attributes = _get_renamed_namedtuple_attributes(attributes)
279 # The following snippet is derived from the CPython Lib/collections/__init__.py sources
280 # <snippet>
281 for name in (typename, *attributes):
282 if not isinstance(name, str):
283 raise AstroidTypeError("Type names and field names must be strings")
284 if not name.isidentifier():
285 raise AstroidValueError(
286 "Type names and field names must be valid" + f"identifiers: {name!r}"
287 )
288 if keyword.iskeyword(name):
289 raise AstroidValueError(
290 f"Type names and field names cannot be a keyword: {name!r}"
291 )
293 seen = set()
294 for name in attributes:
295 if name.startswith("_") and not rename:
296 raise AstroidValueError(
297 f"Field names cannot start with an underscore: {name!r}"
298 )
299 if name in seen:
300 raise AstroidValueError(f"Encountered duplicate field name: {name!r}")
301 seen.add(name)
302 # </snippet>
304 return attributes
307def infer_enum(
308 node: nodes.Call, context: InferenceContext | None = None
309) -> Iterator[bases.Instance]:
310 """Specific inference function for enum Call node."""
311 # Raise `UseInferenceDefault` if `node` is a call to a a user-defined Enum.
312 try:
313 inferred = node.func.infer(context)
314 except (InferenceError, StopIteration) as exc:
315 raise UseInferenceDefault from exc
317 if not any(
318 isinstance(item, nodes.ClassDef) and item.qname() == ENUM_QNAME
319 for item in inferred
320 ):
321 raise UseInferenceDefault
323 enum_meta = _extract_single_node(
324 """
325 class EnumMeta(object):
326 'docstring'
327 def __call__(self, node):
328 class EnumAttribute(object):
329 name = ''
330 value = 0
331 return EnumAttribute()
332 def __iter__(self):
333 class EnumAttribute(object):
334 name = ''
335 value = 0
336 return [EnumAttribute()]
337 def __reversed__(self):
338 class EnumAttribute(object):
339 name = ''
340 value = 0
341 return (EnumAttribute, )
342 def __next__(self):
343 return next(iter(self))
344 def __getitem__(self, attr):
345 class Value(object):
346 @property
347 def name(self):
348 return ''
349 @property
350 def value(self):
351 return attr
353 return Value()
354 __members__ = ['']
355 """
356 )
358 # FIXME arguably, the base here shouldn't be the EnumMeta class definition
359 # itself, but a reference (Name) to it. Otherwise, the invariant that all
360 # children of a node have that node as their parent is broken.
361 class_node = infer_func_form(
362 node,
363 enum_meta,
364 parent=SYNTHETIC_ROOT,
365 context=context,
366 enum=True,
367 )[0]
368 return iter([class_node.instantiate_class()])
371INT_FLAG_ADDITION_METHODS = """
372 def __or__(self, other):
373 return {name}(self.value | other.value)
374 def __and__(self, other):
375 return {name}(self.value & other.value)
376 def __xor__(self, other):
377 return {name}(self.value ^ other.value)
378 def __add__(self, other):
379 return {name}(self.value + other.value)
380 def __div__(self, other):
381 return {name}(self.value / other.value)
382 def __invert__(self):
383 return {name}(~self.value)
384 def __mul__(self, other):
385 return {name}(self.value * other.value)
386"""
389def infer_enum_class(node: nodes.ClassDef) -> nodes.ClassDef:
390 """Specific inference for enums."""
391 for basename in (b for cls in node.mro() for b in cls.basenames):
392 if node.root().name == "enum":
393 # Skip if the class is directly from enum module.
394 break
395 dunder_members = {}
396 target_names = set()
397 for local, values in node.locals.items():
398 if (
399 any(not isinstance(value, nodes.AssignName) for value in values)
400 or local == "_ignore_"
401 ):
402 continue
404 stmt = values[0].statement()
405 if isinstance(stmt, nodes.Assign):
406 if isinstance(stmt.targets[0], nodes.Tuple):
407 targets = stmt.targets[0].itered()
408 else:
409 targets = stmt.targets
410 elif isinstance(stmt, nodes.AnnAssign):
411 targets = [stmt.target]
412 else:
413 continue
415 inferred_return_value = None
416 if stmt.value is not None:
417 if isinstance(stmt.value, nodes.Const):
418 if isinstance(stmt.value.value, str):
419 inferred_return_value = repr(stmt.value.value)
420 else:
421 inferred_return_value = stmt.value.value
422 else:
423 inferred_return_value = stmt.value.as_string()
425 new_targets = []
426 for target in targets:
427 if isinstance(target, nodes.Starred):
428 continue
429 target_names.add(target.name)
430 # Replace all the assignments with our mocked class.
431 classdef = dedent(
432 """
433 class {name}({types}):
434 @property
435 def value(self):
436 return {return_value}
437 @property
438 def _value_(self):
439 return {return_value}
440 @property
441 def name(self):
442 return "{name}"
443 @property
444 def _name_(self):
445 return "{name}"
446 """.format(
447 name=target.name,
448 types=", ".join(node.basenames),
449 return_value=inferred_return_value,
450 )
451 )
452 if "IntFlag" in basename:
453 # Alright, we need to add some additional methods.
454 # Unfortunately we still can't infer the resulting objects as
455 # Enum members, but once we'll be able to do that, the following
456 # should result in some nice symbolic execution
457 classdef += INT_FLAG_ADDITION_METHODS.format(name=target.name)
459 fake = AstroidBuilder(
460 AstroidManager(), apply_transforms=False
461 ).string_build(classdef)[target.name]
462 fake.parent = target.parent
463 for method in node.mymethods():
464 fake.locals[method.name] = [method]
465 new_targets.append(fake.instantiate_class())
466 if stmt.value is None:
467 continue
468 dunder_members[local] = fake
469 node.locals[local] = new_targets
471 # The undocumented `_value2member_map_` member:
472 node.locals["_value2member_map_"] = [
473 nodes.Dict(
474 parent=node,
475 lineno=node.lineno,
476 col_offset=node.col_offset,
477 end_lineno=node.end_lineno,
478 end_col_offset=node.end_col_offset,
479 )
480 ]
482 members = nodes.Dict(
483 parent=node,
484 lineno=node.lineno,
485 col_offset=node.col_offset,
486 end_lineno=node.end_lineno,
487 end_col_offset=node.end_col_offset,
488 )
489 members.postinit(
490 [
491 (
492 nodes.Const(k, parent=members),
493 nodes.Name(
494 v.name,
495 parent=members,
496 lineno=v.lineno,
497 col_offset=v.col_offset,
498 end_lineno=v.end_lineno,
499 end_col_offset=v.end_col_offset,
500 ),
501 )
502 for k, v in dunder_members.items()
503 ]
504 )
505 node.locals["__members__"] = [members]
506 # The enum.Enum class itself defines two @DynamicClassAttribute data-descriptors
507 # "name" and "value" (which we override in the mocked class for each enum member
508 # above). When dealing with inference of an arbitrary instance of the enum
509 # class, e.g. in a method defined in the class body like:
510 # class SomeEnum(enum.Enum):
511 # def method(self):
512 # self.name # <- here
513 # In the absence of an enum member called "name" or "value", these attributes
514 # should resolve to the descriptor on that particular instance, i.e. enum member.
515 # For "value", we have no idea what that should be, but for "name", we at least
516 # know that it should be a string, so infer that as a guess.
517 if "name" not in target_names:
518 code = dedent(
519 '''
520 @property
521 def name(self):
522 """The name of the Enum member.
524 This is a reconstruction by astroid: enums are too dynamic to understand, but we at least
525 know 'name' should be a string, so this is astroid's best guess.
526 """
527 return ''
528 '''
529 )
530 name_dynamicclassattr = AstroidBuilder(AstroidManager()).string_build(code)[
531 "name"
532 ]
533 node.locals["name"] = [name_dynamicclassattr]
534 break
535 return node
538def infer_typing_namedtuple_class(class_node, context: InferenceContext | None = None):
539 """Infer a subclass of typing.NamedTuple."""
540 # Check if it has the corresponding bases
541 annassigns_fields = [
542 annassign.target.name
543 for annassign in class_node.body
544 if isinstance(annassign, nodes.AnnAssign)
545 ]
546 code = dedent(
547 """
548 from collections import namedtuple
549 namedtuple({typename!r}, {fields!r})
550 """
551 ).format(typename=class_node.name, fields=",".join(annassigns_fields))
552 node = extract_node(code)
553 try:
554 generated_class_node = next(infer_named_tuple(node, context))
555 except StopIteration as e:
556 raise InferenceError(node=node, context=context) from e
557 for method in class_node.mymethods():
558 generated_class_node.locals[method.name] = [method]
560 for body_node in class_node.body:
561 if isinstance(body_node, nodes.Assign):
562 for target in body_node.targets:
563 attr = target.name
564 generated_class_node.locals[attr] = class_node.locals[attr]
565 elif isinstance(body_node, nodes.ClassDef):
566 generated_class_node.locals[body_node.name] = [body_node]
568 return iter((generated_class_node,))
571def infer_typing_namedtuple_function(node, context: InferenceContext | None = None):
572 """
573 Starting with python3.9, NamedTuple is a function of the typing module.
574 The class NamedTuple is build dynamically through a call to `type` during
575 initialization of the `_NamedTuple` variable.
576 """
577 klass = extract_node(
578 """
579 from typing import _NamedTuple
580 _NamedTuple
581 """
582 )
583 return klass.infer(context)
586def infer_typing_namedtuple(
587 node: nodes.Call, context: InferenceContext | None = None
588) -> Iterator[nodes.ClassDef]:
589 """Infer a typing.NamedTuple(...) call."""
590 # This is essentially a namedtuple with different arguments
591 # so we extract the args and infer a named tuple.
592 try:
593 func = next(node.func.infer())
594 except (InferenceError, StopIteration) as exc:
595 raise UseInferenceDefault from exc
597 if func.qname() not in TYPING_NAMEDTUPLE_QUALIFIED:
598 raise UseInferenceDefault
600 if len(node.args) != 2:
601 raise UseInferenceDefault
603 if not isinstance(node.args[1], (nodes.List, nodes.Tuple)):
604 raise UseInferenceDefault
606 return infer_named_tuple(node, context)
609def _get_namedtuple_fields(node: nodes.Call) -> str:
610 """Get and return fields of a NamedTuple in code-as-a-string.
612 Because the fields are represented in their code form we can
613 extract a node from them later on.
614 """
615 names = []
616 container = None
617 try:
618 container = next(node.args[1].infer())
619 except (InferenceError, StopIteration) as exc:
620 raise UseInferenceDefault from exc
621 # We pass on IndexError as we'll try to infer 'field_names' from the keywords
622 except IndexError:
623 pass
624 if not container:
625 for keyword_node in node.keywords:
626 if keyword_node.arg == "field_names":
627 try:
628 container = next(keyword_node.value.infer())
629 except (InferenceError, StopIteration) as exc:
630 raise UseInferenceDefault from exc
631 break
632 if not isinstance(container, nodes.BaseContainer):
633 raise UseInferenceDefault
634 for elt in container.elts:
635 if isinstance(elt, nodes.Const):
636 names.append(elt.as_string())
637 continue
638 if not isinstance(elt, (nodes.List, nodes.Tuple)):
639 raise UseInferenceDefault
640 if len(elt.elts) != 2:
641 raise UseInferenceDefault
642 names.append(elt.elts[0].as_string())
644 if names:
645 field_names = f"({','.join(names)},)"
646 else:
647 field_names = ""
648 return field_names
651def _is_enum_subclass(cls: astroid.ClassDef) -> bool:
652 """Return whether cls is a subclass of an Enum."""
653 return cls.is_subtype_of("enum.Enum")
656def register(manager: AstroidManager) -> None:
657 manager.register_transform(
658 nodes.Call, inference_tip(infer_named_tuple), _looks_like_namedtuple
659 )
660 manager.register_transform(nodes.Call, inference_tip(infer_enum), _looks_like_enum)
661 manager.register_transform(
662 nodes.ClassDef, infer_enum_class, predicate=_is_enum_subclass
663 )
664 manager.register_transform(
665 nodes.ClassDef,
666 inference_tip(infer_typing_namedtuple_class),
667 _has_namedtuple_base,
668 )
669 manager.register_transform(
670 nodes.FunctionDef,
671 inference_tip(infer_typing_namedtuple_function),
672 lambda node: node.name == "NamedTuple"
673 and getattr(node.root(), "name", None) == "typing",
674 )
675 manager.register_transform(
676 nodes.Call,
677 inference_tip(infer_typing_namedtuple),
678 _looks_like_typing_namedtuple,
679 )