Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""
2Contains all classes and functions to deal with lists, dicts, generators and
3iterators in general.
4"""
5from typing import Any
7from jedi.inference import compiled
8from jedi.inference import analysis
9from jedi.inference.lazy_value import LazyKnownValue, LazyKnownValues, \
10 LazyTreeValue
11from jedi.inference.helpers import get_int_or_none, is_string, \
12 reraise_getitem_errors, SimpleGetItemNotFound
13from jedi.inference.utils import safe_property, to_list
14from jedi.inference.cache import inference_state_method_cache
15from jedi.inference.filters import LazyAttributeOverwrite, publish_method
16from jedi.inference.base_value import ValueSet, Value, NO_VALUES, \
17 ContextualizedNode, iterate_values, sentinel, \
18 LazyValueWrapper
19from jedi.parser_utils import get_sync_comp_fors
20from jedi.inference.context import CompForContext
21from jedi.inference.value.dynamic_arrays import check_array_additions
24class IterableMixin:
25 py__iter__: Any
26 inference_state: Any
28 def py__next__(self, contextualized_node=None):
29 return self.py__iter__(contextualized_node)
31 def py__stop_iteration_returns(self):
32 return ValueSet([compiled.builtin_from_name(self.inference_state, 'None')])
34 # At the moment, safe values are simple values like "foo", 1 and not
35 # lists/dicts. Therefore as a small speed optimization we can just do the
36 # default instead of resolving the lazy wrapped values, that are just
37 # doing this in the end as well.
38 # This mostly speeds up patterns like `sys.version_info >= (3, 0)` in
39 # typeshed.
40 get_safe_value = Value.get_safe_value
43class GeneratorBase(LazyAttributeOverwrite, IterableMixin):
44 array_type = None
46 def _get_wrapped_value(self):
47 instance, = self._get_cls().execute_annotation(None)
48 return instance
50 def _get_cls(self):
51 generator, = self.inference_state.types_module.py__getattribute__('GeneratorType')
52 return generator
54 def py__bool__(self):
55 return True
57 @publish_method('__iter__')
58 def _iter(self, arguments):
59 return ValueSet([self])
61 @publish_method('send')
62 @publish_method('__next__')
63 def _next(self, arguments):
64 return ValueSet.from_sets(lazy_value.infer() for lazy_value in self.py__iter__())
66 def py__stop_iteration_returns(self):
67 return ValueSet([compiled.builtin_from_name(self.inference_state, 'None')])
69 @property
70 def name(self):
71 return compiled.CompiledValueName(self, 'Generator')
73 def get_annotated_class_object(self):
74 from jedi.inference.gradual.generics import TupleGenericManager
75 gen_values = self.merge_types_of_iterate().py__class__()
76 gm = TupleGenericManager((gen_values, NO_VALUES, NO_VALUES))
77 return self._get_cls().with_generics(gm)
80class Generator(GeneratorBase):
81 """Handling of `yield` functions."""
82 def __init__(self, inference_state, func_execution_context):
83 super().__init__(inference_state)
84 self._func_execution_context = func_execution_context
86 def py__iter__(self, contextualized_node=None):
87 iterators = self._func_execution_context.infer_annotations()
88 if iterators:
89 return iterators.iterate(contextualized_node)
90 return self._func_execution_context.get_yield_lazy_values()
92 def py__stop_iteration_returns(self):
93 return self._func_execution_context.get_return_values()
95 def __repr__(self):
96 return "<%s of %s>" % (type(self).__name__, self._func_execution_context)
99def comprehension_from_atom(inference_state, value, atom):
100 bracket = atom.children[0]
101 test_list_comp = atom.children[1]
103 if bracket == '{':
104 if atom.children[1].children[1] == ':':
105 sync_comp_for = test_list_comp.children[3]
106 if sync_comp_for.type == 'comp_for':
107 sync_comp_for = sync_comp_for.children[1]
109 return DictComprehension(
110 inference_state,
111 value,
112 sync_comp_for_node=sync_comp_for,
113 key_node=test_list_comp.children[0],
114 value_node=test_list_comp.children[2],
115 )
116 else:
117 cls = SetComprehension
118 elif bracket == '(':
119 cls = GeneratorComprehension
120 elif bracket == '[':
121 cls = ListComprehension
123 sync_comp_for = test_list_comp.children[1]
124 if sync_comp_for.type == 'comp_for':
125 sync_comp_for = sync_comp_for.children[1]
127 return cls(
128 inference_state,
129 defining_context=value,
130 sync_comp_for_node=sync_comp_for,
131 entry_node=test_list_comp.children[0],
132 )
135class ComprehensionMixin:
136 _defining_context: Any
137 _entry_node: Any
138 array_type: Any
139 _value_node: Any
140 _sync_comp_for_node: Any
142 @inference_state_method_cache()
143 def _get_comp_for_context(self, parent_context, comp_for):
144 return CompForContext(parent_context, comp_for)
146 def _nested(self, comp_fors, parent_context=None):
147 comp_for = comp_fors[0]
149 is_async = comp_for.parent.type == 'comp_for'
151 input_node = comp_for.children[3]
152 parent_context = parent_context or self._defining_context
153 input_types = parent_context.infer_node(input_node)
155 cn = ContextualizedNode(parent_context, input_node)
156 iterated = input_types.iterate(cn, is_async=is_async)
157 exprlist = comp_for.children[1]
158 for i, lazy_value in enumerate(iterated):
159 types = lazy_value.infer()
160 dct = unpack_tuple_to_dict(parent_context, types, exprlist)
161 context = self._get_comp_for_context(
162 parent_context,
163 comp_for,
164 )
165 with context.predefine_names(comp_for, dct):
166 try:
167 yield from self._nested(comp_fors[1:], context)
168 except IndexError:
169 iterated = context.infer_node(self._entry_node)
170 if self.array_type == 'dict':
171 yield iterated, context.infer_node(self._value_node)
172 else:
173 yield iterated
175 @inference_state_method_cache(default=[])
176 @to_list
177 def _iterate(self):
178 comp_fors = tuple(get_sync_comp_fors(self._sync_comp_for_node))
179 yield from self._nested(comp_fors)
181 def py__iter__(self, contextualized_node=None):
182 for set_ in self._iterate():
183 yield LazyKnownValues(set_)
185 def __repr__(self):
186 return "<%s of %s>" % (type(self).__name__, self._sync_comp_for_node)
189class _DictMixin:
190 get_mapping_item_values: Any
192 def _get_generics(self):
193 return tuple(c_set.py__class__() for c_set in self.get_mapping_item_values())
196class Sequence(LazyAttributeOverwrite, IterableMixin):
197 api_type = 'instance'
199 @property
200 def name(self):
201 return compiled.CompiledValueName(self, self.array_type)
203 def _get_generics(self):
204 return (self.merge_types_of_iterate().py__class__(),)
206 @inference_state_method_cache(default=())
207 def _cached_generics(self):
208 return self._get_generics()
210 def _get_wrapped_value(self):
211 from jedi.inference.gradual.base import GenericClass
212 from jedi.inference.gradual.generics import TupleGenericManager
213 klass = compiled.builtin_from_name(self.inference_state, self.array_type)
214 c, = GenericClass(
215 klass,
216 TupleGenericManager(self._cached_generics())
217 ).execute_annotation(None)
218 return c
220 def py__bool__(self):
221 return None # We don't know the length, because of appends.
223 @safe_property
224 def parent(self):
225 return self.inference_state.builtins_module
227 def py__getitem__(self, index_value_set, contextualized_node):
228 if self.array_type == 'dict':
229 return self._dict_values()
230 return iterate_values(ValueSet([self]))
233class _BaseComprehension(ComprehensionMixin):
234 def __init__(self, inference_state, defining_context, sync_comp_for_node, entry_node):
235 assert sync_comp_for_node.type == 'sync_comp_for'
236 super().__init__(inference_state) # type: ignore[call-arg]
237 self._defining_context = defining_context
238 self._sync_comp_for_node = sync_comp_for_node
239 self._entry_node = entry_node
242class ListComprehension(_BaseComprehension, Sequence):
243 array_type = 'list'
245 def py__simple_getitem__(self, index):
246 if isinstance(index, slice):
247 return ValueSet([self])
249 all_types = list(self.py__iter__())
250 with reraise_getitem_errors(IndexError, TypeError):
251 lazy_value = all_types[index]
252 return lazy_value.infer()
255class SetComprehension(_BaseComprehension, Sequence):
256 array_type = 'set'
259class GeneratorComprehension(_BaseComprehension, GeneratorBase):
260 pass
263class _DictKeyMixin:
264 _dict_keys: Any
265 _dict_values: Any
267 # TODO merge with _DictMixin?
268 def get_mapping_item_values(self):
269 return self._dict_keys(), self._dict_values()
271 def get_key_values(self):
272 # TODO merge with _dict_keys?
273 return self._dict_keys()
276class DictComprehension(ComprehensionMixin, Sequence, _DictKeyMixin):
277 array_type = 'dict'
279 def __init__(self, inference_state, defining_context, sync_comp_for_node, key_node, value_node):
280 assert sync_comp_for_node.type == 'sync_comp_for'
281 super().__init__(inference_state)
282 self._defining_context = defining_context
283 self._sync_comp_for_node = sync_comp_for_node
284 self._entry_node = key_node
285 self._value_node = value_node
287 def py__iter__(self, contextualized_node=None):
288 for keys, values in self._iterate():
289 yield LazyKnownValues(keys)
291 def py__simple_getitem__(self, index):
292 for keys, values in self._iterate():
293 for k in keys:
294 # Be careful in the future if refactoring, index could be a
295 # slice object.
296 if k.get_safe_value(default=object()) == index:
297 return values
298 raise SimpleGetItemNotFound()
300 def _dict_keys(self):
301 return ValueSet.from_sets(keys for keys, values in self._iterate())
303 def _dict_values(self):
304 return ValueSet.from_sets(values for keys, values in self._iterate())
306 @publish_method('values')
307 def _imitate_values(self, arguments):
308 lazy_value = LazyKnownValues(self._dict_values())
309 return ValueSet([FakeList(self.inference_state, [lazy_value])])
311 @publish_method('items')
312 def _imitate_items(self, arguments):
313 lazy_values = [
314 LazyKnownValue(
315 FakeTuple(
316 self.inference_state,
317 [LazyKnownValues(key),
318 LazyKnownValues(value)]
319 )
320 )
321 for key, value in self._iterate()
322 ]
324 return ValueSet([FakeList(self.inference_state, lazy_values)])
326 def exact_key_items(self):
327 # NOTE: A smarter thing can probably done here to achieve better
328 # completions, but at least like this jedi doesn't crash
329 return []
332class SequenceLiteralValue(Sequence):
333 _TUPLE_LIKE = 'testlist_star_expr', 'testlist', 'subscriptlist'
334 mapping = {'(': 'tuple',
335 '[': 'list',
336 '{': 'set'}
338 def __init__(self, inference_state, defining_context, atom):
339 super().__init__(inference_state)
340 self.atom = atom
341 self._defining_context = defining_context
343 if self.atom.type in self._TUPLE_LIKE:
344 self.array_type = 'tuple'
345 else:
346 self.array_type = SequenceLiteralValue.mapping[atom.children[0]]
347 """The builtin name of the array (list, set, tuple or dict)."""
349 def _get_generics(self):
350 if self.array_type == 'tuple':
351 return tuple(x.infer().py__class__() for x in self.py__iter__())
352 return super()._get_generics()
354 def py__simple_getitem__(self, index):
355 """Here the index is an int/str. Raises IndexError/KeyError."""
356 if isinstance(index, slice):
357 return ValueSet([self])
358 else:
359 with reraise_getitem_errors(TypeError, KeyError, IndexError):
360 node = self.get_tree_entries()[index]
361 if node == ':' or node.type == 'subscript':
362 return NO_VALUES
363 return self._defining_context.infer_node(node)
365 def py__iter__(self, contextualized_node=None):
366 """
367 While values returns the possible values for any array field, this
368 function returns the value for a certain index.
369 """
370 for node in self.get_tree_entries():
371 if node == ':' or node.type == 'subscript':
372 # TODO this should probably use at least part of the code
373 # of infer_subscript_list.
374 yield LazyKnownValue(Slice(self._defining_context, None, None, None))
375 else:
376 yield LazyTreeValue(self._defining_context, node)
377 yield from check_array_additions(self._defining_context, self)
379 def py__len__(self):
380 # This function is not really used often. It's more of a try.
381 return len(self.get_tree_entries())
383 def get_tree_entries(self):
384 c = self.atom.children
386 if self.atom.type in self._TUPLE_LIKE:
387 return c[::2]
389 array_node = c[1]
390 if array_node in (']', '}', ')'):
391 return [] # Direct closing bracket, doesn't contain items.
393 if array_node.type == 'testlist_comp':
394 # filter out (for now) pep 448 single-star unpacking
395 return [value for value in array_node.children[::2]
396 if value.type != "star_expr"]
397 elif array_node.type == 'dictorsetmaker':
398 kv = []
399 iterator = iter(array_node.children)
400 for key in iterator:
401 if key == "**":
402 # dict with pep 448 double-star unpacking
403 # for now ignoring the values imported by **
404 next(iterator)
405 next(iterator, None) # Possible comma.
406 else:
407 op = next(iterator, None)
408 if op is None or op == ',':
409 if key.type == "star_expr":
410 # pep 448 single-star unpacking
411 # for now ignoring values imported by *
412 pass
413 else:
414 kv.append(key) # A set.
415 else:
416 assert op == ':' # A dict.
417 kv.append((key, next(iterator)))
418 next(iterator, None) # Possible comma.
419 return kv
420 else:
421 if array_node.type == "star_expr":
422 # pep 448 single-star unpacking
423 # for now ignoring values imported by *
424 return []
425 else:
426 return [array_node]
428 def __repr__(self):
429 return "<%s of %s>" % (self.__class__.__name__, self.atom)
432class DictLiteralValue(_DictMixin, SequenceLiteralValue, _DictKeyMixin):
433 array_type = 'dict'
435 def __init__(self, inference_state, defining_context, atom):
436 # Intentionally don't call the super class. This is definitely a sign
437 # that the architecture is bad and we should refactor.
438 Sequence.__init__(self, inference_state)
439 self._defining_context = defining_context
440 self.atom = atom
442 def py__simple_getitem__(self, index):
443 """Here the index is an int/str. Raises IndexError/KeyError."""
444 compiled_value_index = compiled.create_simple_object(self.inference_state, index)
445 for key, value in self.get_tree_entries():
446 for k in self._defining_context.infer_node(key):
447 for key_v in k.execute_operation(compiled_value_index, '=='):
448 if key_v.get_safe_value():
449 return self._defining_context.infer_node(value)
450 raise SimpleGetItemNotFound('No key found in dictionary %s.' % self)
452 def py__iter__(self, contextualized_node=None):
453 """
454 While values returns the possible values for any array field, this
455 function returns the value for a certain index.
456 """
457 # Get keys.
458 types = NO_VALUES
459 for k, _ in self.get_tree_entries():
460 types |= self._defining_context.infer_node(k)
461 # We don't know which dict index comes first, therefore always
462 # yield all the types.
463 for _ in types:
464 yield LazyKnownValues(types)
466 @publish_method('values')
467 def _imitate_values(self, arguments):
468 lazy_value = LazyKnownValues(self._dict_values())
469 return ValueSet([FakeList(self.inference_state, [lazy_value])])
471 @publish_method('items')
472 def _imitate_items(self, arguments):
473 lazy_values = [
474 LazyKnownValue(FakeTuple(
475 self.inference_state,
476 (LazyTreeValue(self._defining_context, key_node),
477 LazyTreeValue(self._defining_context, value_node))
478 )) for key_node, value_node in self.get_tree_entries()
479 ]
481 return ValueSet([FakeList(self.inference_state, lazy_values)])
483 def exact_key_items(self):
484 """
485 Returns a generator of tuples like dict.items(), where the key is
486 resolved (as a string) and the values are still lazy values.
487 """
488 for key_node, value in self.get_tree_entries():
489 for key in self._defining_context.infer_node(key_node):
490 if is_string(key):
491 yield key.get_safe_value(), LazyTreeValue(self._defining_context, value)
493 def _dict_values(self):
494 return ValueSet.from_sets(
495 self._defining_context.infer_node(v)
496 for k, v in self.get_tree_entries()
497 )
499 def _dict_keys(self):
500 return ValueSet.from_sets(
501 self._defining_context.infer_node(k)
502 for k, v in self.get_tree_entries()
503 )
506class _FakeSequence(Sequence):
507 def __init__(self, inference_state, lazy_value_list):
508 """
509 type should be one of "tuple", "list"
510 """
511 super().__init__(inference_state)
512 self._lazy_value_list = lazy_value_list
514 def py__simple_getitem__(self, index):
515 if isinstance(index, slice):
516 return ValueSet([self])
518 with reraise_getitem_errors(IndexError, TypeError):
519 lazy_value = self._lazy_value_list[index]
520 return lazy_value.infer()
522 def py__iter__(self, contextualized_node=None):
523 return self._lazy_value_list
525 def py__bool__(self):
526 return bool(len(self._lazy_value_list))
528 def __repr__(self):
529 return "<%s of %s>" % (type(self).__name__, self._lazy_value_list)
532class FakeTuple(_FakeSequence):
533 array_type = 'tuple'
536class FakeList(_FakeSequence):
537 array_type = 'tuple'
540class FakeDict(_DictMixin, Sequence, _DictKeyMixin):
541 array_type = 'dict'
543 def __init__(self, inference_state, dct):
544 super().__init__(inference_state)
545 self._dct = dct
547 def py__iter__(self, contextualized_node=None):
548 for key in self._dct:
549 yield LazyKnownValue(compiled.create_simple_object(self.inference_state, key))
551 def py__simple_getitem__(self, index):
552 with reraise_getitem_errors(KeyError, TypeError):
553 lazy_value = self._dct[index]
554 return lazy_value.infer()
556 @publish_method('values')
557 def _values(self, arguments):
558 return ValueSet([FakeTuple(
559 self.inference_state,
560 [LazyKnownValues(self._dict_values())]
561 )])
563 def _dict_values(self):
564 return ValueSet.from_sets(lazy_value.infer() for lazy_value in self._dct.values())
566 def _dict_keys(self):
567 return ValueSet.from_sets(lazy_value.infer() for lazy_value in self.py__iter__())
569 def exact_key_items(self):
570 return self._dct.items()
572 def __repr__(self):
573 return '<%s: %s>' % (self.__class__.__name__, self._dct)
576class MergedArray(Sequence):
577 def __init__(self, inference_state, arrays):
578 super().__init__(inference_state)
579 self.array_type = arrays[-1].array_type
580 self._arrays = arrays
582 def py__iter__(self, contextualized_node=None):
583 for array in self._arrays:
584 yield from array.py__iter__()
586 def py__simple_getitem__(self, index):
587 return ValueSet.from_sets(lazy_value.infer() for lazy_value in self.py__iter__())
590def unpack_tuple_to_dict(context, types, exprlist):
591 """
592 Unpacking tuple assignments in for statements and expr_stmts.
593 """
594 if exprlist.type == 'name':
595 return {exprlist.value: types}
596 elif exprlist.type == 'atom' and exprlist.children[0] in ('(', '['):
597 return unpack_tuple_to_dict(context, types, exprlist.children[1])
598 elif exprlist.type in ('testlist', 'testlist_comp', 'exprlist',
599 'testlist_star_expr'):
600 dct = {}
601 parts = iter(exprlist.children[::2])
602 n = 0
603 for lazy_value in types.iterate(ContextualizedNode(context, exprlist)):
604 n += 1
605 try:
606 part = next(parts)
607 except StopIteration:
608 analysis.add(context, 'value-error-too-many-values', part,
609 message="ValueError: too many values to unpack (expected %s)" % n)
610 else:
611 dct.update(unpack_tuple_to_dict(context, lazy_value.infer(), part))
612 has_parts = next(parts, None)
613 if types and has_parts is not None:
614 analysis.add(context, 'value-error-too-few-values', has_parts,
615 message="ValueError: need more than %s values to unpack" % n)
616 return dct
617 elif exprlist.type == 'power' or exprlist.type == 'atom_expr':
618 # Something like ``arr[x], var = ...``.
619 # This is something that is not yet supported, would also be difficult
620 # to write into a dict.
621 return {}
622 elif exprlist.type == 'star_expr': # `a, *b, c = x` type unpackings
623 # Currently we're not supporting them.
624 return {}
625 raise NotImplementedError
628class Slice(LazyValueWrapper):
629 def __init__(self, python_context, start, stop, step):
630 self.inference_state = python_context.inference_state
631 self._context = python_context
632 # All of them are either a Precedence or None.
633 self._start = start
634 self._stop = stop
635 self._step = step
637 def _get_wrapped_value(self):
638 value = compiled.builtin_from_name(self._context.inference_state, 'slice')
639 slice_value, = value.execute_with_values()
640 return slice_value
642 def get_safe_value(self, default=sentinel):
643 """
644 Imitate CompiledValue.obj behavior and return a ``builtin.slice()``
645 object.
646 """
647 def get(element):
648 if element is None:
649 return None
651 result = self._context.infer_node(element)
652 if len(result) != 1:
653 # For simplicity, we want slices to be clear defined with just
654 # one type. Otherwise we will return an empty slice object.
655 raise IndexError
657 value, = result
658 return get_int_or_none(value)
660 try:
661 return slice(get(self._start), get(self._stop), get(self._step))
662 except IndexError:
663 return slice(None, None, None)