Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""
2Helpers for the API
3"""
4import re
5import sys
6from collections import namedtuple
7from contextlib import contextmanager
8from textwrap import dedent
9from itertools import chain
10from functools import wraps
11from inspect import Parameter
12from typing import Callable, TypeVar, Any, ParamSpec
14from parso.python.parser import Parser
15from parso.python import tree
17from jedi.inference.base_value import NO_VALUES
18from jedi.inference.syntax_tree import infer_atom
19from jedi.inference.helpers import infer_call_of_leaf
20from jedi.inference.compiled import get_string_value_set
21from jedi.cache import signature_time_cache, memoize_method
22from jedi.parser_utils import get_parent_scope
24T = TypeVar("T")
25P = ParamSpec("P")
27CompletionParts = namedtuple('CompletionParts', ['path', 'has_dot', 'name'])
29RECURSION_LIMIT = 3000
32@contextmanager
33def higher_recursion_limit():
34 """Jedi needs a higher recursion limit for parsing and inference. Raise it
35 only temporarily, so importing Jedi does not change global interpreter
36 state and a user-defined higher limit is never lowered."""
37 old_limit = sys.getrecursionlimit()
38 if old_limit >= RECURSION_LIMIT:
39 yield
40 else:
41 sys.setrecursionlimit(RECURSION_LIMIT)
42 try:
43 yield
44 finally:
45 sys.setrecursionlimit(old_limit)
48def recursion_limit(func: Callable[P, T]) -> Callable[P, T]:
49 @wraps(func)
50 def wrapper(*args, **kwargs):
51 with higher_recursion_limit():
52 return func(*args, **kwargs)
53 return wrapper
56def recursion_limit_generator(func):
57 @wraps(func)
58 def wrapper(*args, **kwargs):
59 with higher_recursion_limit():
60 iterator = iter(func(*args, **kwargs))
61 while True:
62 with higher_recursion_limit():
63 try:
64 value = next(iterator)
65 except StopIteration:
66 return
67 # Restore the caller's limit before yielding a result.
68 yield value
69 return wrapper
72def _start_match(string, like_name):
73 return string.startswith(like_name)
76def _fuzzy_match(string, like_name):
77 if len(like_name) <= 1:
78 return like_name in string
79 pos = string.find(like_name[0])
80 if pos >= 0:
81 return _fuzzy_match(string[pos + 1:], like_name[1:])
82 return False
85def match(string, like_name, fuzzy=False):
86 if fuzzy:
87 return _fuzzy_match(string, like_name)
88 else:
89 return _start_match(string, like_name)
92def sorted_definitions(defs):
93 # Note: `or ''` below is required because `module_path` could be
94 return sorted(defs, key=lambda x: (str(x.module_path or ''),
95 x.line or 0,
96 x.column or 0,
97 x.name))
100def get_on_completion_name(module_node, lines, position):
101 leaf = module_node.get_leaf_for_position(position)
102 if leaf is None or leaf.type in ('string', 'error_leaf'):
103 # Completions inside strings are a bit special, we need to parse the
104 # string. The same is true for comments and error_leafs.
105 line = lines[position[0] - 1]
106 # The first step of completions is to get the name
107 return re.search(r'(?!\d)\w+$|$', line[:position[1]]).group(0)
108 elif leaf.type not in ('name', 'keyword'):
109 return ''
111 return leaf.value[:position[1] - leaf.start_pos[1]]
114def _get_code(code_lines, start_pos, end_pos):
115 # Get relevant lines.
116 lines = code_lines[start_pos[0] - 1:end_pos[0]]
117 # Remove the parts at the end of the line.
118 lines[-1] = lines[-1][:end_pos[1]]
119 # Remove first line indentation.
120 lines[0] = lines[0][start_pos[1]:]
121 return ''.join(lines)
124class OnErrorLeaf(Exception):
125 @property
126 def error_leaf(self):
127 return self.args[0]
130def _get_code_for_stack(code_lines, leaf, position):
131 # It might happen that we're on whitespace or on a comment. This means
132 # that we would not get the right leaf.
133 if leaf.start_pos >= position:
134 # If we're not on a comment simply get the previous leaf and proceed.
135 leaf = leaf.get_previous_leaf()
136 if leaf is None:
137 return '' # At the beginning of the file.
139 is_after_newline = leaf.type == 'newline'
140 while leaf.type == 'newline':
141 leaf = leaf.get_previous_leaf()
142 if leaf is None:
143 return ''
145 if leaf.type == 'error_leaf' or leaf.type == 'string':
146 if leaf.start_pos[0] < position[0]:
147 # On a different line, we just begin anew.
148 return ''
150 # Error leafs cannot be parsed, completion in strings is also
151 # impossible.
152 raise OnErrorLeaf(leaf)
153 else:
154 user_stmt = leaf
155 while True:
156 if user_stmt.parent.type in ('file_input', 'suite', 'simple_stmt'):
157 break
158 user_stmt = user_stmt.parent
160 if is_after_newline:
161 if user_stmt.start_pos[1] > position[1]:
162 # This means that it's actually a dedent and that means that we
163 # start without value (part of a suite).
164 return ''
166 # This is basically getting the relevant lines.
167 return _get_code(code_lines, user_stmt.get_start_pos_of_prefix(), position)
170def get_stack_at_position(grammar, code_lines, leaf, pos):
171 """
172 Returns the possible node names (e.g. import_from, xor_test or yield_stmt).
173 """
174 class EndMarkerReached(Exception):
175 pass
177 def tokenize_without_endmarker(code):
178 # TODO This is for now not an official parso API that exists purely
179 # for Jedi.
180 tokens = grammar._tokenize(code)
181 for token in tokens:
182 if token.string == safeword:
183 raise EndMarkerReached()
184 elif token.prefix.endswith(safeword):
185 # This happens with comments.
186 raise EndMarkerReached()
187 elif token.string.endswith(safeword):
188 yield token # Probably an f-string literal that was not finished.
189 raise EndMarkerReached()
190 else:
191 yield token
193 # The code might be indedented, just remove it.
194 code = dedent(_get_code_for_stack(code_lines, leaf, pos))
195 # We use a word to tell Jedi when we have reached the start of the
196 # completion.
197 # Use Z as a prefix because it's not part of a number suffix.
198 safeword = 'ZZZ_USER_WANTS_TO_COMPLETE_HERE_WITH_JEDI'
199 code = code + ' ' + safeword
201 p = Parser(grammar._pgen_grammar, error_recovery=True)
202 try:
203 p.parse(tokens=tokenize_without_endmarker(code))
204 except EndMarkerReached:
205 return p.stack
206 raise SystemError(
207 "This really shouldn't happen. There's a bug in Jedi:\n%s"
208 % list(tokenize_without_endmarker(code))
209 )
212def infer(inference_state, context, leaf):
213 if leaf.type == 'name':
214 return inference_state.infer(context, leaf)
216 parent = leaf.parent
217 definitions = NO_VALUES
218 if parent.type == 'atom':
219 # e.g. `(a + b)`
220 definitions = context.infer_node(leaf.parent)
221 elif parent.type == 'trailer':
222 # e.g. `a()`
223 definitions = infer_call_of_leaf(context, leaf)
224 elif isinstance(leaf, tree.Literal):
225 # e.g. `"foo"` or `1.0`
226 return infer_atom(context, leaf)
227 elif leaf.type in ('fstring_string', 'fstring_start', 'fstring_end'):
228 return get_string_value_set(inference_state)
229 return definitions
232def filter_follow_imports(names, follow_builtin_imports=False):
233 for name in names:
234 if name.is_import():
235 new_names = list(filter_follow_imports(
236 name.goto(),
237 follow_builtin_imports=follow_builtin_imports,
238 ))
239 found_builtin = False
240 if follow_builtin_imports:
241 for new_name in new_names:
242 if new_name.start_pos is None:
243 found_builtin = True
245 if found_builtin:
246 yield name
247 else:
248 yield from new_names
249 else:
250 yield name
253class CallDetails:
254 def __init__(self, bracket_leaf, children, position):
255 self.bracket_leaf = bracket_leaf
256 self._children = children
257 self._position = position
259 @property
260 def index(self):
261 return _get_index_and_key(self._children, self._position)[0]
263 @property
264 def keyword_name_str(self):
265 return _get_index_and_key(self._children, self._position)[1]
267 @memoize_method
268 def _list_arguments(self):
269 return list(_iter_arguments(self._children, self._position))
271 def calculate_index(self, param_names):
272 positional_count = 0
273 used_names = set()
274 star_count = -1
275 args = self._list_arguments()
276 if not args:
277 if param_names:
278 return 0
279 else:
280 return None
282 is_kwarg = False
283 for i, (star_count, key_start, had_equal) in enumerate(args):
284 is_kwarg |= had_equal | (star_count == 2)
285 if star_count:
286 pass # For now do nothing, we don't know what's in there here.
287 else:
288 if i + 1 != len(args): # Not last
289 if had_equal:
290 used_names.add(key_start)
291 else:
292 positional_count += 1
294 for i, param_name in enumerate(param_names):
295 kind = param_name.get_kind()
297 if not is_kwarg:
298 if kind == Parameter.VAR_POSITIONAL:
299 return i
300 if kind in (Parameter.POSITIONAL_OR_KEYWORD, Parameter.POSITIONAL_ONLY):
301 if i == positional_count:
302 return i
304 if key_start is not None and not star_count == 1 or star_count == 2:
305 if param_name.string_name not in used_names \
306 and (kind == Parameter.KEYWORD_ONLY
307 or kind == Parameter.POSITIONAL_OR_KEYWORD
308 and positional_count <= i):
309 if star_count:
310 return i
311 if had_equal:
312 if param_name.string_name == key_start:
313 return i
314 else:
315 if param_name.string_name.startswith(key_start):
316 return i
318 if kind == Parameter.VAR_KEYWORD:
319 return i
320 return None
322 def iter_used_keyword_arguments(self):
323 for star_count, key_start, had_equal in list(self._list_arguments()):
324 if had_equal and key_start:
325 yield key_start
327 def count_positional_arguments(self):
328 count = 0
329 for star_count, key_start, had_equal in self._list_arguments()[:-1]:
330 if star_count or key_start:
331 break
332 count += 1
333 return count
336def _iter_arguments(nodes, position):
337 def remove_after_pos(name):
338 if name.type != 'name':
339 return None
340 return name.value[:position[1] - name.start_pos[1]]
342 # Returns Generator[Tuple[star_count, Optional[key_start: str], had_equal]]
343 nodes_before = [c for c in nodes if c.start_pos < position]
344 if nodes_before[-1].type == 'arglist':
345 yield from _iter_arguments(nodes_before[-1].children, position)
346 return
348 previous_node_yielded = False
349 stars_seen = 0
350 for i, node in enumerate(nodes_before):
351 if node.type == 'argument':
352 previous_node_yielded = True
353 first = node.children[0]
354 second = node.children[1]
355 if second == '=':
356 if second.start_pos < position and first.type == 'name':
357 yield 0, first.value, True
358 else:
359 yield 0, remove_after_pos(first), False
360 elif first in ('*', '**'):
361 yield len(first.value), remove_after_pos(second), False
362 else:
363 # Must be a Comprehension
364 first_leaf = node.get_first_leaf()
365 if first_leaf.type == 'name' and first_leaf.start_pos >= position:
366 yield 0, remove_after_pos(first_leaf), False
367 else:
368 yield 0, None, False
369 stars_seen = 0
370 elif node.type == 'testlist_star_expr':
371 for n in node.children[::2]:
372 if n.type == 'star_expr':
373 stars_seen = 1
374 n = n.children[1]
375 yield stars_seen, remove_after_pos(n), False
376 stars_seen = 0
377 # The count of children is even if there's a comma at the end.
378 previous_node_yielded = bool(len(node.children) % 2)
379 elif isinstance(node, tree.PythonLeaf) and node.value == ',':
380 if not previous_node_yielded:
381 yield stars_seen, '', False
382 stars_seen = 0
383 previous_node_yielded = False
384 elif isinstance(node, tree.PythonLeaf) and node.value in ('*', '**'):
385 stars_seen = len(node.value)
386 elif node == '=' and nodes_before[-1]:
387 previous_node_yielded = True
388 before = nodes_before[i - 1]
389 if before.type == 'name':
390 yield 0, before.value, True
391 else:
392 yield 0, None, False
393 # Just ignore the star that is probably a syntax error.
394 stars_seen = 0
396 if not previous_node_yielded:
397 if nodes_before[-1].type == 'name':
398 yield stars_seen, remove_after_pos(nodes_before[-1]), False
399 else:
400 yield stars_seen, '', False
403def _get_index_and_key(nodes, position):
404 """
405 Returns the amount of commas and the keyword argument string.
406 """
407 nodes_before = [c for c in nodes if c.start_pos < position]
408 if nodes_before[-1].type == 'arglist':
409 return _get_index_and_key(nodes_before[-1].children, position)
411 key_str = None
413 last = nodes_before[-1]
414 if last.type == 'argument' and last.children[1] == '=' \
415 and last.children[1].end_pos <= position:
416 # Checked if the argument
417 key_str = last.children[0].value
418 elif last == '=':
419 key_str = nodes_before[-2].value
421 return nodes_before.count(','), key_str
424def _get_signature_details_from_error_node(node, additional_children, position):
425 for index, element in reversed(list(enumerate(node.children))):
426 # `index > 0` means that it's a trailer and not an atom.
427 if element == '(' and element.end_pos <= position and index > 0:
428 # It's an error node, we don't want to match too much, just
429 # until the parentheses is enough.
430 children = node.children[index:]
431 name = element.get_previous_leaf()
432 if name is None:
433 continue
434 if name.type == 'name' or name.parent.type in ('trailer', 'atom'):
435 return CallDetails(element, children + additional_children, position)
438def get_signature_details(module, position):
439 leaf = module.get_leaf_for_position(position, include_prefixes=True)
440 # It's easier to deal with the previous token than the next one in this
441 # case.
442 if leaf.start_pos >= position:
443 # Whitespace / comments after the leaf count towards the previous leaf.
444 leaf = leaf.get_previous_leaf()
445 if leaf is None:
446 return None
448 # Now that we know where we are in the syntax tree, we start to look at
449 # parents for possible function definitions.
450 node = leaf.parent
451 while node is not None:
452 if node.type in ('funcdef', 'classdef', 'decorated', 'async_stmt'):
453 # Don't show signatures if there's stuff before it that just
454 # makes it feel strange to have a signature.
455 return None
457 additional_children = []
458 for n in reversed(node.children):
459 if n.start_pos < position:
460 if n.type == 'error_node':
461 result = _get_signature_details_from_error_node(
462 n, additional_children, position
463 )
464 if result is not None:
465 return result
467 additional_children[0:0] = n.children
468 continue
469 additional_children.insert(0, n)
471 # Find a valid trailer
472 if node.type == 'trailer' and node.children[0] == '(' \
473 or node.type == 'decorator' and node.children[2] == '(':
474 # Additionally we have to check that an ending parenthesis isn't
475 # interpreted wrong. There are two cases:
476 # 1. Cursor before paren -> The current signature is good
477 # 2. Cursor after paren -> We need to skip the current signature
478 if not (leaf is node.children[-1] and position >= leaf.end_pos):
479 leaf = node.get_previous_leaf()
480 if leaf is None:
481 return None
482 return CallDetails(
483 node.children[0] if node.type == 'trailer' else node.children[2],
484 node.children,
485 position
486 )
488 node = node.parent
490 return None
493@signature_time_cache("call_signatures_validity")
494def cache_signatures(inference_state, context, bracket_leaf, code_lines, user_pos):
495 """This function calculates the cache key."""
496 line_index = user_pos[0] - 1
498 before_cursor = code_lines[line_index][:user_pos[1]]
499 other_lines = code_lines[bracket_leaf.start_pos[0]:line_index]
500 whole = ''.join(other_lines + [before_cursor])
501 before_bracket = re.match(r'.*\(', whole, re.DOTALL)
503 module_path = context.get_root_context().py__file__()
504 if module_path is None:
505 yield None # Don't cache!
506 else:
507 yield (module_path, before_bracket, bracket_leaf.start_pos)
508 yield infer(
509 inference_state,
510 context,
511 bracket_leaf.get_previous_leaf(),
512 )
515def validate_line_column(func: Callable[P, T]) -> Callable[P, T]:
516 @wraps(func)
517 def wrapper(self, line=None, column=None, *args, **kwargs) -> Any:
518 line = max(len(self._code_lines), 1) if line is None else line
519 if not (0 < line <= len(self._code_lines)):
520 raise ValueError('`line` parameter is not in a valid range.')
522 line_string = self._code_lines[line - 1]
523 line_len = len(line_string)
524 if line_string.endswith('\r\n'):
525 line_len -= 2
526 elif line_string.endswith('\n'):
527 line_len -= 1
529 column = line_len if column is None else column
530 if not (0 <= column <= line_len):
531 raise ValueError('`column` parameter (%d) is not in a valid range '
532 '(0-%d) for line %d (%r).' % (
533 column, line_len, line, line_string))
534 return func(self, line, column, *args, **kwargs) # type: ignore[arg-type]
535 return wrapper # type: ignore[return-value]
538def get_module_names(module, all_scopes, definitions=True, references=False):
539 """
540 Returns a dictionary with name parts as keys and their call paths as
541 values.
542 """
543 def def_ref_filter(name):
544 is_def = name.is_definition()
545 return definitions and is_def or references and not is_def
547 names = list(chain.from_iterable(module.get_used_names().values()))
548 if not all_scopes:
549 # We have to filter all the names that don't have the module as a
550 # parent_scope. There's None as a parent, because nodes in the module
551 # node have the parent module and not suite as all the others.
552 # Therefore it's important to catch that case.
554 def is_module_scope_name(name):
555 parent_scope = get_parent_scope(name)
556 # async functions have an extra wrapper. Strip it.
557 if parent_scope and parent_scope.type == 'async_stmt':
558 parent_scope = parent_scope.parent
559 return parent_scope in (module, None)
561 names = [n for n in names if is_module_scope_name(n)]
562 return filter(def_ref_filter, names)
565def split_search_string(name):
566 type, _, dotted_names = name.rpartition(' ')
567 if type == 'def':
568 type = 'function'
569 return type, dotted_names.split('.')