Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""
2:mod:`jedi.inference.imports` is here to resolve import statements and return
3the modules/classes/functions/whatever, which they stand for. However there's
4not any actual importing done. This module is about finding modules in the
5filesystem. This can be quite tricky sometimes, because Python imports are not
6always that simple.
8This module also supports import autocompletion, which means to complete
9statements like ``from datetim`` (cursor at the end would return ``datetime``).
10"""
11import os
12from pathlib import Path
14from parso.python import tree
16from jedi import debug
17from jedi import settings
18from jedi.file_io import FolderIO
19from jedi.parser_utils import get_cached_code_lines
20from jedi.inference import sys_path
21from jedi.inference import helpers
22from jedi.inference import compiled
23from jedi.inference import analysis
24from jedi.inference.utils import unite
25from jedi.inference.cache import inference_state_method_cache
26from jedi.inference.names import ImportName, SubModuleName
27from jedi.inference.base_value import ValueSet, NO_VALUES
28from jedi.inference.gradual.typeshed import import_module_decorator, \
29 create_stub_module, parse_stub_module
30from jedi.inference.compiled.subprocess.functions import ImplicitNSInfo
31from jedi.plugins import plugin_manager
34class ModuleCache:
35 def __init__(self):
36 self._name_cache = {}
38 def add(self, string_names, value_set):
39 if string_names is not None:
40 self._name_cache[string_names] = value_set
42 def get(self, string_names):
43 return self._name_cache.get(string_names)
46# This memoization is needed, because otherwise we will infinitely loop on
47# certain imports.
48@inference_state_method_cache(default=NO_VALUES)
49def infer_import(context, tree_name):
50 module_context = context.get_root_context()
51 from_import_name, import_path, level, values = \
52 _prepare_infer_import(module_context, tree_name)
53 if values:
55 if from_import_name is not None:
56 values = values.py__getattribute__(
57 from_import_name,
58 name_context=context,
59 analysis_errors=False
60 )
62 if not values:
63 path = import_path + (from_import_name,)
64 importer = Importer(context.inference_state, path, module_context, level)
65 values = importer.follow()
66 debug.dbg('after import: %s', values)
67 return values
70@inference_state_method_cache(default=[])
71def goto_import(context, tree_name):
72 module_context = context.get_root_context()
73 from_import_name, import_path, level, values = \
74 _prepare_infer_import(module_context, tree_name)
75 if not values:
76 return []
78 if from_import_name is not None:
79 names = unite([
80 c.goto(
81 from_import_name,
82 name_context=context,
83 analysis_errors=False
84 ) for c in values
85 ])
86 # Avoid recursion on the same names.
87 if names and not any(n.tree_name is tree_name for n in names):
88 return names
90 path = import_path + (from_import_name,)
91 importer = Importer(context.inference_state, path, module_context, level)
92 values = importer.follow()
93 return set(s.name for s in values)
96def _prepare_infer_import(module_context, tree_name):
97 import_node = tree_name.search_ancestor('import_name', 'import_from')
98 import_path = import_node.get_path_for_name(tree_name)
99 from_import_name = None
100 try:
101 from_names = import_node.get_from_names()
102 except AttributeError:
103 # Is an import_name
104 pass
105 else:
106 if len(from_names) + 1 == len(import_path):
107 # We have to fetch the from_names part first and then check
108 # if from_names exists in the modules.
109 from_import_name = import_path[-1]
110 import_path = from_names
112 importer = Importer(module_context.inference_state, tuple(import_path),
113 module_context, import_node.level)
115 return from_import_name, tuple(import_path), import_node.level, importer.follow()
118def _add_error(value, name, message):
119 if hasattr(name, 'parent') and value is not None:
120 analysis.add(value, 'import-error', name, message)
121 else:
122 debug.warning('ImportError without origin: ' + message)
125def _level_to_base_import_path(project_path, directory, level):
126 """
127 In case the level is outside of the currently known package (something like
128 import .....foo), we can still try our best to help the user for
129 completions.
130 """
131 for i in range(level - 1):
132 old = directory
133 directory = os.path.dirname(directory)
134 if old == directory:
135 return None, None
137 d = directory
138 level_import_paths = []
139 # Now that we are on the level that the user wants to be, calculate the
140 # import path for it.
141 while True:
142 if d == project_path:
143 return level_import_paths, d
144 dir_name = os.path.basename(d)
145 if dir_name:
146 level_import_paths.insert(0, dir_name)
147 d = os.path.dirname(d)
148 else:
149 return None, directory
152class Importer:
153 def __init__(self, inference_state, import_path, module_context, level=0):
154 """
155 An implementation similar to ``__import__``. Use `follow`
156 to actually follow the imports.
158 *level* specifies whether to use absolute or relative imports. 0 (the
159 default) means only perform absolute imports. Positive values for level
160 indicate the number of parent directories to search relative to the
161 directory of the module calling ``__import__()`` (see PEP 328 for the
162 details).
164 :param import_path: List of namespaces (strings or Names).
165 """
166 debug.speed('import %s %s' % (import_path, module_context))
167 self._inference_state = inference_state
168 self.level = level
169 self._module_context = module_context
171 self._fixed_sys_path = None
172 self._infer_possible = True
173 if level:
174 base = module_context.get_value().py__package__()
175 # We need to care for two cases, the first one is if it's a valid
176 # Python import. This import has a properly defined module name
177 # chain like `foo.bar.baz` and an import in baz is made for
178 # `..lala.` It can then resolve to `foo.bar.lala`.
179 # The else here is a heuristic for all other cases, if for example
180 # in `foo` you search for `...bar`, it's obviously out of scope.
181 # However since Jedi tries to just do it's best, we help the user
182 # here, because he might have specified something wrong in his
183 # project.
184 if level <= len(base):
185 # Here we basically rewrite the level to 0.
186 base = tuple(base)
187 if level > 1:
188 base = base[:-level + 1]
189 import_path = base + tuple(import_path)
190 else:
191 path = module_context.py__file__()
192 project_path = self._inference_state.project.path
193 import_path = list(import_path)
194 if path is None:
195 # If no path is defined, our best guess is that the current
196 # file is edited by a user on the current working
197 # directory. We need to add an initial path, because it
198 # will get removed as the name of the current file.
199 directory = project_path
200 else:
201 directory = os.path.dirname(path)
203 base_import_path, base_directory = _level_to_base_import_path(
204 project_path, directory, level,
205 )
206 if base_directory is None:
207 # Everything is lost, the relative import does point
208 # somewhere out of the filesystem.
209 self._infer_possible = False
210 else:
211 self._fixed_sys_path = [base_directory]
213 if base_import_path is None:
214 if import_path:
215 _add_error(
216 module_context, import_path[0],
217 message='Attempted relative import beyond top-level package.'
218 )
219 else:
220 import_path = base_import_path + import_path
221 self.import_path = import_path
223 @property
224 def _str_import_path(self):
225 """Returns the import path as pure strings instead of `Name`."""
226 return tuple(
227 name.value if isinstance(name, tree.Name) else name
228 for name in self.import_path
229 )
231 def _sys_path_with_modifications(self, is_completion):
232 if self._fixed_sys_path is not None:
233 return self._fixed_sys_path
235 return (
236 # For import completions we don't want to see init paths, but for
237 # inference we want to show the user as much as possible.
238 # See GH #1446.
239 self._inference_state.get_sys_path(add_init_paths=not is_completion)
240 + [
241 str(p) for p
242 in sys_path.check_sys_path_modifications(self._module_context)
243 ]
244 )
246 def follow(self):
247 if not self.import_path:
248 if self._fixed_sys_path:
249 # This is a bit of a special case, that maybe should be
250 # revisited. If the project path is wrong or the user uses
251 # relative imports the wrong way, we might end up here, where
252 # the `fixed_sys_path == project.path` in that case we kind of
253 # use the project.path.parent directory as our path. This is
254 # usually not a problem, except if imports in other places are
255 # using the same names. Example:
256 #
257 # foo/ < #1
258 # - setup.py
259 # - foo/ < #2
260 # - __init__.py
261 # - foo.py < #3
262 #
263 # If the top foo is our project folder and somebody uses
264 # `from . import foo` in `setup.py`, it will resolve to foo #2,
265 # which means that the import for foo.foo is cached as
266 # `__init__.py` (#2) and not as `foo.py` (#3). This is usually
267 # not an issue, because this case is probably pretty rare, but
268 # might be an issue for some people.
269 #
270 # However for most normal cases where we work with different
271 # file names, this code path hits where we basically change the
272 # project path to an ancestor of project path.
273 from jedi.inference.value.namespace import ImplicitNamespaceValue
274 import_path = (os.path.basename(self._fixed_sys_path[0]),)
275 ns = ImplicitNamespaceValue(
276 self._inference_state,
277 string_names=import_path,
278 paths=self._fixed_sys_path,
279 )
280 return ValueSet({ns})
281 return NO_VALUES
282 if not self._infer_possible:
283 return NO_VALUES
285 # Check caches first
286 from_cache = self._inference_state.stub_module_cache.get(self._str_import_path)
287 if from_cache is not None:
288 return ValueSet({from_cache})
289 from_cache = self._inference_state.module_cache.get(self._str_import_path)
290 if from_cache is not None:
291 return from_cache
293 sys_path = self._sys_path_with_modifications(is_completion=False)
295 return import_module_by_names(
296 self._inference_state, self.import_path, sys_path, self._module_context
297 )
299 def _get_module_names(self, search_path=None, in_module=None):
300 """
301 Get the names of all modules in the search_path. This means file names
302 and not names defined in the files.
303 """
304 if search_path is None:
305 sys_path = self._sys_path_with_modifications(is_completion=True)
306 else:
307 sys_path = search_path
308 return list(iter_module_names(
309 self._inference_state, self._module_context, sys_path,
310 module_cls=ImportName if in_module is None else SubModuleName,
311 add_builtin_modules=search_path is None and in_module is None,
312 ))
314 def completion_names(self, inference_state, only_modules=False):
315 """
316 :param only_modules: Indicates wheter it's possible to import a
317 definition that is not defined in a module.
318 """
319 if not self._infer_possible:
320 return []
322 names = []
323 if self.import_path:
324 # flask
325 if self._str_import_path == ('flask', 'ext'):
326 # List Flask extensions like ``flask_foo``
327 for mod in self._get_module_names():
328 modname = mod.string_name
329 if modname.startswith('flask_'):
330 extname = modname[len('flask_'):]
331 names.append(ImportName(self._module_context, extname))
332 # Now the old style: ``flaskext.foo``
333 for dir in self._sys_path_with_modifications(is_completion=True):
334 flaskext = os.path.join(dir, 'flaskext')
335 if os.path.isdir(flaskext):
336 names += self._get_module_names([flaskext])
338 values = self.follow()
339 for value in values:
340 # Non-modules are not completable.
341 if value.api_type not in ('module', 'namespace'): # not a module
342 continue
343 if not value.is_compiled():
344 # sub_modules_dict is not implemented for compiled modules.
345 names += value.sub_modules_dict().values()
347 if not only_modules:
348 from jedi.inference.gradual.conversion import convert_values
350 both_values = values | convert_values(values)
351 for c in both_values:
352 for filter in c.get_filters():
353 names += filter.values()
354 else:
355 if self.level:
356 # We only get here if the level cannot be properly calculated.
357 names += self._get_module_names(self._fixed_sys_path)
358 else:
359 # This is just the list of global imports.
360 names += self._get_module_names()
361 return names
364def import_module_by_names(inference_state, import_names, sys_path=None,
365 module_context=None, prefer_stubs=True):
366 if sys_path is None:
367 sys_path = inference_state.get_sys_path()
369 str_import_names = tuple(
370 i.value if isinstance(i, tree.Name) else i
371 for i in import_names
372 )
373 base = [None]
374 for i, name in enumerate(import_names):
375 base = value_set = ValueSet.from_sets([
376 import_module(
377 inference_state,
378 str_import_names[:i+1],
379 parent_module_value,
380 sys_path,
381 prefer_stubs=prefer_stubs, # type: ignore[call-arg]
382 ) for parent_module_value in base
383 ])
384 if not value_set:
385 message = 'No module named ' + '.'.join(str_import_names)
386 if module_context is not None:
387 _add_error(module_context, name, message)
388 else:
389 debug.warning(message)
390 return NO_VALUES
391 return value_set
394@plugin_manager.decorate()
395@import_module_decorator
396def import_module(inference_state, import_names, parent_module_value, sys_path):
397 """
398 This method is very similar to importlib's `_gcd_import`.
399 """
400 if import_names[0] in settings.auto_import_modules:
401 module = _load_builtin_module(inference_state, import_names, sys_path)
402 if module is None:
403 return NO_VALUES
404 return ValueSet([module])
406 module_name = '.'.join(import_names)
407 if parent_module_value is None:
408 # Override the sys.path. It works only good that way.
409 # Injecting the path directly into `find_module` did not work.
410 file_io_or_ns, is_pkg = inference_state.compiled_subprocess.get_module_info(
411 string=import_names[-1],
412 full_name=module_name,
413 sys_path=sys_path,
414 is_global_search=True,
415 )
416 if is_pkg is None:
417 return NO_VALUES
418 else:
419 try:
420 py__path__ = parent_module_value.py__path__
421 except AttributeError:
422 # Due to meta_path, the value can be pretty much any compiled value
423 # and we're not guaranteed that the attribute is available, see
424 # also https://github.com/davidhalter/jedi/issues/2118
425 return NO_VALUES
426 else:
427 paths = py__path__()
428 if paths is None:
429 # The module might not be a package.
430 return NO_VALUES
432 file_io_or_ns, is_pkg = inference_state.compiled_subprocess.get_module_info(
433 string=import_names[-1],
434 path=paths,
435 full_name=module_name,
436 is_global_search=False,
437 )
438 if is_pkg is None:
439 return NO_VALUES
441 if isinstance(file_io_or_ns, ImplicitNSInfo):
442 from jedi.inference.value.namespace import ImplicitNamespaceValue
443 module = ImplicitNamespaceValue(
444 inference_state,
445 string_names=tuple(file_io_or_ns.name.split('.')),
446 paths=file_io_or_ns.paths,
447 )
448 elif file_io_or_ns is None:
449 module = _load_builtin_module(inference_state, import_names, sys_path)
450 if module is None:
451 return NO_VALUES
452 else:
453 module = _load_python_module(
454 inference_state, file_io_or_ns,
455 import_names=import_names,
456 is_package=is_pkg,
457 )
459 if parent_module_value is None:
460 debug.dbg('global search_module %s: %s', import_names[-1], module)
461 else:
462 debug.dbg('search_module %s in paths %s: %s', module_name, paths, module)
463 return ValueSet([module])
466def _load_python_module(inference_state, file_io,
467 import_names=None, is_package=False):
468 module_node = inference_state.parse(
469 file_io=file_io,
470 cache=True,
471 diff_cache=settings.fast_parser,
472 cache_path=settings.cache_directory,
473 )
475 from jedi.inference.value import ModuleValue
476 return ModuleValue(
477 inference_state, module_node,
478 file_io=file_io,
479 string_names=import_names,
480 code_lines=get_cached_code_lines(inference_state.grammar, file_io.path),
481 is_package=is_package,
482 )
485def _load_builtin_module(inference_state, import_names, sys_path):
486 project = inference_state.project
487 if sys_path is None:
488 sys_path = inference_state.get_sys_path()
489 if not project._load_unsafe_extensions:
490 safe_paths = set(project._get_base_sys_path(inference_state))
491 sys_path = [p for p in sys_path if p in safe_paths]
493 dotted_name = '.'.join(import_names)
494 assert dotted_name is not None
495 module = compiled.load_module(inference_state, dotted_name=dotted_name, sys_path=sys_path)
496 if module is None:
497 # The file might raise an ImportError e.g. and therefore not be
498 # importable.
499 return None
500 return module
503def load_module_from_path(inference_state, file_io, import_names=None, is_package=None):
504 """
505 This should pretty much only be used for get_modules_containing_name. It's
506 here to ensure that a random path is still properly loaded into the Jedi
507 module structure.
508 """
509 path = Path(file_io.path)
510 if import_names is None:
511 e_sys_path = inference_state.get_sys_path()
512 import_names, is_package = sys_path.transform_path_to_dotted(e_sys_path, path)
513 else:
514 assert isinstance(is_package, bool)
516 is_stub = path.suffix == '.pyi'
517 if is_stub:
518 folder_io = file_io.get_parent_folder()
519 if folder_io.path.endswith('-stubs'):
520 folder_io = FolderIO(folder_io.path[:-6])
521 if path.name == '__init__.pyi':
522 python_file_io = folder_io.get_file_io('__init__.py')
523 else:
524 python_file_io = folder_io.get_file_io(import_names[-1] + '.py')
526 try:
527 v = load_module_from_path(
528 inference_state, python_file_io,
529 import_names, is_package=is_package
530 )
531 values = ValueSet([v])
532 except FileNotFoundError:
533 values = NO_VALUES
535 return create_stub_module(
536 inference_state, inference_state.latest_grammar, values,
537 parse_stub_module(inference_state, file_io), file_io, import_names
538 )
539 else:
540 module = _load_python_module(
541 inference_state, file_io,
542 import_names=import_names,
543 is_package=is_package,
544 )
545 inference_state.module_cache.add(import_names, ValueSet([module]))
546 return module
549def load_namespace_from_path(inference_state, folder_io):
550 import_names, is_package = sys_path.transform_path_to_dotted(
551 inference_state.get_sys_path(),
552 Path(folder_io.path)
553 )
554 from jedi.inference.value.namespace import ImplicitNamespaceValue
555 return ImplicitNamespaceValue(inference_state, import_names, [folder_io.path])
558def follow_error_node_imports_if_possible(context, name):
559 error_node = name.search_ancestor('error_node')
560 if error_node is not None:
561 # Get the first command start of a started simple_stmt. The error
562 # node is sometimes a small_stmt and sometimes a simple_stmt. Check
563 # for ; leaves that start a new statements.
564 start_index = 0
565 for index, n in enumerate(error_node.children):
566 if n.start_pos > name.start_pos:
567 break
568 if n == ';':
569 start_index = index + 1
570 nodes = error_node.children[start_index:]
571 first_name = nodes[0].get_first_leaf().value
573 # Make it possible to infer stuff like `import foo.` or
574 # `from foo.bar`.
575 if first_name in ('from', 'import'):
576 is_import_from = first_name == 'from'
577 level, names = helpers.parse_dotted_names(
578 nodes,
579 is_import_from=is_import_from,
580 until_node=name,
581 )
582 return Importer(
583 context.inference_state, names, context.get_root_context(), level).follow()
584 return None
587def iter_module_names(inference_state, module_context, search_path,
588 module_cls=ImportName, add_builtin_modules=True):
589 """
590 Get the names of all modules in the search_path. This means file names
591 and not names defined in the files.
592 """
593 # add builtin module names
594 if add_builtin_modules:
595 for name in inference_state.compiled_subprocess.get_builtin_module_names():
596 yield module_cls(module_context, name)
598 for name in inference_state.compiled_subprocess.iter_module_names(search_path):
599 yield module_cls(module_context, name)