Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""
2:mod:`jedi.inference.imports` is here to resolve import statements and return
3the modules/classes/functions/whatever, which they stand for. However there's
4not any actual importing done. This module is about finding modules in the
5filesystem. This can be quite tricky sometimes, because Python imports are not
6always that simple.
8This module also supports import autocompletion, which means to complete
9statements like ``from datetim`` (cursor at the end would return ``datetime``).
10"""
11import os
12from pathlib import Path
14from parso.python import tree
16from jedi import debug
17from jedi import settings
18from jedi.file_io import FolderIO
19from jedi.parser_utils import get_cached_code_lines
20from jedi.inference import sys_path
21from jedi.inference import helpers
22from jedi.inference import compiled
23from jedi.inference import analysis
24from jedi.inference.utils import unite
25from jedi.inference.cache import inference_state_method_cache
26from jedi.inference.names import ImportName, SubModuleName
27from jedi.inference.base_value import ValueSet, NO_VALUES
28from jedi.inference.gradual.typeshed import import_module_decorator, \
29 create_stub_module, parse_stub_module
30from jedi.inference.compiled.subprocess.functions import ImplicitNSInfo
31from jedi.plugins import plugin_manager
34class ModuleCache:
35 def __init__(self):
36 self._name_cache = {}
38 def add(self, string_names, value_set):
39 if string_names is not None:
40 self._name_cache[string_names] = value_set
42 def get(self, string_names):
43 return self._name_cache.get(string_names)
46# This memoization is needed, because otherwise we will infinitely loop on
47# certain imports.
48@inference_state_method_cache(default=NO_VALUES)
49def infer_import(context, tree_name):
50 module_context = context.get_root_context()
51 from_import_name, import_path, level, values = \
52 _prepare_infer_import(module_context, tree_name)
53 if values:
55 if from_import_name is not None:
56 values = values.py__getattribute__(
57 from_import_name,
58 name_context=context,
59 analysis_errors=False
60 )
62 if not values:
63 path = import_path + (from_import_name,)
64 importer = Importer(context.inference_state, path, module_context, level)
65 values = importer.follow()
66 debug.dbg('after import: %s', values)
67 return values
70@inference_state_method_cache(default=[])
71def goto_import(context, tree_name):
72 module_context = context.get_root_context()
73 from_import_name, import_path, level, values = \
74 _prepare_infer_import(module_context, tree_name)
75 if not values:
76 return []
78 if from_import_name is not None:
79 names = unite([
80 c.goto(
81 from_import_name,
82 name_context=context,
83 analysis_errors=False
84 ) for c in values
85 ])
86 # Avoid recursion on the same names.
87 if names and not any(n.tree_name is tree_name for n in names):
88 return names
90 path = import_path + (from_import_name,)
91 importer = Importer(context.inference_state, path, module_context, level)
92 values = importer.follow()
93 return set(s.name for s in values)
96def _prepare_infer_import(module_context, tree_name):
97 import_node = tree_name.search_ancestor('import_name', 'import_from')
98 import_path = import_node.get_path_for_name(tree_name)
99 from_import_name = None
100 try:
101 from_names = import_node.get_from_names()
102 except AttributeError:
103 # Is an import_name
104 pass
105 else:
106 if len(from_names) + 1 == len(import_path):
107 # We have to fetch the from_names part first and then check
108 # if from_names exists in the modules.
109 from_import_name = import_path[-1]
110 import_path = from_names
112 importer = Importer(module_context.inference_state, tuple(import_path),
113 module_context, import_node.level)
115 return from_import_name, tuple(import_path), import_node.level, importer.follow()
118def _add_error(value, name, message):
119 if hasattr(name, 'parent') and value is not None:
120 analysis.add(value, 'import-error', name, message)
121 else:
122 debug.warning('ImportError without origin: ' + message)
125def _level_to_base_import_path(project_path, directory, level):
126 """
127 In case the level is outside of the currently known package (something like
128 import .....foo), we can still try our best to help the user for
129 completions.
130 """
131 for i in range(level - 1):
132 old = directory
133 directory = os.path.dirname(directory)
134 if old == directory:
135 return None, None
137 d = directory
138 level_import_paths = []
139 # Now that we are on the level that the user wants to be, calculate the
140 # import path for it.
141 while True:
142 if d == project_path:
143 return level_import_paths, d
144 dir_name = os.path.basename(d)
145 if dir_name:
146 level_import_paths.insert(0, dir_name)
147 d = os.path.dirname(d)
148 else:
149 return None, directory
152class Importer:
153 def __init__(self, inference_state, import_path, module_context, level=0):
154 """
155 An implementation similar to ``__import__``. Use `follow`
156 to actually follow the imports.
158 *level* specifies whether to use absolute or relative imports. 0 (the
159 default) means only perform absolute imports. Positive values for level
160 indicate the number of parent directories to search relative to the
161 directory of the module calling ``__import__()`` (see PEP 328 for the
162 details).
164 :param import_path: List of namespaces (strings or Names).
165 """
166 debug.speed('import %s %s' % (import_path, module_context))
167 self._inference_state = inference_state
168 self.level = level
169 self._module_context = module_context
171 self._fixed_sys_path = None
172 self._infer_possible = True
173 if level:
174 base = module_context.get_value().py__package__()
175 # We need to care for two cases, the first one is if it's a valid
176 # Python import. This import has a properly defined module name
177 # chain like `foo.bar.baz` and an import in baz is made for
178 # `..lala.` It can then resolve to `foo.bar.lala`.
179 # The else here is a heuristic for all other cases, if for example
180 # in `foo` you search for `...bar`, it's obviously out of scope.
181 # However since Jedi tries to just do it's best, we help the user
182 # here, because he might have specified something wrong in his
183 # project.
184 if level <= len(base):
185 # Here we basically rewrite the level to 0.
186 base = tuple(base)
187 if level > 1:
188 base = base[:-level + 1]
189 import_path = base + tuple(import_path)
190 else:
191 path = module_context.py__file__()
192 project_path = self._inference_state.project.path
193 import_path = list(import_path)
194 if path is None:
195 # If no path is defined, our best guess is that the current
196 # file is edited by a user on the current working
197 # directory. We need to add an initial path, because it
198 # will get removed as the name of the current file.
199 directory = project_path
200 else:
201 directory = os.path.dirname(path)
203 base_import_path, base_directory = _level_to_base_import_path(
204 project_path, directory, level,
205 )
206 if base_directory is None:
207 # Everything is lost, the relative import does point
208 # somewhere out of the filesystem.
209 self._infer_possible = False
210 else:
211 self._fixed_sys_path = [base_directory]
213 if base_import_path is None:
214 if import_path:
215 _add_error(
216 module_context, import_path[0],
217 message='Attempted relative import beyond top-level package.'
218 )
219 else:
220 import_path = base_import_path + import_path
221 self.import_path = import_path
223 @property
224 def _str_import_path(self):
225 """Returns the import path as pure strings instead of `Name`."""
226 return tuple(
227 name.value if isinstance(name, tree.Name) else name
228 for name in self.import_path
229 )
231 def _sys_path_with_modifications(self, is_completion):
232 if self._fixed_sys_path is not None:
233 return self._fixed_sys_path
235 return (
236 # For import completions we don't want to see init paths, but for
237 # inference we want to show the user as much as possible.
238 # See GH #1446.
239 self._inference_state.get_sys_path(add_init_paths=not is_completion)
240 + [
241 str(p) for p
242 in sys_path.check_sys_path_modifications(self._module_context)
243 ]
244 )
246 def follow(self):
247 if not self.import_path:
248 if self._fixed_sys_path:
249 # This is a bit of a special case, that maybe should be
250 # revisited. If the project path is wrong or the user uses
251 # relative imports the wrong way, we might end up here, where
252 # the `fixed_sys_path == project.path` in that case we kind of
253 # use the project.path.parent directory as our path. This is
254 # usually not a problem, except if imports in other places are
255 # using the same names. Example:
256 #
257 # foo/ < #1
258 # - setup.py
259 # - foo/ < #2
260 # - __init__.py
261 # - foo.py < #3
262 #
263 # If the top foo is our project folder and somebody uses
264 # `from . import foo` in `setup.py`, it will resolve to foo #2,
265 # which means that the import for foo.foo is cached as
266 # `__init__.py` (#2) and not as `foo.py` (#3). This is usually
267 # not an issue, because this case is probably pretty rare, but
268 # might be an issue for some people.
269 #
270 # However for most normal cases where we work with different
271 # file names, this code path hits where we basically change the
272 # project path to an ancestor of project path.
273 from jedi.inference.value.namespace import ImplicitNamespaceValue
274 import_path = (os.path.basename(self._fixed_sys_path[0]),)
275 ns = ImplicitNamespaceValue(
276 self._inference_state,
277 string_names=import_path,
278 paths=self._fixed_sys_path,
279 )
280 return ValueSet({ns})
281 return NO_VALUES
282 if not self._infer_possible:
283 return NO_VALUES
285 # Check caches first
286 from_cache = self._inference_state.stub_module_cache.get(self._str_import_path)
287 if from_cache is not None:
288 return ValueSet({from_cache})
289 from_cache = self._inference_state.module_cache.get(self._str_import_path)
290 if from_cache is not None:
291 return from_cache
293 sys_path = self._sys_path_with_modifications(is_completion=False)
295 return import_module_by_names(
296 self._inference_state, self.import_path, sys_path, self._module_context
297 )
299 def _get_module_names(self, search_path=None, in_module=None):
300 """
301 Get the names of all modules in the search_path. This means file names
302 and not names defined in the files.
303 """
304 if search_path is None:
305 sys_path = self._sys_path_with_modifications(is_completion=True)
306 else:
307 sys_path = search_path
308 return list(iter_module_names(
309 self._inference_state, self._module_context, sys_path,
310 module_cls=ImportName if in_module is None else SubModuleName,
311 add_builtin_modules=search_path is None and in_module is None,
312 ))
314 def completion_names(self, inference_state, only_modules=False):
315 """
316 :param only_modules: Indicates wheter it's possible to import a
317 definition that is not defined in a module.
318 """
319 if not self._infer_possible:
320 return []
322 names = []
323 if self.import_path:
324 # flask
325 if self._str_import_path == ('flask', 'ext'):
326 # List Flask extensions like ``flask_foo``
327 for mod in self._get_module_names():
328 modname = mod.string_name
329 if modname.startswith('flask_'):
330 extname = modname[len('flask_'):]
331 names.append(ImportName(self._module_context, extname))
332 # Now the old style: ``flaskext.foo``
333 for dir in self._sys_path_with_modifications(is_completion=True):
334 flaskext = os.path.join(dir, 'flaskext')
335 if os.path.isdir(flaskext):
336 names += self._get_module_names([flaskext])
338 values = self.follow()
339 for value in values:
340 # Non-modules are not completable.
341 if value.api_type not in ('module', 'namespace'): # not a module
342 continue
343 if not value.is_compiled():
344 # sub_modules_dict is not implemented for compiled modules.
345 names += value.sub_modules_dict().values()
347 if not only_modules:
348 from jedi.inference.gradual.conversion import convert_values
350 both_values = values | convert_values(values)
351 for c in both_values:
352 for filter in c.get_filters():
353 names += filter.values()
354 else:
355 if self.level:
356 # We only get here if the level cannot be properly calculated.
357 names += self._get_module_names(self._fixed_sys_path)
358 else:
359 # This is just the list of global imports.
360 names += self._get_module_names()
361 return names
364def import_module_by_names(inference_state, import_names, sys_path=None,
365 module_context=None, prefer_stubs=True):
366 if sys_path is None:
367 sys_path = inference_state.get_sys_path()
369 str_import_names = tuple(
370 i.value if isinstance(i, tree.Name) else i
371 for i in import_names
372 )
373 value_set = [None]
374 for i, name in enumerate(import_names):
375 value_set = ValueSet.from_sets([
376 import_module(
377 inference_state,
378 str_import_names[:i+1],
379 parent_module_value,
380 sys_path,
381 prefer_stubs=prefer_stubs,
382 ) for parent_module_value in value_set
383 ])
384 if not value_set:
385 message = 'No module named ' + '.'.join(str_import_names)
386 if module_context is not None:
387 _add_error(module_context, name, message)
388 else:
389 debug.warning(message)
390 return NO_VALUES
391 return value_set
394@plugin_manager.decorate()
395@import_module_decorator
396def import_module(inference_state, import_names, parent_module_value, sys_path):
397 """
398 This method is very similar to importlib's `_gcd_import`.
399 """
400 if import_names[0] in settings.auto_import_modules:
401 module = _load_builtin_module(inference_state, import_names, sys_path)
402 if module is None:
403 return NO_VALUES
404 return ValueSet([module])
406 module_name = '.'.join(import_names)
407 if parent_module_value is None:
408 # Override the sys.path. It works only good that way.
409 # Injecting the path directly into `find_module` did not work.
410 file_io_or_ns, is_pkg = inference_state.compiled_subprocess.get_module_info(
411 string=import_names[-1],
412 full_name=module_name,
413 sys_path=sys_path,
414 is_global_search=True,
415 )
416 if is_pkg is None:
417 return NO_VALUES
418 else:
419 paths = parent_module_value.py__path__()
420 if paths is None:
421 # The module might not be a package.
422 return NO_VALUES
424 file_io_or_ns, is_pkg = inference_state.compiled_subprocess.get_module_info(
425 string=import_names[-1],
426 path=paths,
427 full_name=module_name,
428 is_global_search=False,
429 )
430 if is_pkg is None:
431 return NO_VALUES
433 if isinstance(file_io_or_ns, ImplicitNSInfo):
434 from jedi.inference.value.namespace import ImplicitNamespaceValue
435 module = ImplicitNamespaceValue(
436 inference_state,
437 string_names=tuple(file_io_or_ns.name.split('.')),
438 paths=file_io_or_ns.paths,
439 )
440 elif file_io_or_ns is None:
441 module = _load_builtin_module(inference_state, import_names, sys_path)
442 if module is None:
443 return NO_VALUES
444 else:
445 module = _load_python_module(
446 inference_state, file_io_or_ns,
447 import_names=import_names,
448 is_package=is_pkg,
449 )
451 if parent_module_value is None:
452 debug.dbg('global search_module %s: %s', import_names[-1], module)
453 else:
454 debug.dbg('search_module %s in paths %s: %s', module_name, paths, module)
455 return ValueSet([module])
458def _load_python_module(inference_state, file_io,
459 import_names=None, is_package=False):
460 module_node = inference_state.parse(
461 file_io=file_io,
462 cache=True,
463 diff_cache=settings.fast_parser,
464 cache_path=settings.cache_directory,
465 )
467 from jedi.inference.value import ModuleValue
468 return ModuleValue(
469 inference_state, module_node,
470 file_io=file_io,
471 string_names=import_names,
472 code_lines=get_cached_code_lines(inference_state.grammar, file_io.path),
473 is_package=is_package,
474 )
477def _load_builtin_module(inference_state, import_names=None, sys_path=None):
478 project = inference_state.project
479 if sys_path is None:
480 sys_path = inference_state.get_sys_path()
481 if not project._load_unsafe_extensions:
482 safe_paths = set(project._get_base_sys_path(inference_state))
483 sys_path = [p for p in sys_path if p in safe_paths]
485 dotted_name = '.'.join(import_names)
486 assert dotted_name is not None
487 module = compiled.load_module(inference_state, dotted_name=dotted_name, sys_path=sys_path)
488 if module is None:
489 # The file might raise an ImportError e.g. and therefore not be
490 # importable.
491 return None
492 return module
495def load_module_from_path(inference_state, file_io, import_names=None, is_package=None):
496 """
497 This should pretty much only be used for get_modules_containing_name. It's
498 here to ensure that a random path is still properly loaded into the Jedi
499 module structure.
500 """
501 path = Path(file_io.path)
502 if import_names is None:
503 e_sys_path = inference_state.get_sys_path()
504 import_names, is_package = sys_path.transform_path_to_dotted(e_sys_path, path)
505 else:
506 assert isinstance(is_package, bool)
508 is_stub = path.suffix == '.pyi'
509 if is_stub:
510 folder_io = file_io.get_parent_folder()
511 if folder_io.path.endswith('-stubs'):
512 folder_io = FolderIO(folder_io.path[:-6])
513 if path.name == '__init__.pyi':
514 python_file_io = folder_io.get_file_io('__init__.py')
515 else:
516 python_file_io = folder_io.get_file_io(import_names[-1] + '.py')
518 try:
519 v = load_module_from_path(
520 inference_state, python_file_io,
521 import_names, is_package=is_package
522 )
523 values = ValueSet([v])
524 except FileNotFoundError:
525 values = NO_VALUES
527 return create_stub_module(
528 inference_state, inference_state.latest_grammar, values,
529 parse_stub_module(inference_state, file_io), file_io, import_names
530 )
531 else:
532 module = _load_python_module(
533 inference_state, file_io,
534 import_names=import_names,
535 is_package=is_package,
536 )
537 inference_state.module_cache.add(import_names, ValueSet([module]))
538 return module
541def load_namespace_from_path(inference_state, folder_io):
542 import_names, is_package = sys_path.transform_path_to_dotted(
543 inference_state.get_sys_path(),
544 Path(folder_io.path)
545 )
546 from jedi.inference.value.namespace import ImplicitNamespaceValue
547 return ImplicitNamespaceValue(inference_state, import_names, [folder_io.path])
550def follow_error_node_imports_if_possible(context, name):
551 error_node = name.search_ancestor('error_node')
552 if error_node is not None:
553 # Get the first command start of a started simple_stmt. The error
554 # node is sometimes a small_stmt and sometimes a simple_stmt. Check
555 # for ; leaves that start a new statements.
556 start_index = 0
557 for index, n in enumerate(error_node.children):
558 if n.start_pos > name.start_pos:
559 break
560 if n == ';':
561 start_index = index + 1
562 nodes = error_node.children[start_index:]
563 first_name = nodes[0].get_first_leaf().value
565 # Make it possible to infer stuff like `import foo.` or
566 # `from foo.bar`.
567 if first_name in ('from', 'import'):
568 is_import_from = first_name == 'from'
569 level, names = helpers.parse_dotted_names(
570 nodes,
571 is_import_from=is_import_from,
572 until_node=name,
573 )
574 return Importer(
575 context.inference_state, names, context.get_root_context(), level).follow()
576 return None
579def iter_module_names(inference_state, module_context, search_path,
580 module_cls=ImportName, add_builtin_modules=True):
581 """
582 Get the names of all modules in the search_path. This means file names
583 and not names defined in the files.
584 """
585 # add builtin module names
586 if add_builtin_modules:
587 for name in inference_state.compiled_subprocess.get_builtin_module_names():
588 yield module_cls(module_context, name)
590 for name in inference_state.compiled_subprocess.iter_module_names(search_path):
591 yield module_cls(module_context, name)