Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/pkgutil.py: 17%
362 statements
« prev ^ index » next coverage.py v7.0.1, created at 2022-12-25 06:11 +0000
1"""Utilities to support packages."""
3from collections import namedtuple
4from functools import singledispatch as simplegeneric
5import importlib
6import importlib.util
7import importlib.machinery
8import os
9import os.path
10import sys
11from types import ModuleType
12import warnings
14__all__ = [
15 'get_importer', 'iter_importers', 'get_loader', 'find_loader',
16 'walk_packages', 'iter_modules', 'get_data',
17 'ImpImporter', 'ImpLoader', 'read_code', 'extend_path',
18 'ModuleInfo',
19]
22ModuleInfo = namedtuple('ModuleInfo', 'module_finder name ispkg')
23ModuleInfo.__doc__ = 'A namedtuple with minimal info about a module.'
26def _get_spec(finder, name):
27 """Return the finder-specific module spec."""
28 # Works with legacy finders.
29 try:
30 find_spec = finder.find_spec
31 except AttributeError:
32 loader = finder.find_module(name)
33 if loader is None:
34 return None
35 return importlib.util.spec_from_loader(name, loader)
36 else:
37 return find_spec(name)
40def read_code(stream):
41 # This helper is needed in order for the PEP 302 emulation to
42 # correctly handle compiled files
43 import marshal
45 magic = stream.read(4)
46 if magic != importlib.util.MAGIC_NUMBER:
47 return None
49 stream.read(12) # Skip rest of the header
50 return marshal.load(stream)
53def walk_packages(path=None, prefix='', onerror=None):
54 """Yields ModuleInfo for all modules recursively
55 on path, or, if path is None, all accessible modules.
57 'path' should be either None or a list of paths to look for
58 modules in.
60 'prefix' is a string to output on the front of every module name
61 on output.
63 Note that this function must import all *packages* (NOT all
64 modules!) on the given path, in order to access the __path__
65 attribute to find submodules.
67 'onerror' is a function which gets called with one argument (the
68 name of the package which was being imported) if any exception
69 occurs while trying to import a package. If no onerror function is
70 supplied, ImportErrors are caught and ignored, while all other
71 exceptions are propagated, terminating the search.
73 Examples:
75 # list all modules python can access
76 walk_packages()
78 # list all submodules of ctypes
79 walk_packages(ctypes.__path__, ctypes.__name__+'.')
80 """
82 def seen(p, m={}):
83 if p in m:
84 return True
85 m[p] = True
87 for info in iter_modules(path, prefix):
88 yield info
90 if info.ispkg:
91 try:
92 __import__(info.name)
93 except ImportError:
94 if onerror is not None:
95 onerror(info.name)
96 except Exception:
97 if onerror is not None:
98 onerror(info.name)
99 else:
100 raise
101 else:
102 path = getattr(sys.modules[info.name], '__path__', None) or []
104 # don't traverse path items we've seen before
105 path = [p for p in path if not seen(p)]
107 yield from walk_packages(path, info.name+'.', onerror)
110def iter_modules(path=None, prefix=''):
111 """Yields ModuleInfo for all submodules on path,
112 or, if path is None, all top-level modules on sys.path.
114 'path' should be either None or a list of paths to look for
115 modules in.
117 'prefix' is a string to output on the front of every module name
118 on output.
119 """
120 if path is None:
121 importers = iter_importers()
122 elif isinstance(path, str):
123 raise ValueError("path must be None or list of paths to look for "
124 "modules in")
125 else:
126 importers = map(get_importer, path)
128 yielded = {}
129 for i in importers:
130 for name, ispkg in iter_importer_modules(i, prefix):
131 if name not in yielded:
132 yielded[name] = 1
133 yield ModuleInfo(i, name, ispkg)
136@simplegeneric
137def iter_importer_modules(importer, prefix=''):
138 if not hasattr(importer, 'iter_modules'):
139 return []
140 return importer.iter_modules(prefix)
143# Implement a file walker for the normal importlib path hook
144def _iter_file_finder_modules(importer, prefix=''):
145 if importer.path is None or not os.path.isdir(importer.path):
146 return
148 yielded = {}
149 import inspect
150 try:
151 filenames = os.listdir(importer.path)
152 except OSError:
153 # ignore unreadable directories like import does
154 filenames = []
155 filenames.sort() # handle packages before same-named modules
157 for fn in filenames:
158 modname = inspect.getmodulename(fn)
159 if modname=='__init__' or modname in yielded:
160 continue
162 path = os.path.join(importer.path, fn)
163 ispkg = False
165 if not modname and os.path.isdir(path) and '.' not in fn:
166 modname = fn
167 try:
168 dircontents = os.listdir(path)
169 except OSError:
170 # ignore unreadable directories like import does
171 dircontents = []
172 for fn in dircontents:
173 subname = inspect.getmodulename(fn)
174 if subname=='__init__':
175 ispkg = True
176 break
177 else:
178 continue # not a package
180 if modname and '.' not in modname:
181 yielded[modname] = 1
182 yield prefix + modname, ispkg
184iter_importer_modules.register(
185 importlib.machinery.FileFinder, _iter_file_finder_modules)
188def _import_imp():
189 global imp
190 with warnings.catch_warnings():
191 warnings.simplefilter('ignore', DeprecationWarning)
192 imp = importlib.import_module('imp')
194class ImpImporter:
195 """PEP 302 Finder that wraps Python's "classic" import algorithm
197 ImpImporter(dirname) produces a PEP 302 finder that searches that
198 directory. ImpImporter(None) produces a PEP 302 finder that searches
199 the current sys.path, plus any modules that are frozen or built-in.
201 Note that ImpImporter does not currently support being used by placement
202 on sys.meta_path.
203 """
205 def __init__(self, path=None):
206 global imp
207 warnings.warn("This emulation is deprecated, use 'importlib' instead",
208 DeprecationWarning)
209 _import_imp()
210 self.path = path
212 def find_module(self, fullname, path=None):
213 # Note: we ignore 'path' argument since it is only used via meta_path
214 subname = fullname.split(".")[-1]
215 if subname != fullname and self.path is None:
216 return None
217 if self.path is None:
218 path = None
219 else:
220 path = [os.path.realpath(self.path)]
221 try:
222 file, filename, etc = imp.find_module(subname, path)
223 except ImportError:
224 return None
225 return ImpLoader(fullname, file, filename, etc)
227 def iter_modules(self, prefix=''):
228 if self.path is None or not os.path.isdir(self.path):
229 return
231 yielded = {}
232 import inspect
233 try:
234 filenames = os.listdir(self.path)
235 except OSError:
236 # ignore unreadable directories like import does
237 filenames = []
238 filenames.sort() # handle packages before same-named modules
240 for fn in filenames:
241 modname = inspect.getmodulename(fn)
242 if modname=='__init__' or modname in yielded:
243 continue
245 path = os.path.join(self.path, fn)
246 ispkg = False
248 if not modname and os.path.isdir(path) and '.' not in fn:
249 modname = fn
250 try:
251 dircontents = os.listdir(path)
252 except OSError:
253 # ignore unreadable directories like import does
254 dircontents = []
255 for fn in dircontents:
256 subname = inspect.getmodulename(fn)
257 if subname=='__init__':
258 ispkg = True
259 break
260 else:
261 continue # not a package
263 if modname and '.' not in modname:
264 yielded[modname] = 1
265 yield prefix + modname, ispkg
268class ImpLoader:
269 """PEP 302 Loader that wraps Python's "classic" import algorithm
270 """
271 code = source = None
273 def __init__(self, fullname, file, filename, etc):
274 warnings.warn("This emulation is deprecated, use 'importlib' instead",
275 DeprecationWarning)
276 _import_imp()
277 self.file = file
278 self.filename = filename
279 self.fullname = fullname
280 self.etc = etc
282 def load_module(self, fullname):
283 self._reopen()
284 try:
285 mod = imp.load_module(fullname, self.file, self.filename, self.etc)
286 finally:
287 if self.file:
288 self.file.close()
289 # Note: we don't set __loader__ because we want the module to look
290 # normal; i.e. this is just a wrapper for standard import machinery
291 return mod
293 def get_data(self, pathname):
294 with open(pathname, "rb") as file:
295 return file.read()
297 def _reopen(self):
298 if self.file and self.file.closed:
299 mod_type = self.etc[2]
300 if mod_type==imp.PY_SOURCE:
301 self.file = open(self.filename, 'r')
302 elif mod_type in (imp.PY_COMPILED, imp.C_EXTENSION):
303 self.file = open(self.filename, 'rb')
305 def _fix_name(self, fullname):
306 if fullname is None:
307 fullname = self.fullname
308 elif fullname != self.fullname:
309 raise ImportError("Loader for module %s cannot handle "
310 "module %s" % (self.fullname, fullname))
311 return fullname
313 def is_package(self, fullname):
314 fullname = self._fix_name(fullname)
315 return self.etc[2]==imp.PKG_DIRECTORY
317 def get_code(self, fullname=None):
318 fullname = self._fix_name(fullname)
319 if self.code is None:
320 mod_type = self.etc[2]
321 if mod_type==imp.PY_SOURCE:
322 source = self.get_source(fullname)
323 self.code = compile(source, self.filename, 'exec')
324 elif mod_type==imp.PY_COMPILED:
325 self._reopen()
326 try:
327 self.code = read_code(self.file)
328 finally:
329 self.file.close()
330 elif mod_type==imp.PKG_DIRECTORY:
331 self.code = self._get_delegate().get_code()
332 return self.code
334 def get_source(self, fullname=None):
335 fullname = self._fix_name(fullname)
336 if self.source is None:
337 mod_type = self.etc[2]
338 if mod_type==imp.PY_SOURCE:
339 self._reopen()
340 try:
341 self.source = self.file.read()
342 finally:
343 self.file.close()
344 elif mod_type==imp.PY_COMPILED:
345 if os.path.exists(self.filename[:-1]):
346 with open(self.filename[:-1], 'r') as f:
347 self.source = f.read()
348 elif mod_type==imp.PKG_DIRECTORY:
349 self.source = self._get_delegate().get_source()
350 return self.source
352 def _get_delegate(self):
353 finder = ImpImporter(self.filename)
354 spec = _get_spec(finder, '__init__')
355 return spec.loader
357 def get_filename(self, fullname=None):
358 fullname = self._fix_name(fullname)
359 mod_type = self.etc[2]
360 if mod_type==imp.PKG_DIRECTORY:
361 return self._get_delegate().get_filename()
362 elif mod_type in (imp.PY_SOURCE, imp.PY_COMPILED, imp.C_EXTENSION):
363 return self.filename
364 return None
367try:
368 import zipimport
369 from zipimport import zipimporter
371 def iter_zipimport_modules(importer, prefix=''):
372 dirlist = sorted(zipimport._zip_directory_cache[importer.archive])
373 _prefix = importer.prefix
374 plen = len(_prefix)
375 yielded = {}
376 import inspect
377 for fn in dirlist:
378 if not fn.startswith(_prefix):
379 continue
381 fn = fn[plen:].split(os.sep)
383 if len(fn)==2 and fn[1].startswith('__init__.py'):
384 if fn[0] not in yielded:
385 yielded[fn[0]] = 1
386 yield prefix + fn[0], True
388 if len(fn)!=1:
389 continue
391 modname = inspect.getmodulename(fn[0])
392 if modname=='__init__':
393 continue
395 if modname and '.' not in modname and modname not in yielded:
396 yielded[modname] = 1
397 yield prefix + modname, False
399 iter_importer_modules.register(zipimporter, iter_zipimport_modules)
401except ImportError:
402 pass
405def get_importer(path_item):
406 """Retrieve a finder for the given path item
408 The returned finder is cached in sys.path_importer_cache
409 if it was newly created by a path hook.
411 The cache (or part of it) can be cleared manually if a
412 rescan of sys.path_hooks is necessary.
413 """
414 try:
415 importer = sys.path_importer_cache[path_item]
416 except KeyError:
417 for path_hook in sys.path_hooks:
418 try:
419 importer = path_hook(path_item)
420 sys.path_importer_cache.setdefault(path_item, importer)
421 break
422 except ImportError:
423 pass
424 else:
425 importer = None
426 return importer
429def iter_importers(fullname=""):
430 """Yield finders for the given module name
432 If fullname contains a '.', the finders will be for the package
433 containing fullname, otherwise they will be all registered top level
434 finders (i.e. those on both sys.meta_path and sys.path_hooks).
436 If the named module is in a package, that package is imported as a side
437 effect of invoking this function.
439 If no module name is specified, all top level finders are produced.
440 """
441 if fullname.startswith('.'):
442 msg = "Relative module name {!r} not supported".format(fullname)
443 raise ImportError(msg)
444 if '.' in fullname:
445 # Get the containing package's __path__
446 pkg_name = fullname.rpartition(".")[0]
447 pkg = importlib.import_module(pkg_name)
448 path = getattr(pkg, '__path__', None)
449 if path is None:
450 return
451 else:
452 yield from sys.meta_path
453 path = sys.path
454 for item in path:
455 yield get_importer(item)
458def get_loader(module_or_name):
459 """Get a "loader" object for module_or_name
461 Returns None if the module cannot be found or imported.
462 If the named module is not already imported, its containing package
463 (if any) is imported, in order to establish the package __path__.
464 """
465 if module_or_name in sys.modules:
466 module_or_name = sys.modules[module_or_name]
467 if module_or_name is None:
468 return None
469 if isinstance(module_or_name, ModuleType):
470 module = module_or_name
471 loader = getattr(module, '__loader__', None)
472 if loader is not None:
473 return loader
474 if getattr(module, '__spec__', None) is None:
475 return None
476 fullname = module.__name__
477 else:
478 fullname = module_or_name
479 return find_loader(fullname)
482def find_loader(fullname):
483 """Find a "loader" object for fullname
485 This is a backwards compatibility wrapper around
486 importlib.util.find_spec that converts most failures to ImportError
487 and only returns the loader rather than the full spec
488 """
489 if fullname.startswith('.'):
490 msg = "Relative module name {!r} not supported".format(fullname)
491 raise ImportError(msg)
492 try:
493 spec = importlib.util.find_spec(fullname)
494 except (ImportError, AttributeError, TypeError, ValueError) as ex:
495 # This hack fixes an impedance mismatch between pkgutil and
496 # importlib, where the latter raises other errors for cases where
497 # pkgutil previously raised ImportError
498 msg = "Error while finding loader for {!r} ({}: {})"
499 raise ImportError(msg.format(fullname, type(ex), ex)) from ex
500 return spec.loader if spec is not None else None
503def extend_path(path, name):
504 """Extend a package's path.
506 Intended use is to place the following code in a package's __init__.py:
508 from pkgutil import extend_path
509 __path__ = extend_path(__path__, __name__)
511 This will add to the package's __path__ all subdirectories of
512 directories on sys.path named after the package. This is useful
513 if one wants to distribute different parts of a single logical
514 package as multiple directories.
516 It also looks for *.pkg files beginning where * matches the name
517 argument. This feature is similar to *.pth files (see site.py),
518 except that it doesn't special-case lines starting with 'import'.
519 A *.pkg file is trusted at face value: apart from checking for
520 duplicates, all entries found in a *.pkg file are added to the
521 path, regardless of whether they are exist the filesystem. (This
522 is a feature.)
524 If the input path is not a list (as is the case for frozen
525 packages) it is returned unchanged. The input path is not
526 modified; an extended copy is returned. Items are only appended
527 to the copy at the end.
529 It is assumed that sys.path is a sequence. Items of sys.path that
530 are not (unicode or 8-bit) strings referring to existing
531 directories are ignored. Unicode items of sys.path that cause
532 errors when used as filenames may cause this function to raise an
533 exception (in line with os.path.isdir() behavior).
534 """
536 if not isinstance(path, list):
537 # This could happen e.g. when this is called from inside a
538 # frozen package. Return the path unchanged in that case.
539 return path
541 sname_pkg = name + ".pkg"
543 path = path[:] # Start with a copy of the existing path
545 parent_package, _, final_name = name.rpartition('.')
546 if parent_package:
547 try:
548 search_path = sys.modules[parent_package].__path__
549 except (KeyError, AttributeError):
550 # We can't do anything: find_loader() returns None when
551 # passed a dotted name.
552 return path
553 else:
554 search_path = sys.path
556 for dir in search_path:
557 if not isinstance(dir, str):
558 continue
560 finder = get_importer(dir)
561 if finder is not None:
562 portions = []
563 if hasattr(finder, 'find_spec'):
564 spec = finder.find_spec(final_name)
565 if spec is not None:
566 portions = spec.submodule_search_locations or []
567 # Is this finder PEP 420 compliant?
568 elif hasattr(finder, 'find_loader'):
569 _, portions = finder.find_loader(final_name)
571 for portion in portions:
572 # XXX This may still add duplicate entries to path on
573 # case-insensitive filesystems
574 if portion not in path:
575 path.append(portion)
577 # XXX Is this the right thing for subpackages like zope.app?
578 # It looks for a file named "zope.app.pkg"
579 pkgfile = os.path.join(dir, sname_pkg)
580 if os.path.isfile(pkgfile):
581 try:
582 f = open(pkgfile)
583 except OSError as msg:
584 sys.stderr.write("Can't open %s: %s\n" %
585 (pkgfile, msg))
586 else:
587 with f:
588 for line in f:
589 line = line.rstrip('\n')
590 if not line or line.startswith('#'):
591 continue
592 path.append(line) # Don't check for existence!
594 return path
597def get_data(package, resource):
598 """Get a resource from a package.
600 This is a wrapper round the PEP 302 loader get_data API. The package
601 argument should be the name of a package, in standard module format
602 (foo.bar). The resource argument should be in the form of a relative
603 filename, using '/' as the path separator. The parent directory name '..'
604 is not allowed, and nor is a rooted name (starting with a '/').
606 The function returns a binary string, which is the contents of the
607 specified resource.
609 For packages located in the filesystem, which have already been imported,
610 this is the rough equivalent of
612 d = os.path.dirname(sys.modules[package].__file__)
613 data = open(os.path.join(d, resource), 'rb').read()
615 If the package cannot be located or loaded, or it uses a PEP 302 loader
616 which does not support get_data(), then None is returned.
617 """
619 spec = importlib.util.find_spec(package)
620 if spec is None:
621 return None
622 loader = spec.loader
623 if loader is None or not hasattr(loader, 'get_data'):
624 return None
625 # XXX needs test
626 mod = (sys.modules.get(package) or
627 importlib._bootstrap._load(spec))
628 if mod is None or not hasattr(mod, '__file__'):
629 return None
631 # Modify the resource name to be compatible with the loader.get_data
632 # signature - an os.path format "filename" starting with the dirname of
633 # the package's __file__
634 parts = resource.split('/')
635 parts.insert(0, os.path.dirname(mod.__file__))
636 resource_name = os.path.join(*parts)
637 return loader.get_data(resource_name)