Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/pip/_internal/metadata/base.py: 41%
314 statements
« prev ^ index » next coverage.py v7.2.7, created at 2023-06-07 06:48 +0000
1import csv
2import email.message
3import functools
4import json
5import logging
6import pathlib
7import re
8import zipfile
9from typing import (
10 IO,
11 TYPE_CHECKING,
12 Any,
13 Collection,
14 Container,
15 Dict,
16 Iterable,
17 Iterator,
18 List,
19 NamedTuple,
20 Optional,
21 Tuple,
22 Union,
23)
25from pip._vendor.packaging.requirements import Requirement
26from pip._vendor.packaging.specifiers import InvalidSpecifier, SpecifierSet
27from pip._vendor.packaging.utils import NormalizedName
28from pip._vendor.packaging.version import LegacyVersion, Version
30from pip._internal.exceptions import NoneMetadataError
31from pip._internal.locations import site_packages, user_site
32from pip._internal.models.direct_url import (
33 DIRECT_URL_METADATA_NAME,
34 DirectUrl,
35 DirectUrlValidationError,
36)
37from pip._internal.utils.compat import stdlib_pkgs # TODO: Move definition here.
38from pip._internal.utils.egg_link import egg_link_path_from_sys_path
39from pip._internal.utils.misc import is_local, normalize_path
40from pip._internal.utils.packaging import safe_extra
41from pip._internal.utils.urls import url_to_path
43from ._json import msg_to_json
45if TYPE_CHECKING:
46 from typing import Protocol
47else:
48 Protocol = object
50DistributionVersion = Union[LegacyVersion, Version]
52InfoPath = Union[str, pathlib.PurePath]
54logger = logging.getLogger(__name__)
57class BaseEntryPoint(Protocol):
58 @property
59 def name(self) -> str:
60 raise NotImplementedError()
62 @property
63 def value(self) -> str:
64 raise NotImplementedError()
66 @property
67 def group(self) -> str:
68 raise NotImplementedError()
71def _convert_installed_files_path(
72 entry: Tuple[str, ...],
73 info: Tuple[str, ...],
74) -> str:
75 """Convert a legacy installed-files.txt path into modern RECORD path.
77 The legacy format stores paths relative to the info directory, while the
78 modern format stores paths relative to the package root, e.g. the
79 site-packages directory.
81 :param entry: Path parts of the installed-files.txt entry.
82 :param info: Path parts of the egg-info directory relative to package root.
83 :returns: The converted entry.
85 For best compatibility with symlinks, this does not use ``abspath()`` or
86 ``Path.resolve()``, but tries to work with path parts:
88 1. While ``entry`` starts with ``..``, remove the equal amounts of parts
89 from ``info``; if ``info`` is empty, start appending ``..`` instead.
90 2. Join the two directly.
91 """
92 while entry and entry[0] == "..":
93 if not info or info[-1] == "..":
94 info += ("..",)
95 else:
96 info = info[:-1]
97 entry = entry[1:]
98 return str(pathlib.Path(*info, *entry))
101class RequiresEntry(NamedTuple):
102 requirement: str
103 extra: str
104 marker: str
107class BaseDistribution(Protocol):
108 @classmethod
109 def from_directory(cls, directory: str) -> "BaseDistribution":
110 """Load the distribution from a metadata directory.
112 :param directory: Path to a metadata directory, e.g. ``.dist-info``.
113 """
114 raise NotImplementedError()
116 @classmethod
117 def from_metadata_file_contents(
118 cls,
119 metadata_contents: bytes,
120 filename: str,
121 project_name: str,
122 ) -> "BaseDistribution":
123 """Load the distribution from the contents of a METADATA file.
125 This is used to implement PEP 658 by generating a "shallow" dist object that can
126 be used for resolution without downloading or building the actual dist yet.
128 :param metadata_contents: The contents of a METADATA file.
129 :param filename: File name for the dist with this metadata.
130 :param project_name: Name of the project this dist represents.
131 """
132 raise NotImplementedError()
134 @classmethod
135 def from_wheel(cls, wheel: "Wheel", name: str) -> "BaseDistribution":
136 """Load the distribution from a given wheel.
138 :param wheel: A concrete wheel definition.
139 :param name: File name of the wheel.
141 :raises InvalidWheel: Whenever loading of the wheel causes a
142 :py:exc:`zipfile.BadZipFile` exception to be thrown.
143 :raises UnsupportedWheel: If the wheel is a valid zip, but malformed
144 internally.
145 """
146 raise NotImplementedError()
148 def __repr__(self) -> str:
149 return f"{self.raw_name} {self.version} ({self.location})"
151 def __str__(self) -> str:
152 return f"{self.raw_name} {self.version}"
154 @property
155 def location(self) -> Optional[str]:
156 """Where the distribution is loaded from.
158 A string value is not necessarily a filesystem path, since distributions
159 can be loaded from other sources, e.g. arbitrary zip archives. ``None``
160 means the distribution is created in-memory.
162 Do not canonicalize this value with e.g. ``pathlib.Path.resolve()``. If
163 this is a symbolic link, we want to preserve the relative path between
164 it and files in the distribution.
165 """
166 raise NotImplementedError()
168 @property
169 def editable_project_location(self) -> Optional[str]:
170 """The project location for editable distributions.
172 This is the directory where pyproject.toml or setup.py is located.
173 None if the distribution is not installed in editable mode.
174 """
175 # TODO: this property is relatively costly to compute, memoize it ?
176 direct_url = self.direct_url
177 if direct_url:
178 if direct_url.is_local_editable():
179 return url_to_path(direct_url.url)
180 else:
181 # Search for an .egg-link file by walking sys.path, as it was
182 # done before by dist_is_editable().
183 egg_link_path = egg_link_path_from_sys_path(self.raw_name)
184 if egg_link_path:
185 # TODO: get project location from second line of egg_link file
186 # (https://github.com/pypa/pip/issues/10243)
187 return self.location
188 return None
190 @property
191 def installed_location(self) -> Optional[str]:
192 """The distribution's "installed" location.
194 This should generally be a ``site-packages`` directory. This is
195 usually ``dist.location``, except for legacy develop-installed packages,
196 where ``dist.location`` is the source code location, and this is where
197 the ``.egg-link`` file is.
199 The returned location is normalized (in particular, with symlinks removed).
200 """
201 raise NotImplementedError()
203 @property
204 def info_location(self) -> Optional[str]:
205 """Location of the .[egg|dist]-info directory or file.
207 Similarly to ``location``, a string value is not necessarily a
208 filesystem path. ``None`` means the distribution is created in-memory.
210 For a modern .dist-info installation on disk, this should be something
211 like ``{location}/{raw_name}-{version}.dist-info``.
213 Do not canonicalize this value with e.g. ``pathlib.Path.resolve()``. If
214 this is a symbolic link, we want to preserve the relative path between
215 it and other files in the distribution.
216 """
217 raise NotImplementedError()
219 @property
220 def installed_by_distutils(self) -> bool:
221 """Whether this distribution is installed with legacy distutils format.
223 A distribution installed with "raw" distutils not patched by setuptools
224 uses one single file at ``info_location`` to store metadata. We need to
225 treat this specially on uninstallation.
226 """
227 info_location = self.info_location
228 if not info_location:
229 return False
230 return pathlib.Path(info_location).is_file()
232 @property
233 def installed_as_egg(self) -> bool:
234 """Whether this distribution is installed as an egg.
236 This usually indicates the distribution was installed by (older versions
237 of) easy_install.
238 """
239 location = self.location
240 if not location:
241 return False
242 return location.endswith(".egg")
244 @property
245 def installed_with_setuptools_egg_info(self) -> bool:
246 """Whether this distribution is installed with the ``.egg-info`` format.
248 This usually indicates the distribution was installed with setuptools
249 with an old pip version or with ``single-version-externally-managed``.
251 Note that this ensure the metadata store is a directory. distutils can
252 also installs an ``.egg-info``, but as a file, not a directory. This
253 property is *False* for that case. Also see ``installed_by_distutils``.
254 """
255 info_location = self.info_location
256 if not info_location:
257 return False
258 if not info_location.endswith(".egg-info"):
259 return False
260 return pathlib.Path(info_location).is_dir()
262 @property
263 def installed_with_dist_info(self) -> bool:
264 """Whether this distribution is installed with the "modern format".
266 This indicates a "modern" installation, e.g. storing metadata in the
267 ``.dist-info`` directory. This applies to installations made by
268 setuptools (but through pip, not directly), or anything using the
269 standardized build backend interface (PEP 517).
270 """
271 info_location = self.info_location
272 if not info_location:
273 return False
274 if not info_location.endswith(".dist-info"):
275 return False
276 return pathlib.Path(info_location).is_dir()
278 @property
279 def canonical_name(self) -> NormalizedName:
280 raise NotImplementedError()
282 @property
283 def version(self) -> DistributionVersion:
284 raise NotImplementedError()
286 @property
287 def setuptools_filename(self) -> str:
288 """Convert a project name to its setuptools-compatible filename.
290 This is a copy of ``pkg_resources.to_filename()`` for compatibility.
291 """
292 return self.raw_name.replace("-", "_")
294 @property
295 def direct_url(self) -> Optional[DirectUrl]:
296 """Obtain a DirectUrl from this distribution.
298 Returns None if the distribution has no `direct_url.json` metadata,
299 or if `direct_url.json` is invalid.
300 """
301 try:
302 content = self.read_text(DIRECT_URL_METADATA_NAME)
303 except FileNotFoundError:
304 return None
305 try:
306 return DirectUrl.from_json(content)
307 except (
308 UnicodeDecodeError,
309 json.JSONDecodeError,
310 DirectUrlValidationError,
311 ) as e:
312 logger.warning(
313 "Error parsing %s for %s: %s",
314 DIRECT_URL_METADATA_NAME,
315 self.canonical_name,
316 e,
317 )
318 return None
320 @property
321 def installer(self) -> str:
322 try:
323 installer_text = self.read_text("INSTALLER")
324 except (OSError, ValueError, NoneMetadataError):
325 return "" # Fail silently if the installer file cannot be read.
326 for line in installer_text.splitlines():
327 cleaned_line = line.strip()
328 if cleaned_line:
329 return cleaned_line
330 return ""
332 @property
333 def requested(self) -> bool:
334 return self.is_file("REQUESTED")
336 @property
337 def editable(self) -> bool:
338 return bool(self.editable_project_location)
340 @property
341 def local(self) -> bool:
342 """If distribution is installed in the current virtual environment.
344 Always True if we're not in a virtualenv.
345 """
346 if self.installed_location is None:
347 return False
348 return is_local(self.installed_location)
350 @property
351 def in_usersite(self) -> bool:
352 if self.installed_location is None or user_site is None:
353 return False
354 return self.installed_location.startswith(normalize_path(user_site))
356 @property
357 def in_site_packages(self) -> bool:
358 if self.installed_location is None or site_packages is None:
359 return False
360 return self.installed_location.startswith(normalize_path(site_packages))
362 def is_file(self, path: InfoPath) -> bool:
363 """Check whether an entry in the info directory is a file."""
364 raise NotImplementedError()
366 def iter_distutils_script_names(self) -> Iterator[str]:
367 """Find distutils 'scripts' entries metadata.
369 If 'scripts' is supplied in ``setup.py``, distutils records those in the
370 installed distribution's ``scripts`` directory, a file for each script.
371 """
372 raise NotImplementedError()
374 def read_text(self, path: InfoPath) -> str:
375 """Read a file in the info directory.
377 :raise FileNotFoundError: If ``path`` does not exist in the directory.
378 :raise NoneMetadataError: If ``path`` exists in the info directory, but
379 cannot be read.
380 """
381 raise NotImplementedError()
383 def iter_entry_points(self) -> Iterable[BaseEntryPoint]:
384 raise NotImplementedError()
386 def _metadata_impl(self) -> email.message.Message:
387 raise NotImplementedError()
389 @functools.lru_cache(maxsize=1)
390 def _metadata_cached(self) -> email.message.Message:
391 # When we drop python 3.7 support, move this to the metadata property and use
392 # functools.cached_property instead of lru_cache.
393 metadata = self._metadata_impl()
394 self._add_egg_info_requires(metadata)
395 return metadata
397 @property
398 def metadata(self) -> email.message.Message:
399 """Metadata of distribution parsed from e.g. METADATA or PKG-INFO.
401 This should return an empty message if the metadata file is unavailable.
403 :raises NoneMetadataError: If the metadata file is available, but does
404 not contain valid metadata.
405 """
406 return self._metadata_cached()
408 @property
409 def metadata_dict(self) -> Dict[str, Any]:
410 """PEP 566 compliant JSON-serializable representation of METADATA or PKG-INFO.
412 This should return an empty dict if the metadata file is unavailable.
414 :raises NoneMetadataError: If the metadata file is available, but does
415 not contain valid metadata.
416 """
417 return msg_to_json(self.metadata)
419 @property
420 def metadata_version(self) -> Optional[str]:
421 """Value of "Metadata-Version:" in distribution metadata, if available."""
422 return self.metadata.get("Metadata-Version")
424 @property
425 def raw_name(self) -> str:
426 """Value of "Name:" in distribution metadata."""
427 # The metadata should NEVER be missing the Name: key, but if it somehow
428 # does, fall back to the known canonical name.
429 return self.metadata.get("Name", self.canonical_name)
431 @property
432 def requires_python(self) -> SpecifierSet:
433 """Value of "Requires-Python:" in distribution metadata.
435 If the key does not exist or contains an invalid value, an empty
436 SpecifierSet should be returned.
437 """
438 value = self.metadata.get("Requires-Python")
439 if value is None:
440 return SpecifierSet()
441 try:
442 # Convert to str to satisfy the type checker; this can be a Header object.
443 spec = SpecifierSet(str(value))
444 except InvalidSpecifier as e:
445 message = "Package %r has an invalid Requires-Python: %s"
446 logger.warning(message, self.raw_name, e)
447 return SpecifierSet()
448 return spec
450 def iter_dependencies(self, extras: Collection[str] = ()) -> Iterable[Requirement]:
451 """Dependencies of this distribution.
453 For modern .dist-info distributions, this is the collection of
454 "Requires-Dist:" entries in distribution metadata.
455 """
456 raise NotImplementedError()
458 def iter_provided_extras(self) -> Iterable[str]:
459 """Extras provided by this distribution.
461 For modern .dist-info distributions, this is the collection of
462 "Provides-Extra:" entries in distribution metadata.
463 """
464 raise NotImplementedError()
466 def _iter_declared_entries_from_record(self) -> Optional[Iterator[str]]:
467 try:
468 text = self.read_text("RECORD")
469 except FileNotFoundError:
470 return None
471 # This extra Path-str cast normalizes entries.
472 return (str(pathlib.Path(row[0])) for row in csv.reader(text.splitlines()))
474 def _iter_declared_entries_from_legacy(self) -> Optional[Iterator[str]]:
475 try:
476 text = self.read_text("installed-files.txt")
477 except FileNotFoundError:
478 return None
479 paths = (p for p in text.splitlines(keepends=False) if p)
480 root = self.location
481 info = self.info_location
482 if root is None or info is None:
483 return paths
484 try:
485 info_rel = pathlib.Path(info).relative_to(root)
486 except ValueError: # info is not relative to root.
487 return paths
488 if not info_rel.parts: # info *is* root.
489 return paths
490 return (
491 _convert_installed_files_path(pathlib.Path(p).parts, info_rel.parts)
492 for p in paths
493 )
495 def iter_declared_entries(self) -> Optional[Iterator[str]]:
496 """Iterate through file entries declared in this distribution.
498 For modern .dist-info distributions, this is the files listed in the
499 ``RECORD`` metadata file. For legacy setuptools distributions, this
500 comes from ``installed-files.txt``, with entries normalized to be
501 compatible with the format used by ``RECORD``.
503 :return: An iterator for listed entries, or None if the distribution
504 contains neither ``RECORD`` nor ``installed-files.txt``.
505 """
506 return (
507 self._iter_declared_entries_from_record()
508 or self._iter_declared_entries_from_legacy()
509 )
511 def _iter_requires_txt_entries(self) -> Iterator[RequiresEntry]:
512 """Parse a ``requires.txt`` in an egg-info directory.
514 This is an INI-ish format where an egg-info stores dependencies. A
515 section name describes extra other environment markers, while each entry
516 is an arbitrary string (not a key-value pair) representing a dependency
517 as a requirement string (no markers).
519 There is a construct in ``importlib.metadata`` called ``Sectioned`` that
520 does mostly the same, but the format is currently considered private.
521 """
522 try:
523 content = self.read_text("requires.txt")
524 except FileNotFoundError:
525 return
526 extra = marker = "" # Section-less entries don't have markers.
527 for line in content.splitlines():
528 line = line.strip()
529 if not line or line.startswith("#"): # Comment; ignored.
530 continue
531 if line.startswith("[") and line.endswith("]"): # A section header.
532 extra, _, marker = line.strip("[]").partition(":")
533 continue
534 yield RequiresEntry(requirement=line, extra=extra, marker=marker)
536 def _iter_egg_info_extras(self) -> Iterable[str]:
537 """Get extras from the egg-info directory."""
538 known_extras = {""}
539 for entry in self._iter_requires_txt_entries():
540 if entry.extra in known_extras:
541 continue
542 known_extras.add(entry.extra)
543 yield entry.extra
545 def _iter_egg_info_dependencies(self) -> Iterable[str]:
546 """Get distribution dependencies from the egg-info directory.
548 To ease parsing, this converts a legacy dependency entry into a PEP 508
549 requirement string. Like ``_iter_requires_txt_entries()``, there is code
550 in ``importlib.metadata`` that does mostly the same, but not do exactly
551 what we need.
553 Namely, ``importlib.metadata`` does not normalize the extra name before
554 putting it into the requirement string, which causes marker comparison
555 to fail because the dist-info format do normalize. This is consistent in
556 all currently available PEP 517 backends, although not standardized.
557 """
558 for entry in self._iter_requires_txt_entries():
559 if entry.extra and entry.marker:
560 marker = f'({entry.marker}) and extra == "{safe_extra(entry.extra)}"'
561 elif entry.extra:
562 marker = f'extra == "{safe_extra(entry.extra)}"'
563 elif entry.marker:
564 marker = entry.marker
565 else:
566 marker = ""
567 if marker:
568 yield f"{entry.requirement} ; {marker}"
569 else:
570 yield entry.requirement
572 def _add_egg_info_requires(self, metadata: email.message.Message) -> None:
573 """Add egg-info requires.txt information to the metadata."""
574 if not metadata.get_all("Requires-Dist"):
575 for dep in self._iter_egg_info_dependencies():
576 metadata["Requires-Dist"] = dep
577 if not metadata.get_all("Provides-Extra"):
578 for extra in self._iter_egg_info_extras():
579 metadata["Provides-Extra"] = extra
582class BaseEnvironment:
583 """An environment containing distributions to introspect."""
585 @classmethod
586 def default(cls) -> "BaseEnvironment":
587 raise NotImplementedError()
589 @classmethod
590 def from_paths(cls, paths: Optional[List[str]]) -> "BaseEnvironment":
591 raise NotImplementedError()
593 def get_distribution(self, name: str) -> Optional["BaseDistribution"]:
594 """Given a requirement name, return the installed distributions.
596 The name may not be normalized. The implementation must canonicalize
597 it for lookup.
598 """
599 raise NotImplementedError()
601 def _iter_distributions(self) -> Iterator["BaseDistribution"]:
602 """Iterate through installed distributions.
604 This function should be implemented by subclass, but never called
605 directly. Use the public ``iter_distribution()`` instead, which
606 implements additional logic to make sure the distributions are valid.
607 """
608 raise NotImplementedError()
610 def iter_all_distributions(self) -> Iterator[BaseDistribution]:
611 """Iterate through all installed distributions without any filtering."""
612 for dist in self._iter_distributions():
613 # Make sure the distribution actually comes from a valid Python
614 # packaging distribution. Pip's AdjacentTempDirectory leaves folders
615 # e.g. ``~atplotlib.dist-info`` if cleanup was interrupted. The
616 # valid project name pattern is taken from PEP 508.
617 project_name_valid = re.match(
618 r"^([A-Z0-9]|[A-Z0-9][A-Z0-9._-]*[A-Z0-9])$",
619 dist.canonical_name,
620 flags=re.IGNORECASE,
621 )
622 if not project_name_valid:
623 logger.warning(
624 "Ignoring invalid distribution %s (%s)",
625 dist.canonical_name,
626 dist.location,
627 )
628 continue
629 yield dist
631 def iter_installed_distributions(
632 self,
633 local_only: bool = True,
634 skip: Container[str] = stdlib_pkgs,
635 include_editables: bool = True,
636 editables_only: bool = False,
637 user_only: bool = False,
638 ) -> Iterator[BaseDistribution]:
639 """Return a list of installed distributions.
641 This is based on ``iter_all_distributions()`` with additional filtering
642 options. Note that ``iter_installed_distributions()`` without arguments
643 is *not* equal to ``iter_all_distributions()``, since some of the
644 configurations exclude packages by default.
646 :param local_only: If True (default), only return installations
647 local to the current virtualenv, if in a virtualenv.
648 :param skip: An iterable of canonicalized project names to ignore;
649 defaults to ``stdlib_pkgs``.
650 :param include_editables: If False, don't report editables.
651 :param editables_only: If True, only report editables.
652 :param user_only: If True, only report installations in the user
653 site directory.
654 """
655 it = self.iter_all_distributions()
656 if local_only:
657 it = (d for d in it if d.local)
658 if not include_editables:
659 it = (d for d in it if not d.editable)
660 if editables_only:
661 it = (d for d in it if d.editable)
662 if user_only:
663 it = (d for d in it if d.in_usersite)
664 return (d for d in it if d.canonical_name not in skip)
667class Wheel(Protocol):
668 location: str
670 def as_zipfile(self) -> zipfile.ZipFile:
671 raise NotImplementedError()
674class FilesystemWheel(Wheel):
675 def __init__(self, location: str) -> None:
676 self.location = location
678 def as_zipfile(self) -> zipfile.ZipFile:
679 return zipfile.ZipFile(self.location, allowZip64=True)
682class MemoryWheel(Wheel):
683 def __init__(self, location: str, stream: IO[bytes]) -> None:
684 self.location = location
685 self.stream = stream
687 def as_zipfile(self) -> zipfile.ZipFile:
688 return zipfile.ZipFile(self.stream, allowZip64=True)