Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/pip/_internal/metadata/base.py: 41%
311 statements
« prev ^ index » next coverage.py v7.4.3, created at 2024-02-26 06:33 +0000
1import csv
2import email.message
3import functools
4import json
5import logging
6import pathlib
7import re
8import zipfile
9from typing import (
10 IO,
11 Any,
12 Collection,
13 Container,
14 Dict,
15 Iterable,
16 Iterator,
17 List,
18 NamedTuple,
19 Optional,
20 Protocol,
21 Tuple,
22 Union,
23)
25from pip._vendor.packaging.requirements import Requirement
26from pip._vendor.packaging.specifiers import InvalidSpecifier, SpecifierSet
27from pip._vendor.packaging.utils import NormalizedName, canonicalize_name
28from pip._vendor.packaging.version import LegacyVersion, Version
30from pip._internal.exceptions import NoneMetadataError
31from pip._internal.locations import site_packages, user_site
32from pip._internal.models.direct_url import (
33 DIRECT_URL_METADATA_NAME,
34 DirectUrl,
35 DirectUrlValidationError,
36)
37from pip._internal.utils.compat import stdlib_pkgs # TODO: Move definition here.
38from pip._internal.utils.egg_link import egg_link_path_from_sys_path
39from pip._internal.utils.misc import is_local, normalize_path
40from pip._internal.utils.urls import url_to_path
42from ._json import msg_to_json
44DistributionVersion = Union[LegacyVersion, Version]
46InfoPath = Union[str, pathlib.PurePath]
48logger = logging.getLogger(__name__)
51class BaseEntryPoint(Protocol):
52 @property
53 def name(self) -> str:
54 raise NotImplementedError()
56 @property
57 def value(self) -> str:
58 raise NotImplementedError()
60 @property
61 def group(self) -> str:
62 raise NotImplementedError()
65def _convert_installed_files_path(
66 entry: Tuple[str, ...],
67 info: Tuple[str, ...],
68) -> str:
69 """Convert a legacy installed-files.txt path into modern RECORD path.
71 The legacy format stores paths relative to the info directory, while the
72 modern format stores paths relative to the package root, e.g. the
73 site-packages directory.
75 :param entry: Path parts of the installed-files.txt entry.
76 :param info: Path parts of the egg-info directory relative to package root.
77 :returns: The converted entry.
79 For best compatibility with symlinks, this does not use ``abspath()`` or
80 ``Path.resolve()``, but tries to work with path parts:
82 1. While ``entry`` starts with ``..``, remove the equal amounts of parts
83 from ``info``; if ``info`` is empty, start appending ``..`` instead.
84 2. Join the two directly.
85 """
86 while entry and entry[0] == "..":
87 if not info or info[-1] == "..":
88 info += ("..",)
89 else:
90 info = info[:-1]
91 entry = entry[1:]
92 return str(pathlib.Path(*info, *entry))
95class RequiresEntry(NamedTuple):
96 requirement: str
97 extra: str
98 marker: str
101class BaseDistribution(Protocol):
102 @classmethod
103 def from_directory(cls, directory: str) -> "BaseDistribution":
104 """Load the distribution from a metadata directory.
106 :param directory: Path to a metadata directory, e.g. ``.dist-info``.
107 """
108 raise NotImplementedError()
110 @classmethod
111 def from_metadata_file_contents(
112 cls,
113 metadata_contents: bytes,
114 filename: str,
115 project_name: str,
116 ) -> "BaseDistribution":
117 """Load the distribution from the contents of a METADATA file.
119 This is used to implement PEP 658 by generating a "shallow" dist object that can
120 be used for resolution without downloading or building the actual dist yet.
122 :param metadata_contents: The contents of a METADATA file.
123 :param filename: File name for the dist with this metadata.
124 :param project_name: Name of the project this dist represents.
125 """
126 raise NotImplementedError()
128 @classmethod
129 def from_wheel(cls, wheel: "Wheel", name: str) -> "BaseDistribution":
130 """Load the distribution from a given wheel.
132 :param wheel: A concrete wheel definition.
133 :param name: File name of the wheel.
135 :raises InvalidWheel: Whenever loading of the wheel causes a
136 :py:exc:`zipfile.BadZipFile` exception to be thrown.
137 :raises UnsupportedWheel: If the wheel is a valid zip, but malformed
138 internally.
139 """
140 raise NotImplementedError()
142 def __repr__(self) -> str:
143 return f"{self.raw_name} {self.version} ({self.location})"
145 def __str__(self) -> str:
146 return f"{self.raw_name} {self.version}"
148 @property
149 def location(self) -> Optional[str]:
150 """Where the distribution is loaded from.
152 A string value is not necessarily a filesystem path, since distributions
153 can be loaded from other sources, e.g. arbitrary zip archives. ``None``
154 means the distribution is created in-memory.
156 Do not canonicalize this value with e.g. ``pathlib.Path.resolve()``. If
157 this is a symbolic link, we want to preserve the relative path between
158 it and files in the distribution.
159 """
160 raise NotImplementedError()
162 @property
163 def editable_project_location(self) -> Optional[str]:
164 """The project location for editable distributions.
166 This is the directory where pyproject.toml or setup.py is located.
167 None if the distribution is not installed in editable mode.
168 """
169 # TODO: this property is relatively costly to compute, memoize it ?
170 direct_url = self.direct_url
171 if direct_url:
172 if direct_url.is_local_editable():
173 return url_to_path(direct_url.url)
174 else:
175 # Search for an .egg-link file by walking sys.path, as it was
176 # done before by dist_is_editable().
177 egg_link_path = egg_link_path_from_sys_path(self.raw_name)
178 if egg_link_path:
179 # TODO: get project location from second line of egg_link file
180 # (https://github.com/pypa/pip/issues/10243)
181 return self.location
182 return None
184 @property
185 def installed_location(self) -> Optional[str]:
186 """The distribution's "installed" location.
188 This should generally be a ``site-packages`` directory. This is
189 usually ``dist.location``, except for legacy develop-installed packages,
190 where ``dist.location`` is the source code location, and this is where
191 the ``.egg-link`` file is.
193 The returned location is normalized (in particular, with symlinks removed).
194 """
195 raise NotImplementedError()
197 @property
198 def info_location(self) -> Optional[str]:
199 """Location of the .[egg|dist]-info directory or file.
201 Similarly to ``location``, a string value is not necessarily a
202 filesystem path. ``None`` means the distribution is created in-memory.
204 For a modern .dist-info installation on disk, this should be something
205 like ``{location}/{raw_name}-{version}.dist-info``.
207 Do not canonicalize this value with e.g. ``pathlib.Path.resolve()``. If
208 this is a symbolic link, we want to preserve the relative path between
209 it and other files in the distribution.
210 """
211 raise NotImplementedError()
213 @property
214 def installed_by_distutils(self) -> bool:
215 """Whether this distribution is installed with legacy distutils format.
217 A distribution installed with "raw" distutils not patched by setuptools
218 uses one single file at ``info_location`` to store metadata. We need to
219 treat this specially on uninstallation.
220 """
221 info_location = self.info_location
222 if not info_location:
223 return False
224 return pathlib.Path(info_location).is_file()
226 @property
227 def installed_as_egg(self) -> bool:
228 """Whether this distribution is installed as an egg.
230 This usually indicates the distribution was installed by (older versions
231 of) easy_install.
232 """
233 location = self.location
234 if not location:
235 return False
236 return location.endswith(".egg")
238 @property
239 def installed_with_setuptools_egg_info(self) -> bool:
240 """Whether this distribution is installed with the ``.egg-info`` format.
242 This usually indicates the distribution was installed with setuptools
243 with an old pip version or with ``single-version-externally-managed``.
245 Note that this ensure the metadata store is a directory. distutils can
246 also installs an ``.egg-info``, but as a file, not a directory. This
247 property is *False* for that case. Also see ``installed_by_distutils``.
248 """
249 info_location = self.info_location
250 if not info_location:
251 return False
252 if not info_location.endswith(".egg-info"):
253 return False
254 return pathlib.Path(info_location).is_dir()
256 @property
257 def installed_with_dist_info(self) -> bool:
258 """Whether this distribution is installed with the "modern format".
260 This indicates a "modern" installation, e.g. storing metadata in the
261 ``.dist-info`` directory. This applies to installations made by
262 setuptools (but through pip, not directly), or anything using the
263 standardized build backend interface (PEP 517).
264 """
265 info_location = self.info_location
266 if not info_location:
267 return False
268 if not info_location.endswith(".dist-info"):
269 return False
270 return pathlib.Path(info_location).is_dir()
272 @property
273 def canonical_name(self) -> NormalizedName:
274 raise NotImplementedError()
276 @property
277 def version(self) -> DistributionVersion:
278 raise NotImplementedError()
280 @property
281 def setuptools_filename(self) -> str:
282 """Convert a project name to its setuptools-compatible filename.
284 This is a copy of ``pkg_resources.to_filename()`` for compatibility.
285 """
286 return self.raw_name.replace("-", "_")
288 @property
289 def direct_url(self) -> Optional[DirectUrl]:
290 """Obtain a DirectUrl from this distribution.
292 Returns None if the distribution has no `direct_url.json` metadata,
293 or if `direct_url.json` is invalid.
294 """
295 try:
296 content = self.read_text(DIRECT_URL_METADATA_NAME)
297 except FileNotFoundError:
298 return None
299 try:
300 return DirectUrl.from_json(content)
301 except (
302 UnicodeDecodeError,
303 json.JSONDecodeError,
304 DirectUrlValidationError,
305 ) as e:
306 logger.warning(
307 "Error parsing %s for %s: %s",
308 DIRECT_URL_METADATA_NAME,
309 self.canonical_name,
310 e,
311 )
312 return None
314 @property
315 def installer(self) -> str:
316 try:
317 installer_text = self.read_text("INSTALLER")
318 except (OSError, ValueError, NoneMetadataError):
319 return "" # Fail silently if the installer file cannot be read.
320 for line in installer_text.splitlines():
321 cleaned_line = line.strip()
322 if cleaned_line:
323 return cleaned_line
324 return ""
326 @property
327 def requested(self) -> bool:
328 return self.is_file("REQUESTED")
330 @property
331 def editable(self) -> bool:
332 return bool(self.editable_project_location)
334 @property
335 def local(self) -> bool:
336 """If distribution is installed in the current virtual environment.
338 Always True if we're not in a virtualenv.
339 """
340 if self.installed_location is None:
341 return False
342 return is_local(self.installed_location)
344 @property
345 def in_usersite(self) -> bool:
346 if self.installed_location is None or user_site is None:
347 return False
348 return self.installed_location.startswith(normalize_path(user_site))
350 @property
351 def in_site_packages(self) -> bool:
352 if self.installed_location is None or site_packages is None:
353 return False
354 return self.installed_location.startswith(normalize_path(site_packages))
356 def is_file(self, path: InfoPath) -> bool:
357 """Check whether an entry in the info directory is a file."""
358 raise NotImplementedError()
360 def iter_distutils_script_names(self) -> Iterator[str]:
361 """Find distutils 'scripts' entries metadata.
363 If 'scripts' is supplied in ``setup.py``, distutils records those in the
364 installed distribution's ``scripts`` directory, a file for each script.
365 """
366 raise NotImplementedError()
368 def read_text(self, path: InfoPath) -> str:
369 """Read a file in the info directory.
371 :raise FileNotFoundError: If ``path`` does not exist in the directory.
372 :raise NoneMetadataError: If ``path`` exists in the info directory, but
373 cannot be read.
374 """
375 raise NotImplementedError()
377 def iter_entry_points(self) -> Iterable[BaseEntryPoint]:
378 raise NotImplementedError()
380 def _metadata_impl(self) -> email.message.Message:
381 raise NotImplementedError()
383 @functools.cached_property
384 def metadata(self) -> email.message.Message:
385 """Metadata of distribution parsed from e.g. METADATA or PKG-INFO.
387 This should return an empty message if the metadata file is unavailable.
389 :raises NoneMetadataError: If the metadata file is available, but does
390 not contain valid metadata.
391 """
392 metadata = self._metadata_impl()
393 self._add_egg_info_requires(metadata)
394 return metadata
396 @property
397 def metadata_dict(self) -> Dict[str, Any]:
398 """PEP 566 compliant JSON-serializable representation of METADATA or PKG-INFO.
400 This should return an empty dict if the metadata file is unavailable.
402 :raises NoneMetadataError: If the metadata file is available, but does
403 not contain valid metadata.
404 """
405 return msg_to_json(self.metadata)
407 @property
408 def metadata_version(self) -> Optional[str]:
409 """Value of "Metadata-Version:" in distribution metadata, if available."""
410 return self.metadata.get("Metadata-Version")
412 @property
413 def raw_name(self) -> str:
414 """Value of "Name:" in distribution metadata."""
415 # The metadata should NEVER be missing the Name: key, but if it somehow
416 # does, fall back to the known canonical name.
417 return self.metadata.get("Name", self.canonical_name)
419 @property
420 def requires_python(self) -> SpecifierSet:
421 """Value of "Requires-Python:" in distribution metadata.
423 If the key does not exist or contains an invalid value, an empty
424 SpecifierSet should be returned.
425 """
426 value = self.metadata.get("Requires-Python")
427 if value is None:
428 return SpecifierSet()
429 try:
430 # Convert to str to satisfy the type checker; this can be a Header object.
431 spec = SpecifierSet(str(value))
432 except InvalidSpecifier as e:
433 message = "Package %r has an invalid Requires-Python: %s"
434 logger.warning(message, self.raw_name, e)
435 return SpecifierSet()
436 return spec
438 def iter_dependencies(self, extras: Collection[str] = ()) -> Iterable[Requirement]:
439 """Dependencies of this distribution.
441 For modern .dist-info distributions, this is the collection of
442 "Requires-Dist:" entries in distribution metadata.
443 """
444 raise NotImplementedError()
446 def iter_provided_extras(self) -> Iterable[str]:
447 """Extras provided by this distribution.
449 For modern .dist-info distributions, this is the collection of
450 "Provides-Extra:" entries in distribution metadata.
452 The return value of this function is not particularly useful other than
453 display purposes due to backward compatibility issues and the extra
454 names being poorly normalized prior to PEP 685. If you want to perform
455 logic operations on extras, use :func:`is_extra_provided` instead.
456 """
457 raise NotImplementedError()
459 def is_extra_provided(self, extra: str) -> bool:
460 """Check whether an extra is provided by this distribution.
462 This is needed mostly for compatibility issues with pkg_resources not
463 following the extra normalization rules defined in PEP 685.
464 """
465 raise NotImplementedError()
467 def _iter_declared_entries_from_record(self) -> Optional[Iterator[str]]:
468 try:
469 text = self.read_text("RECORD")
470 except FileNotFoundError:
471 return None
472 # This extra Path-str cast normalizes entries.
473 return (str(pathlib.Path(row[0])) for row in csv.reader(text.splitlines()))
475 def _iter_declared_entries_from_legacy(self) -> Optional[Iterator[str]]:
476 try:
477 text = self.read_text("installed-files.txt")
478 except FileNotFoundError:
479 return None
480 paths = (p for p in text.splitlines(keepends=False) if p)
481 root = self.location
482 info = self.info_location
483 if root is None or info is None:
484 return paths
485 try:
486 info_rel = pathlib.Path(info).relative_to(root)
487 except ValueError: # info is not relative to root.
488 return paths
489 if not info_rel.parts: # info *is* root.
490 return paths
491 return (
492 _convert_installed_files_path(pathlib.Path(p).parts, info_rel.parts)
493 for p in paths
494 )
496 def iter_declared_entries(self) -> Optional[Iterator[str]]:
497 """Iterate through file entries declared in this distribution.
499 For modern .dist-info distributions, this is the files listed in the
500 ``RECORD`` metadata file. For legacy setuptools distributions, this
501 comes from ``installed-files.txt``, with entries normalized to be
502 compatible with the format used by ``RECORD``.
504 :return: An iterator for listed entries, or None if the distribution
505 contains neither ``RECORD`` nor ``installed-files.txt``.
506 """
507 return (
508 self._iter_declared_entries_from_record()
509 or self._iter_declared_entries_from_legacy()
510 )
512 def _iter_requires_txt_entries(self) -> Iterator[RequiresEntry]:
513 """Parse a ``requires.txt`` in an egg-info directory.
515 This is an INI-ish format where an egg-info stores dependencies. A
516 section name describes extra other environment markers, while each entry
517 is an arbitrary string (not a key-value pair) representing a dependency
518 as a requirement string (no markers).
520 There is a construct in ``importlib.metadata`` called ``Sectioned`` that
521 does mostly the same, but the format is currently considered private.
522 """
523 try:
524 content = self.read_text("requires.txt")
525 except FileNotFoundError:
526 return
527 extra = marker = "" # Section-less entries don't have markers.
528 for line in content.splitlines():
529 line = line.strip()
530 if not line or line.startswith("#"): # Comment; ignored.
531 continue
532 if line.startswith("[") and line.endswith("]"): # A section header.
533 extra, _, marker = line.strip("[]").partition(":")
534 continue
535 yield RequiresEntry(requirement=line, extra=extra, marker=marker)
537 def _iter_egg_info_extras(self) -> Iterable[str]:
538 """Get extras from the egg-info directory."""
539 known_extras = {""}
540 for entry in self._iter_requires_txt_entries():
541 extra = canonicalize_name(entry.extra)
542 if extra in known_extras:
543 continue
544 known_extras.add(extra)
545 yield extra
547 def _iter_egg_info_dependencies(self) -> Iterable[str]:
548 """Get distribution dependencies from the egg-info directory.
550 To ease parsing, this converts a legacy dependency entry into a PEP 508
551 requirement string. Like ``_iter_requires_txt_entries()``, there is code
552 in ``importlib.metadata`` that does mostly the same, but not do exactly
553 what we need.
555 Namely, ``importlib.metadata`` does not normalize the extra name before
556 putting it into the requirement string, which causes marker comparison
557 to fail because the dist-info format do normalize. This is consistent in
558 all currently available PEP 517 backends, although not standardized.
559 """
560 for entry in self._iter_requires_txt_entries():
561 extra = canonicalize_name(entry.extra)
562 if extra and entry.marker:
563 marker = f'({entry.marker}) and extra == "{extra}"'
564 elif extra:
565 marker = f'extra == "{extra}"'
566 elif entry.marker:
567 marker = entry.marker
568 else:
569 marker = ""
570 if marker:
571 yield f"{entry.requirement} ; {marker}"
572 else:
573 yield entry.requirement
575 def _add_egg_info_requires(self, metadata: email.message.Message) -> None:
576 """Add egg-info requires.txt information to the metadata."""
577 if not metadata.get_all("Requires-Dist"):
578 for dep in self._iter_egg_info_dependencies():
579 metadata["Requires-Dist"] = dep
580 if not metadata.get_all("Provides-Extra"):
581 for extra in self._iter_egg_info_extras():
582 metadata["Provides-Extra"] = extra
585class BaseEnvironment:
586 """An environment containing distributions to introspect."""
588 @classmethod
589 def default(cls) -> "BaseEnvironment":
590 raise NotImplementedError()
592 @classmethod
593 def from_paths(cls, paths: Optional[List[str]]) -> "BaseEnvironment":
594 raise NotImplementedError()
596 def get_distribution(self, name: str) -> Optional["BaseDistribution"]:
597 """Given a requirement name, return the installed distributions.
599 The name may not be normalized. The implementation must canonicalize
600 it for lookup.
601 """
602 raise NotImplementedError()
604 def _iter_distributions(self) -> Iterator["BaseDistribution"]:
605 """Iterate through installed distributions.
607 This function should be implemented by subclass, but never called
608 directly. Use the public ``iter_distribution()`` instead, which
609 implements additional logic to make sure the distributions are valid.
610 """
611 raise NotImplementedError()
613 def iter_all_distributions(self) -> Iterator[BaseDistribution]:
614 """Iterate through all installed distributions without any filtering."""
615 for dist in self._iter_distributions():
616 # Make sure the distribution actually comes from a valid Python
617 # packaging distribution. Pip's AdjacentTempDirectory leaves folders
618 # e.g. ``~atplotlib.dist-info`` if cleanup was interrupted. The
619 # valid project name pattern is taken from PEP 508.
620 project_name_valid = re.match(
621 r"^([A-Z0-9]|[A-Z0-9][A-Z0-9._-]*[A-Z0-9])$",
622 dist.canonical_name,
623 flags=re.IGNORECASE,
624 )
625 if not project_name_valid:
626 logger.warning(
627 "Ignoring invalid distribution %s (%s)",
628 dist.canonical_name,
629 dist.location,
630 )
631 continue
632 yield dist
634 def iter_installed_distributions(
635 self,
636 local_only: bool = True,
637 skip: Container[str] = stdlib_pkgs,
638 include_editables: bool = True,
639 editables_only: bool = False,
640 user_only: bool = False,
641 ) -> Iterator[BaseDistribution]:
642 """Return a list of installed distributions.
644 This is based on ``iter_all_distributions()`` with additional filtering
645 options. Note that ``iter_installed_distributions()`` without arguments
646 is *not* equal to ``iter_all_distributions()``, since some of the
647 configurations exclude packages by default.
649 :param local_only: If True (default), only return installations
650 local to the current virtualenv, if in a virtualenv.
651 :param skip: An iterable of canonicalized project names to ignore;
652 defaults to ``stdlib_pkgs``.
653 :param include_editables: If False, don't report editables.
654 :param editables_only: If True, only report editables.
655 :param user_only: If True, only report installations in the user
656 site directory.
657 """
658 it = self.iter_all_distributions()
659 if local_only:
660 it = (d for d in it if d.local)
661 if not include_editables:
662 it = (d for d in it if not d.editable)
663 if editables_only:
664 it = (d for d in it if d.editable)
665 if user_only:
666 it = (d for d in it if d.in_usersite)
667 return (d for d in it if d.canonical_name not in skip)
670class Wheel(Protocol):
671 location: str
673 def as_zipfile(self) -> zipfile.ZipFile:
674 raise NotImplementedError()
677class FilesystemWheel(Wheel):
678 def __init__(self, location: str) -> None:
679 self.location = location
681 def as_zipfile(self) -> zipfile.ZipFile:
682 return zipfile.ZipFile(self.location, allowZip64=True)
685class MemoryWheel(Wheel):
686 def __init__(self, location: str, stream: IO[bytes]) -> None:
687 self.location = location
688 self.stream = stream
690 def as_zipfile(self) -> zipfile.ZipFile:
691 return zipfile.ZipFile(self.stream, allowZip64=True)