Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1from __future__ import annotations
3import csv
4import email.message
5import functools
6import json
7import logging
8import pathlib
9import re
10import zipfile
11from collections.abc import Collection, Container, Iterable, Iterator
12from typing import (
13 IO,
14 Any,
15 NamedTuple,
16 Protocol,
17)
19from pip._vendor.packaging.requirements import Requirement
20from pip._vendor.packaging.specifiers import InvalidSpecifier, SpecifierSet
21from pip._vendor.packaging.utils import NormalizedName, canonicalize_name
22from pip._vendor.packaging.version import Version
24from pip._internal.exceptions import NoneMetadataError
25from pip._internal.locations import site_packages, user_site
26from pip._internal.models.direct_url import (
27 DIRECT_URL_METADATA_NAME,
28 DirectUrl,
29 DirectUrlValidationError,
30)
31from pip._internal.utils.egg_link import egg_link_path_from_sys_path
32from pip._internal.utils.misc import is_local, normalize_path
33from pip._internal.utils.urls import url_to_path
35from ._json import msg_to_json
37# packages in the stdlib that may have installation metadata, but should not be
38# considered 'installed'. this theoretically could be determined based on
39# dist.location (py27:`sysconfig.get_paths()['stdlib']`,
40# py26:sysconfig.get_config_vars('LIBDEST')), but fear platform variation may
41# make this ineffective, so hard-coding
42stdlib_pkgs = {"python", "wsgiref", "argparse"}
45InfoPath = str | pathlib.PurePath
47logger = logging.getLogger(__name__)
50class BaseEntryPoint(Protocol):
51 @property
52 def name(self) -> str:
53 raise NotImplementedError()
55 @property
56 def value(self) -> str:
57 raise NotImplementedError()
59 @property
60 def group(self) -> str:
61 raise NotImplementedError()
64def _convert_installed_files_path(
65 entry: tuple[str, ...],
66 info: tuple[str, ...],
67) -> str:
68 """Convert a legacy installed-files.txt path into modern RECORD path.
70 The legacy format stores paths relative to the info directory, while the
71 modern format stores paths relative to the package root, e.g. the
72 site-packages directory.
74 :param entry: Path parts of the installed-files.txt entry.
75 :param info: Path parts of the egg-info directory relative to package root.
76 :returns: The converted entry.
78 For best compatibility with symlinks, this does not use ``abspath()`` or
79 ``Path.resolve()``, but tries to work with path parts:
81 1. While ``entry`` starts with ``..``, remove the equal amounts of parts
82 from ``info``; if ``info`` is empty, start appending ``..`` instead.
83 2. Join the two directly.
84 """
85 while entry and entry[0] == "..":
86 if not info or info[-1] == "..":
87 info += ("..",)
88 else:
89 info = info[:-1]
90 entry = entry[1:]
91 return str(pathlib.Path(*info, *entry))
94class RequiresEntry(NamedTuple):
95 requirement: str
96 extra: str
97 marker: str
100class BaseDistribution(Protocol):
101 @classmethod
102 def from_directory(cls, directory: str) -> BaseDistribution:
103 """Load the distribution from a metadata directory.
105 :param directory: Path to a metadata directory, e.g. ``.dist-info``.
106 """
107 raise NotImplementedError()
109 @classmethod
110 def from_metadata_file_contents(
111 cls,
112 metadata_contents: bytes,
113 filename: str,
114 project_name: str,
115 ) -> BaseDistribution:
116 """Load the distribution from the contents of a METADATA file.
118 This is used to implement PEP 658 by generating a "shallow" dist object that can
119 be used for resolution without downloading or building the actual dist yet.
121 :param metadata_contents: The contents of a METADATA file.
122 :param filename: File name for the dist with this metadata.
123 :param project_name: Name of the project this dist represents.
124 """
125 raise NotImplementedError()
127 @classmethod
128 def from_wheel(cls, wheel: Wheel, name: str) -> BaseDistribution:
129 """Load the distribution from a given wheel.
131 :param wheel: A concrete wheel definition.
132 :param name: File name of the wheel.
134 :raises InvalidWheel: Whenever loading of the wheel causes a
135 :py:exc:`zipfile.BadZipFile` exception to be thrown.
136 :raises UnsupportedWheel: If the wheel is a valid zip, but malformed
137 internally.
138 """
139 raise NotImplementedError()
141 def __repr__(self) -> str:
142 return f"{self.raw_name} {self.raw_version} ({self.location})"
144 def __str__(self) -> str:
145 return f"{self.raw_name} {self.raw_version}"
147 @property
148 def location(self) -> str | None:
149 """Where the distribution is loaded from.
151 A string value is not necessarily a filesystem path, since distributions
152 can be loaded from other sources, e.g. arbitrary zip archives. ``None``
153 means the distribution is created in-memory.
155 Do not canonicalize this value with e.g. ``pathlib.Path.resolve()``. If
156 this is a symbolic link, we want to preserve the relative path between
157 it and files in the distribution.
158 """
159 raise NotImplementedError()
161 @functools.cached_property
162 def editable_project_location(self) -> str | None:
163 """The project location for editable distributions.
165 This is the directory where pyproject.toml or setup.py is located.
166 None if the distribution is not installed in editable mode.
167 """
168 direct_url = self.direct_url
169 if direct_url:
170 if direct_url.is_local_editable():
171 return url_to_path(direct_url.url)
172 else:
173 # Search for an .egg-link file by walking sys.path, as it was
174 # done before by dist_is_editable().
175 egg_link_path = egg_link_path_from_sys_path(self.raw_name)
176 if egg_link_path:
177 # TODO: get project location from second line of egg_link file
178 # (https://github.com/pypa/pip/issues/10243)
179 return self.location
180 return None
182 @property
183 def installed_location(self) -> str | None:
184 """The distribution's "installed" location.
186 This should generally be a ``site-packages`` directory. This is
187 usually ``dist.location``, except for legacy develop-installed packages,
188 where ``dist.location`` is the source code location, and this is where
189 the ``.egg-link`` file is.
191 The returned location is normalized (in particular, with symlinks removed).
192 """
193 raise NotImplementedError()
195 @property
196 def info_location(self) -> str | None:
197 """Location of the .[egg|dist]-info directory or file.
199 Similarly to ``location``, a string value is not necessarily a
200 filesystem path. ``None`` means the distribution is created in-memory.
202 For a modern .dist-info installation on disk, this should be something
203 like ``{location}/{raw_name}-{version}.dist-info``.
205 Do not canonicalize this value with e.g. ``pathlib.Path.resolve()``. If
206 this is a symbolic link, we want to preserve the relative path between
207 it and other files in the distribution.
208 """
209 raise NotImplementedError()
211 @property
212 def installed_by_distutils(self) -> bool:
213 """Whether this distribution is installed with legacy distutils format.
215 A distribution installed with "raw" distutils not patched by setuptools
216 uses one single file at ``info_location`` to store metadata. We need to
217 treat this specially on uninstallation.
218 """
219 info_location = self.info_location
220 if not info_location:
221 return False
222 return pathlib.Path(info_location).is_file()
224 @property
225 def installed_as_egg(self) -> bool:
226 """Whether this distribution is installed as an egg.
228 This usually indicates the distribution was installed by (older versions
229 of) easy_install.
230 """
231 location = self.location
232 if not location:
233 return False
234 # XXX if the distribution is a zipped egg, location has a trailing /
235 # so we resort to pathlib.Path to check the suffix in a reliable way.
236 return pathlib.Path(location).suffix == ".egg"
238 @property
239 def installed_with_setuptools_egg_info(self) -> bool:
240 """Whether this distribution is installed with the ``.egg-info`` format.
242 This usually indicates the distribution was installed with setuptools
243 with an old pip version or with ``single-version-externally-managed``.
245 Note that this ensure the metadata store is a directory. distutils can
246 also installs an ``.egg-info``, but as a file, not a directory. This
247 property is *False* for that case. Also see ``installed_by_distutils``.
248 """
249 info_location = self.info_location
250 if not info_location:
251 return False
252 if not info_location.endswith(".egg-info"):
253 return False
254 return pathlib.Path(info_location).is_dir()
256 @property
257 def installed_with_dist_info(self) -> bool:
258 """Whether this distribution is installed with the "modern format".
260 This indicates a "modern" installation, e.g. storing metadata in the
261 ``.dist-info`` directory. This applies to installations made by
262 setuptools (but through pip, not directly), or anything using the
263 standardized build backend interface (PEP 517).
264 """
265 info_location = self.info_location
266 if not info_location:
267 return False
268 if not info_location.endswith(".dist-info"):
269 return False
270 return pathlib.Path(info_location).is_dir()
272 @property
273 def canonical_name(self) -> NormalizedName:
274 raise NotImplementedError()
276 @property
277 def version(self) -> Version:
278 raise NotImplementedError()
280 @property
281 def raw_version(self) -> str:
282 raise NotImplementedError()
284 @property
285 def setuptools_filename(self) -> str:
286 """Convert a project name to its setuptools-compatible filename.
288 This is a copy of ``pkg_resources.to_filename()`` for compatibility.
289 """
290 return self.raw_name.replace("-", "_")
292 @property
293 def direct_url(self) -> DirectUrl | None:
294 """Obtain a DirectUrl from this distribution.
296 Returns None if the distribution has no `direct_url.json` metadata,
297 or if `direct_url.json` is invalid.
298 """
299 try:
300 content = self.read_text(DIRECT_URL_METADATA_NAME)
301 except FileNotFoundError:
302 return None
303 try:
304 return DirectUrl.from_json(content)
305 except (
306 UnicodeDecodeError,
307 json.JSONDecodeError,
308 DirectUrlValidationError,
309 ) as e:
310 logger.warning(
311 "Error parsing %s for %s: %s",
312 DIRECT_URL_METADATA_NAME,
313 self.canonical_name,
314 e,
315 )
316 return None
318 @property
319 def installer(self) -> str:
320 try:
321 installer_text = self.read_text("INSTALLER")
322 except (OSError, ValueError, NoneMetadataError):
323 return "" # Fail silently if the installer file cannot be read.
324 for line in installer_text.splitlines():
325 cleaned_line = line.strip()
326 if cleaned_line:
327 return cleaned_line
328 return ""
330 @property
331 def requested(self) -> bool:
332 return self.is_file("REQUESTED")
334 @property
335 def editable(self) -> bool:
336 return bool(self.editable_project_location)
338 @property
339 def local(self) -> bool:
340 """If distribution is installed in the current virtual environment.
342 Always True if we're not in a virtualenv.
343 """
344 if self.installed_location is None:
345 return False
346 return is_local(self.installed_location)
348 @property
349 def in_usersite(self) -> bool:
350 if self.installed_location is None or user_site is None:
351 return False
352 return self.installed_location.startswith(normalize_path(user_site))
354 @property
355 def in_site_packages(self) -> bool:
356 if self.installed_location is None or site_packages is None:
357 return False
358 return self.installed_location.startswith(normalize_path(site_packages))
360 def is_file(self, path: InfoPath) -> bool:
361 """Check whether an entry in the info directory is a file."""
362 raise NotImplementedError()
364 def iter_distutils_script_names(self) -> Iterator[str]:
365 """Find distutils 'scripts' entries metadata.
367 If 'scripts' is supplied in ``setup.py``, distutils records those in the
368 installed distribution's ``scripts`` directory, a file for each script.
369 """
370 raise NotImplementedError()
372 def read_text(self, path: InfoPath) -> str:
373 """Read a file in the info directory.
375 :raise FileNotFoundError: If ``path`` does not exist in the directory.
376 :raise NoneMetadataError: If ``path`` exists in the info directory, but
377 cannot be read.
378 """
379 raise NotImplementedError()
381 def iter_entry_points(self) -> Iterable[BaseEntryPoint]:
382 raise NotImplementedError()
384 def _metadata_impl(self) -> email.message.Message:
385 raise NotImplementedError()
387 @functools.cached_property
388 def metadata(self) -> email.message.Message:
389 """Metadata of distribution parsed from e.g. METADATA or PKG-INFO.
391 This should return an empty message if the metadata file is unavailable.
393 :raises NoneMetadataError: If the metadata file is available, but does
394 not contain valid metadata.
395 """
396 metadata = self._metadata_impl()
397 self._add_egg_info_requires(metadata)
398 return metadata
400 @property
401 def metadata_dict(self) -> dict[str, Any]:
402 """PEP 566 compliant JSON-serializable representation of METADATA or PKG-INFO.
404 This should return an empty dict if the metadata file is unavailable.
406 :raises NoneMetadataError: If the metadata file is available, but does
407 not contain valid metadata.
408 """
409 return msg_to_json(self.metadata)
411 @property
412 def metadata_version(self) -> str | None:
413 """Value of "Metadata-Version:" in distribution metadata, if available."""
414 return self.metadata.get("Metadata-Version")
416 @property
417 def raw_name(self) -> str:
418 """Value of "Name:" in distribution metadata."""
419 # The metadata should NEVER be missing the Name: key, but if it somehow
420 # does, fall back to the known canonical name.
421 return self.metadata.get("Name", self.canonical_name)
423 @property
424 def requires_python(self) -> SpecifierSet:
425 """Value of "Requires-Python:" in distribution metadata.
427 If the key does not exist or contains an invalid value, an empty
428 SpecifierSet should be returned.
429 """
430 value = self.metadata.get("Requires-Python")
431 if value is None:
432 return SpecifierSet()
433 try:
434 # Convert to str to satisfy the type checker; this can be a Header object.
435 spec = SpecifierSet(str(value))
436 except InvalidSpecifier as e:
437 message = "Package %r has an invalid Requires-Python: %s"
438 logger.warning(message, self.raw_name, e)
439 return SpecifierSet()
440 return spec
442 def iter_dependencies(self, extras: Collection[str] = ()) -> Iterable[Requirement]:
443 """Dependencies of this distribution.
445 For modern .dist-info distributions, this is the collection of
446 "Requires-Dist:" entries in distribution metadata.
447 """
448 raise NotImplementedError()
450 def iter_raw_dependencies(self) -> Iterable[str]:
451 """Raw Requires-Dist metadata."""
452 return self.metadata.get_all("Requires-Dist", [])
454 def iter_provided_extras(self) -> Iterable[NormalizedName]:
455 """Extras provided by this distribution.
457 For modern .dist-info distributions, this is the collection of
458 "Provides-Extra:" entries in distribution metadata.
460 The return value of this function is expected to be normalised names,
461 per PEP 685, with the returned value being handled appropriately by
462 `iter_dependencies`.
463 """
464 raise NotImplementedError()
466 def _iter_declared_entries_from_record(self) -> Iterator[str] | None:
467 try:
468 text = self.read_text("RECORD")
469 except FileNotFoundError:
470 return None
471 # This extra Path-str cast normalizes entries.
472 return (str(pathlib.Path(row[0])) for row in csv.reader(text.splitlines()))
474 def _iter_declared_entries_from_legacy(self) -> Iterator[str] | None:
475 try:
476 text = self.read_text("installed-files.txt")
477 except FileNotFoundError:
478 return None
479 paths = (p for p in text.splitlines(keepends=False) if p)
480 root = self.location
481 info = self.info_location
482 if root is None or info is None:
483 return paths
484 try:
485 info_rel = pathlib.Path(info).relative_to(root)
486 except ValueError: # info is not relative to root.
487 return paths
488 if not info_rel.parts: # info *is* root.
489 return paths
490 return (
491 _convert_installed_files_path(pathlib.Path(p).parts, info_rel.parts)
492 for p in paths
493 )
495 def iter_declared_entries(self) -> Iterator[str] | None:
496 """Iterate through file entries declared in this distribution.
498 For modern .dist-info distributions, this is the files listed in the
499 ``RECORD`` metadata file. For legacy setuptools distributions, this
500 comes from ``installed-files.txt``, with entries normalized to be
501 compatible with the format used by ``RECORD``.
503 :return: An iterator for listed entries, or None if the distribution
504 contains neither ``RECORD`` nor ``installed-files.txt``.
505 """
506 return (
507 self._iter_declared_entries_from_record()
508 or self._iter_declared_entries_from_legacy()
509 )
511 def _iter_requires_txt_entries(self) -> Iterator[RequiresEntry]:
512 """Parse a ``requires.txt`` in an egg-info directory.
514 This is an INI-ish format where an egg-info stores dependencies. A
515 section name describes extra other environment markers, while each entry
516 is an arbitrary string (not a key-value pair) representing a dependency
517 as a requirement string (no markers).
519 There is a construct in ``importlib.metadata`` called ``Sectioned`` that
520 does mostly the same, but the format is currently considered private.
521 """
522 try:
523 content = self.read_text("requires.txt")
524 except FileNotFoundError:
525 return
526 extra = marker = "" # Section-less entries don't have markers.
527 for line in content.splitlines():
528 line = line.strip()
529 if not line or line.startswith("#"): # Comment; ignored.
530 continue
531 if line.startswith("[") and line.endswith("]"): # A section header.
532 extra, _, marker = line.strip("[]").partition(":")
533 continue
534 yield RequiresEntry(requirement=line, extra=extra, marker=marker)
536 def _iter_egg_info_extras(self) -> Iterable[str]:
537 """Get extras from the egg-info directory."""
538 known_extras = {""}
539 for entry in self._iter_requires_txt_entries():
540 extra = canonicalize_name(entry.extra)
541 if extra in known_extras:
542 continue
543 known_extras.add(extra)
544 yield extra
546 def _iter_egg_info_dependencies(self) -> Iterable[str]:
547 """Get distribution dependencies from the egg-info directory.
549 To ease parsing, this converts a legacy dependency entry into a PEP 508
550 requirement string. Like ``_iter_requires_txt_entries()``, there is code
551 in ``importlib.metadata`` that does mostly the same, but not do exactly
552 what we need.
554 Namely, ``importlib.metadata`` does not normalize the extra name before
555 putting it into the requirement string, which causes marker comparison
556 to fail because the dist-info format do normalize. This is consistent in
557 all currently available PEP 517 backends, although not standardized.
558 """
559 for entry in self._iter_requires_txt_entries():
560 extra = canonicalize_name(entry.extra)
561 if extra and entry.marker:
562 marker = f'({entry.marker}) and extra == "{extra}"'
563 elif extra:
564 marker = f'extra == "{extra}"'
565 elif entry.marker:
566 marker = entry.marker
567 else:
568 marker = ""
569 if marker:
570 yield f"{entry.requirement} ; {marker}"
571 else:
572 yield entry.requirement
574 def _add_egg_info_requires(self, metadata: email.message.Message) -> None:
575 """Add egg-info requires.txt information to the metadata."""
576 if not metadata.get_all("Requires-Dist"):
577 for dep in self._iter_egg_info_dependencies():
578 metadata["Requires-Dist"] = dep
579 if not metadata.get_all("Provides-Extra"):
580 for extra in self._iter_egg_info_extras():
581 metadata["Provides-Extra"] = extra
584class BaseEnvironment:
585 """An environment containing distributions to introspect."""
587 @classmethod
588 def default(cls) -> BaseEnvironment:
589 raise NotImplementedError()
591 @classmethod
592 def from_paths(cls, paths: list[str] | None) -> BaseEnvironment:
593 raise NotImplementedError()
595 def get_distribution(self, name: str) -> BaseDistribution | None:
596 """Given a requirement name, return the installed distributions.
598 The name may not be normalized. The implementation must canonicalize
599 it for lookup.
600 """
601 raise NotImplementedError()
603 def _iter_distributions(self) -> Iterator[BaseDistribution]:
604 """Iterate through installed distributions.
606 This function should be implemented by subclass, but never called
607 directly. Use the public ``iter_distribution()`` instead, which
608 implements additional logic to make sure the distributions are valid.
609 """
610 raise NotImplementedError()
612 def iter_all_distributions(self) -> Iterator[BaseDistribution]:
613 """Iterate through all installed distributions without any filtering."""
614 for dist in self._iter_distributions():
615 # Make sure the distribution actually comes from a valid Python
616 # packaging distribution. Pip's AdjacentTempDirectory leaves folders
617 # e.g. ``~atplotlib.dist-info`` if cleanup was interrupted. The
618 # valid project name pattern is taken from PEP 508.
619 project_name_valid = re.match(
620 r"^([A-Z0-9]|[A-Z0-9][A-Z0-9._-]*[A-Z0-9])$",
621 dist.canonical_name,
622 flags=re.IGNORECASE,
623 )
624 if not project_name_valid:
625 logger.warning(
626 "Ignoring invalid distribution %s (%s)",
627 dist.canonical_name,
628 dist.location,
629 )
630 continue
631 yield dist
633 def iter_installed_distributions(
634 self,
635 local_only: bool = True,
636 skip: Container[str] = stdlib_pkgs,
637 include_editables: bool = True,
638 editables_only: bool = False,
639 user_only: bool = False,
640 ) -> Iterator[BaseDistribution]:
641 """Return a list of installed distributions.
643 This is based on ``iter_all_distributions()`` with additional filtering
644 options. Note that ``iter_installed_distributions()`` without arguments
645 is *not* equal to ``iter_all_distributions()``, since some of the
646 configurations exclude packages by default.
648 :param local_only: If True (default), only return installations
649 local to the current virtualenv, if in a virtualenv.
650 :param skip: An iterable of canonicalized project names to ignore;
651 defaults to ``stdlib_pkgs``.
652 :param include_editables: If False, don't report editables.
653 :param editables_only: If True, only report editables.
654 :param user_only: If True, only report installations in the user
655 site directory.
656 """
657 it = self.iter_all_distributions()
658 if local_only:
659 it = (d for d in it if d.local)
660 if not include_editables:
661 it = (d for d in it if not d.editable)
662 if editables_only:
663 it = (d for d in it if d.editable)
664 if user_only:
665 it = (d for d in it if d.in_usersite)
666 return (d for d in it if d.canonical_name not in skip)
669class Wheel(Protocol):
670 location: str
672 def as_zipfile(self) -> zipfile.ZipFile:
673 raise NotImplementedError()
676class FilesystemWheel(Wheel):
677 def __init__(self, location: str) -> None:
678 self.location = location
680 def as_zipfile(self) -> zipfile.ZipFile:
681 return zipfile.ZipFile(self.location, allowZip64=True)
684class MemoryWheel(Wheel):
685 def __init__(self, location: str, stream: IO[bytes]) -> None:
686 self.location = location
687 self.stream = stream
689 def as_zipfile(self) -> zipfile.ZipFile:
690 return zipfile.ZipFile(self.stream, allowZip64=True)