Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""Build Environment used for isolation during sdist building"""
3from __future__ import annotations
5import logging
6import os
7import pathlib
8import site
9import sys
10import textwrap
11from collections import OrderedDict
12from collections.abc import Iterable, Sequence
13from contextlib import AbstractContextManager as ContextManager
14from contextlib import nullcontext
15from io import StringIO
16from types import TracebackType
17from typing import TYPE_CHECKING, Protocol, TypedDict
19from pip._vendor.packaging.version import Version
21from pip import __file__ as pip_location
22from pip._internal.cli.spinners import open_rich_spinner, open_spinner
23from pip._internal.exceptions import (
24 BuildDependencyInstallError,
25 DiagnosticPipError,
26 InstallWheelBuildError,
27 PipError,
28)
29from pip._internal.locations import get_platlib, get_purelib, get_scheme
30from pip._internal.metadata import get_default_environment, get_environment
31from pip._internal.utils.deprecation import deprecated
32from pip._internal.utils.logging import VERBOSE, capture_logging
33from pip._internal.utils.packaging import get_requirement
34from pip._internal.utils.subprocess import call_subprocess
35from pip._internal.utils.temp_dir import TempDirectory, tempdir_kinds
37if TYPE_CHECKING:
38 from pip._internal.cache import WheelCache
39 from pip._internal.index.package_finder import PackageFinder
40 from pip._internal.operations.build.build_tracker import BuildTracker
41 from pip._internal.req.req_install import InstallRequirement
42 from pip._internal.resolution.base import BaseResolver
44 class ExtraEnviron(TypedDict, total=False):
45 extra_environ: dict[str, str]
48logger = logging.getLogger(__name__)
51def _dedup(a: str, b: str) -> tuple[str] | tuple[str, str]:
52 return (a, b) if a != b else (a,)
55class _Prefix:
56 def __init__(self, path: str) -> None:
57 self.path = path
58 self.setup = False
59 scheme = get_scheme("", prefix=path)
60 self.bin_dir = scheme.scripts
61 self.lib_dirs = _dedup(scheme.purelib, scheme.platlib)
64def get_runnable_pip() -> str:
65 """Get a file to pass to a Python executable, to run the currently-running pip.
67 This is used to run a pip subprocess, for installing requirements into the build
68 environment.
69 """
70 source = pathlib.Path(pip_location).resolve().parent
72 if not source.is_dir():
73 # This would happen if someone is using pip from inside a zip file. In that
74 # case, we can use that directly.
75 return str(source)
77 return os.fsdecode(source / "__pip-runner__.py")
80def _get_system_sitepackages() -> set[str]:
81 """Get system site packages
83 Usually from site.getsitepackages,
84 but fallback on `get_purelib()/get_platlib()` if unavailable
85 (e.g. in a virtualenv created by virtualenv<20)
87 Returns normalized set of strings.
88 """
89 if hasattr(site, "getsitepackages"):
90 system_sites = site.getsitepackages()
91 else:
92 # virtualenv < 20 overwrites site.py without getsitepackages
93 # fallback on get_purelib/get_platlib.
94 # this is known to miss things, but shouldn't in the cases
95 # where getsitepackages() has been removed (inside a virtualenv)
96 system_sites = [get_purelib(), get_platlib()]
97 return {os.path.normcase(path) for path in system_sites}
100class BuildEnvironmentInstaller(Protocol):
101 """
102 Interface for installing build dependencies into an isolated build
103 environment.
104 """
106 def install(
107 self,
108 requirements: Iterable[str],
109 prefix: _Prefix,
110 *,
111 kind: str,
112 for_req: InstallRequirement | None,
113 ) -> None: ...
116class SubprocessBuildEnvironmentInstaller:
117 """
118 Install build dependencies by calling pip in a subprocess.
119 """
121 def __init__(
122 self,
123 finder: PackageFinder,
124 build_constraints: list[str] | None = None,
125 build_constraint_feature_enabled: bool = False,
126 ) -> None:
127 self.finder = finder
128 self._build_constraints = build_constraints or []
129 self._build_constraint_feature_enabled = build_constraint_feature_enabled
131 def _deprecation_constraint_check(self) -> None:
132 """
133 Check for deprecation warning: PIP_CONSTRAINT affecting build environments.
135 This warns when build-constraint feature is NOT enabled and PIP_CONSTRAINT
136 is not empty.
137 """
138 if self._build_constraint_feature_enabled or self._build_constraints:
139 return
141 pip_constraint = os.environ.get("PIP_CONSTRAINT")
142 if not pip_constraint or not pip_constraint.strip():
143 return
145 deprecated(
146 reason=(
147 "Setting PIP_CONSTRAINT will not affect "
148 "build constraints in the future,"
149 ),
150 replacement=(
151 "to specify build constraints using --build-constraint or "
152 "PIP_BUILD_CONSTRAINT. To disable this warning without "
153 "any build constraints set --use-feature=build-constraint or "
154 'PIP_USE_FEATURE="build-constraint"'
155 ),
156 gone_in="26.2",
157 issue=None,
158 )
160 def install(
161 self,
162 requirements: Iterable[str],
163 prefix: _Prefix,
164 *,
165 kind: str,
166 for_req: InstallRequirement | None,
167 ) -> None:
168 self._deprecation_constraint_check()
170 finder = self.finder
171 args: list[str] = [
172 sys.executable,
173 get_runnable_pip(),
174 "install",
175 "--ignore-installed",
176 "--no-user",
177 "--prefix",
178 prefix.path,
179 "--no-warn-script-location",
180 "--disable-pip-version-check",
181 # As the build environment is ephemeral, it's wasteful to
182 # pre-compile everything, especially as not every Python
183 # module will be used/compiled in most cases.
184 "--no-compile",
185 # The prefix specified two lines above, thus
186 # target from config file or env var should be ignored
187 "--target",
188 "",
189 ]
190 if logger.getEffectiveLevel() <= logging.DEBUG:
191 args.append("-vv")
192 elif logger.getEffectiveLevel() <= VERBOSE:
193 args.append("-v")
194 for format_control in ("no_binary", "only_binary"):
195 formats = getattr(finder.format_control, format_control)
196 args.extend(
197 (
198 "--" + format_control.replace("_", "-"),
199 ",".join(sorted(formats or {":none:"})),
200 )
201 )
203 if finder.release_control is not None:
204 # Use ordered args to preserve the user's original command-line order
205 # This is important because later flags can override earlier ones
206 for attr_name, value in finder.release_control.get_ordered_args():
207 args.extend(("--" + attr_name.replace("_", "-"), value))
209 index_urls = finder.index_urls
210 if index_urls:
211 args.extend(["-i", index_urls[0]])
212 for extra_index in index_urls[1:]:
213 args.extend(["--extra-index-url", extra_index])
214 else:
215 args.append("--no-index")
216 for link in finder.find_links:
217 args.extend(["--find-links", link])
219 if finder.proxy:
220 args.extend(["--proxy", finder.proxy])
221 for host in finder.trusted_hosts:
222 args.extend(["--trusted-host", host])
223 if finder.custom_cert:
224 args.extend(["--cert", finder.custom_cert])
225 if finder.client_cert:
226 args.extend(["--client-cert", finder.client_cert])
227 if finder.prefer_binary:
228 args.append("--prefer-binary")
230 # Handle build constraints
231 if self._build_constraint_feature_enabled:
232 args.extend(["--use-feature", "build-constraint"])
234 if self._build_constraints:
235 # Build constraints must be passed as both constraints
236 # and build constraints, so that nested builds receive
237 # build constraints
238 for constraint_file in self._build_constraints:
239 args.extend(["--constraint", constraint_file])
240 args.extend(["--build-constraint", constraint_file])
242 extra_environ: ExtraEnviron = {}
243 if self._build_constraint_feature_enabled and not self._build_constraints:
244 # If there are no build constraints but the build constraints
245 # feature is enabled then we must ignore regular constraints
246 # in the isolated build environment
247 extra_environ = {"extra_environ": {"_PIP_IN_BUILD_IGNORE_CONSTRAINTS": "1"}}
249 if finder.uploaded_prior_to:
250 args.extend(["--uploaded-prior-to", finder.uploaded_prior_to.isoformat()])
251 args.append("--")
252 args.extend(requirements)
254 identify_requirement = (
255 f" for {for_req.name}" if for_req and for_req.name else ""
256 )
257 with open_spinner(f"Installing {kind}") as spinner:
258 call_subprocess(
259 args,
260 command_desc=f"installing {kind}{identify_requirement}",
261 spinner=spinner,
262 **extra_environ,
263 )
266class InprocessBuildEnvironmentInstaller:
267 """
268 Build dependency installer that runs in the same pip process.
270 This contains a stripped down version of the install command with
271 only the logic necessary for installing build dependencies. The
272 finder, session, build tracker, and wheel cache are reused, but new
273 instances of everything else are created as needed.
275 Options are inherited from the parent install command unless
276 they don't make sense for build dependencies (in which case, they
277 are hard-coded, see comments below).
278 """
280 def __init__(
281 self,
282 *,
283 finder: PackageFinder,
284 build_tracker: BuildTracker,
285 wheel_cache: WheelCache,
286 build_constraints: Sequence[InstallRequirement] = (),
287 verbosity: int = 0,
288 ) -> None:
289 from pip._internal.operations.prepare import RequirementPreparer
291 self._finder = finder
292 self._build_constraints = build_constraints
293 self._wheel_cache = wheel_cache
294 self._level = 0
296 build_dir = TempDirectory(kind="build-env-install", globally_managed=True)
297 self._preparer = RequirementPreparer(
298 build_isolation_installer=self,
299 # Inherited options or state.
300 finder=finder,
301 session=finder._link_collector.session,
302 build_dir=build_dir.path,
303 build_tracker=build_tracker,
304 verbosity=verbosity,
305 # This is irrelevant as it only applies to editable requirements.
306 src_dir="",
307 # Hard-coded options (that should NOT be inherited).
308 download_dir=None,
309 build_isolation=True,
310 check_build_deps=False,
311 progress_bar="off",
312 # TODO: hash-checking should be extended to build deps, but that is
313 # deferred for later as it'd be a breaking change.
314 require_hashes=False,
315 use_user_site=False,
316 lazy_wheel=False,
317 legacy_resolver=False,
318 )
320 def install(
321 self,
322 requirements: Iterable[str],
323 prefix: _Prefix,
324 *,
325 kind: str,
326 for_req: InstallRequirement | None,
327 ) -> None:
328 """Install entrypoint. Manages output capturing and error handling."""
329 capture_logs = not logger.isEnabledFor(VERBOSE) and self._level == 0
330 if capture_logs:
331 # Hide the logs from the installation of build dependencies.
332 # They will be shown only if an error occurs.
333 capture_ctx: ContextManager[StringIO] = capture_logging()
334 spinner: ContextManager[None] = open_rich_spinner(f"Installing {kind}")
335 else:
336 # Otherwise, pass-through all logs (with a header).
337 capture_ctx, spinner = nullcontext(StringIO()), nullcontext()
338 logger.info("Installing %s ...", kind)
340 try:
341 self._level += 1
342 with spinner, capture_ctx as stream:
343 self._install_impl(requirements, prefix)
345 except DiagnosticPipError as exc:
346 # Format similar to a nested subprocess error, where the
347 # causing error is shown first, followed by the build error.
348 logger.info(textwrap.dedent(stream.getvalue()))
349 logger.error("%s", exc, extra={"rich": True})
350 logger.info("")
351 raise BuildDependencyInstallError(
352 for_req, requirements, cause=exc, log_lines=None
353 )
355 except Exception as exc:
356 logs: list[str] | None = textwrap.dedent(stream.getvalue()).splitlines()
357 if not capture_logs:
358 # If logs aren't being captured, then display the error inline
359 # with the rest of the logs.
360 logs = None
361 if isinstance(exc, PipError):
362 logger.error("%s", exc)
363 else:
364 logger.exception("pip crashed unexpectedly")
365 raise BuildDependencyInstallError(
366 for_req, requirements, cause=exc, log_lines=logs
367 )
369 finally:
370 self._level -= 1
372 def _install_impl(self, requirements: Iterable[str], prefix: _Prefix) -> None:
373 """Core build dependency install logic."""
374 from pip._internal.commands.install import installed_packages_summary
375 from pip._internal.req import install_given_reqs
376 from pip._internal.req.constructors import install_req_from_line
377 from pip._internal.wheel_builder import build
379 ireqs = [install_req_from_line(req, user_supplied=True) for req in requirements]
380 ireqs.extend(self._build_constraints)
382 resolver = self._make_resolver()
383 resolved_set = resolver.resolve(ireqs, check_supported_wheels=True)
384 self._preparer.prepare_linked_requirements_more(
385 resolved_set.requirements.values()
386 )
388 reqs_to_build = [
389 r for r in resolved_set.requirements_to_install if not r.is_wheel
390 ]
391 _, build_failures = build(reqs_to_build, self._wheel_cache, verify=True)
392 if build_failures:
393 raise InstallWheelBuildError(build_failures)
395 installed = install_given_reqs(
396 resolver.get_installation_order(resolved_set),
397 prefix=prefix.path,
398 # Hard-coded options (that should NOT be inherited).
399 root=None,
400 home=None,
401 warn_script_location=False,
402 use_user_site=False,
403 # As the build environment is ephemeral, it's wasteful to
404 # pre-compile everything since not all modules will be used.
405 pycompile=False,
406 progress_bar="off",
407 )
409 env = get_environment(list(prefix.lib_dirs))
410 if summary := installed_packages_summary(installed, env):
411 logger.info(summary)
413 def _make_resolver(self) -> BaseResolver:
414 """Create a new resolver for one time use."""
415 # Legacy installer never used the legacy resolver so create a
416 # resolvelib resolver directly. Yuck.
417 from pip._internal.req.constructors import install_req_from_req_string
418 from pip._internal.resolution.resolvelib.resolver import Resolver
420 return Resolver(
421 make_install_req=install_req_from_req_string,
422 # Inherited state.
423 preparer=self._preparer,
424 finder=self._finder,
425 wheel_cache=self._wheel_cache,
426 # Hard-coded options (that should NOT be inherited).
427 ignore_requires_python=False,
428 use_user_site=False,
429 ignore_dependencies=False,
430 ignore_installed=True,
431 force_reinstall=False,
432 upgrade_strategy="to-satisfy-only",
433 py_version_info=None,
434 )
437class BuildEnvironment:
438 """Creates and manages an isolated environment to install build deps"""
440 def __init__(self, installer: BuildEnvironmentInstaller) -> None:
441 self.installer = installer
442 temp_dir = TempDirectory(kind=tempdir_kinds.BUILD_ENV, globally_managed=True)
444 self._prefixes = OrderedDict(
445 (name, _Prefix(os.path.join(temp_dir.path, name)))
446 for name in ("normal", "overlay")
447 )
449 self._bin_dirs: list[str] = []
450 self._lib_dirs: list[str] = []
451 for prefix in reversed(list(self._prefixes.values())):
452 self._bin_dirs.append(prefix.bin_dir)
453 self._lib_dirs.extend(prefix.lib_dirs)
455 # Customize site to:
456 # - ensure .pth files are honored
457 # - prevent access to system site packages
458 system_sites = _get_system_sitepackages()
460 self._site_dir = os.path.join(temp_dir.path, "site")
461 if not os.path.exists(self._site_dir):
462 os.mkdir(self._site_dir)
463 with open(
464 os.path.join(self._site_dir, "sitecustomize.py"), "w", encoding="utf-8"
465 ) as fp:
466 fp.write(
467 textwrap.dedent(
468 """
469 import os, site, sys
471 # First, drop system-sites related paths.
472 original_sys_path = sys.path[:]
473 known_paths = set()
474 for path in {system_sites!r}:
475 site.addsitedir(path, known_paths=known_paths)
476 system_paths = set(
477 os.path.normcase(path)
478 for path in sys.path[len(original_sys_path):]
479 )
480 original_sys_path = [
481 path for path in original_sys_path
482 if os.path.normcase(path) not in system_paths
483 ]
484 sys.path = original_sys_path
486 # Second, add lib directories.
487 # ensuring .pth file are processed.
488 for path in {lib_dirs!r}:
489 assert not path in sys.path
490 site.addsitedir(path)
491 """
492 ).format(system_sites=system_sites, lib_dirs=self._lib_dirs)
493 )
495 def __enter__(self) -> None:
496 self._save_env = {
497 name: os.environ.get(name, None)
498 for name in ("PATH", "PYTHONNOUSERSITE", "PYTHONPATH")
499 }
501 path = self._bin_dirs[:]
502 old_path = self._save_env["PATH"]
503 if old_path:
504 path.extend(old_path.split(os.pathsep))
506 pythonpath = [self._site_dir]
508 os.environ.update(
509 {
510 "PATH": os.pathsep.join(path),
511 "PYTHONNOUSERSITE": "1",
512 "PYTHONPATH": os.pathsep.join(pythonpath),
513 }
514 )
516 def __exit__(
517 self,
518 exc_type: type[BaseException] | None,
519 exc_val: BaseException | None,
520 exc_tb: TracebackType | None,
521 ) -> None:
522 for varname, old_value in self._save_env.items():
523 if old_value is None:
524 os.environ.pop(varname, None)
525 else:
526 os.environ[varname] = old_value
528 def check_requirements(
529 self, reqs: Iterable[str]
530 ) -> tuple[set[tuple[str, str]], set[str]]:
531 """Return 2 sets:
532 - conflicting requirements: set of (installed, wanted) reqs tuples
533 - missing requirements: set of reqs
534 """
535 missing = set()
536 conflicting = set()
537 if reqs:
538 env = (
539 get_environment(self._lib_dirs)
540 if hasattr(self, "_lib_dirs")
541 else get_default_environment()
542 )
543 for req_str in reqs:
544 req = get_requirement(req_str)
545 # We're explicitly evaluating with an empty extra value, since build
546 # environments are not provided any mechanism to select specific extras.
547 if req.marker is not None and not req.marker.evaluate({"extra": ""}):
548 continue
549 dist = env.get_distribution(req.name)
550 if not dist:
551 missing.add(req_str)
552 continue
553 if isinstance(dist.version, Version):
554 installed_req_str = f"{req.name}=={dist.version}"
555 else:
556 installed_req_str = f"{req.name}==={dist.version}"
557 if not req.specifier.contains(dist.version, prereleases=True):
558 conflicting.add((installed_req_str, req_str))
559 # FIXME: Consider direct URL?
560 return conflicting, missing
562 def install_requirements(
563 self,
564 requirements: Iterable[str],
565 prefix_as_string: str,
566 *,
567 kind: str,
568 for_req: InstallRequirement | None = None,
569 ) -> None:
570 prefix = self._prefixes[prefix_as_string]
571 assert not prefix.setup
572 prefix.setup = True
573 if not requirements:
574 return
575 self.installer.install(requirements, prefix, kind=kind, for_req=for_req)
578class NoOpBuildEnvironment(BuildEnvironment):
579 """A no-op drop-in replacement for BuildEnvironment"""
581 def __init__(self) -> None:
582 pass
584 def __enter__(self) -> None:
585 pass
587 def __exit__(
588 self,
589 exc_type: type[BaseException] | None,
590 exc_val: BaseException | None,
591 exc_tb: TracebackType | None,
592 ) -> None:
593 pass
595 def cleanup(self) -> None:
596 pass
598 def install_requirements(
599 self,
600 requirements: Iterable[str],
601 prefix_as_string: str,
602 *,
603 kind: str,
604 for_req: InstallRequirement | None = None,
605 ) -> None:
606 raise NotImplementedError()