Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""Build Environment used for isolation during sdist building"""
3from __future__ import annotations
5import logging
6import os
7import site
8import sys
9import textwrap
10from collections import OrderedDict
11from collections.abc import Iterable, Sequence
12from contextlib import AbstractContextManager as ContextManager
13from contextlib import nullcontext
14from io import StringIO
15from types import TracebackType
16from typing import TYPE_CHECKING, Protocol, TypedDict
18from pip._vendor.packaging.version import Version
20from pip._internal.cli.spinners import open_rich_spinner, open_spinner
21from pip._internal.exceptions import (
22 BuildDependencyInstallError,
23 DiagnosticPipError,
24 InstallWheelBuildError,
25 PipError,
26)
27from pip._internal.locations import get_platlib, get_purelib, get_scheme
28from pip._internal.metadata import get_default_environment, get_environment
29from pip._internal.utils.deprecation import deprecated
30from pip._internal.utils.logging import VERBOSE, capture_logging
31from pip._internal.utils.misc import get_runnable_pip
32from pip._internal.utils.packaging import get_requirement
33from pip._internal.utils.subprocess import call_subprocess
34from pip._internal.utils.temp_dir import TempDirectory, tempdir_kinds
36if TYPE_CHECKING:
37 from pip._internal.cache import WheelCache
38 from pip._internal.index.package_finder import PackageFinder
39 from pip._internal.operations.build.build_tracker import BuildTracker
40 from pip._internal.req.req_install import InstallRequirement
41 from pip._internal.resolution.base import BaseResolver
43 class ExtraEnviron(TypedDict, total=False):
44 extra_environ: dict[str, str]
47logger = logging.getLogger(__name__)
50def _dedup(a: str, b: str) -> tuple[str] | tuple[str, str]:
51 return (a, b) if a != b else (a,)
54class _Prefix:
55 def __init__(self, path: str) -> None:
56 self.path = path
57 self.setup = False
58 scheme = get_scheme("", prefix=path)
59 self.bin_dir = scheme.scripts
60 self.lib_dirs = _dedup(scheme.purelib, scheme.platlib)
63def _get_system_sitepackages() -> set[str]:
64 """Get system site packages
66 Usually from site.getsitepackages,
67 but fallback on `get_purelib()/get_platlib()` if unavailable
68 (e.g. in a virtualenv created by virtualenv<20)
70 Returns normalized set of strings.
71 """
72 if hasattr(site, "getsitepackages"):
73 system_sites = site.getsitepackages()
74 else:
75 # virtualenv < 20 overwrites site.py without getsitepackages
76 # fallback on get_purelib/get_platlib.
77 # this is known to miss things, but shouldn't in the cases
78 # where getsitepackages() has been removed (inside a virtualenv)
79 system_sites = [get_purelib(), get_platlib()]
80 return {os.path.normcase(path) for path in system_sites}
83class BuildEnvironmentInstaller(Protocol):
84 """
85 Interface for installing build dependencies into an isolated build
86 environment.
87 """
89 def install(
90 self,
91 requirements: Iterable[str],
92 prefix: _Prefix,
93 *,
94 kind: str,
95 for_req: InstallRequirement | None,
96 ) -> None: ...
99class SubprocessBuildEnvironmentInstaller:
100 """
101 Install build dependencies by calling pip in a subprocess.
102 """
104 def __init__(
105 self,
106 finder: PackageFinder,
107 build_constraints: list[str] | None = None,
108 build_constraint_feature_enabled: bool = False,
109 ) -> None:
110 self.finder = finder
111 self._build_constraints = build_constraints or []
112 self._build_constraint_feature_enabled = build_constraint_feature_enabled
114 def _deprecation_constraint_check(self) -> None:
115 """
116 Check for deprecation warning: PIP_CONSTRAINT affecting build environments.
118 This warns when build-constraint feature is NOT enabled and PIP_CONSTRAINT
119 is not empty.
120 """
121 if self._build_constraint_feature_enabled or self._build_constraints:
122 return
124 pip_constraint = os.environ.get("PIP_CONSTRAINT")
125 if not pip_constraint or not pip_constraint.strip():
126 return
128 deprecated(
129 reason=(
130 "Setting PIP_CONSTRAINT will not affect "
131 "build constraints in the future,"
132 ),
133 replacement=(
134 "to specify build constraints using --build-constraint or "
135 "PIP_BUILD_CONSTRAINT. To disable this warning without "
136 "any build constraints set --use-feature=build-constraint or "
137 'PIP_USE_FEATURE="build-constraint"'
138 ),
139 gone_in="26.2",
140 issue=None,
141 )
143 def install(
144 self,
145 requirements: Iterable[str],
146 prefix: _Prefix,
147 *,
148 kind: str,
149 for_req: InstallRequirement | None,
150 ) -> None:
151 self._deprecation_constraint_check()
153 finder = self.finder
154 args: list[str] = [
155 sys.executable,
156 get_runnable_pip(),
157 "install",
158 "--ignore-installed",
159 "--no-user",
160 "--prefix",
161 prefix.path,
162 "--no-warn-script-location",
163 "--disable-pip-version-check",
164 # As the build environment is ephemeral, it's wasteful to
165 # pre-compile everything, especially as not every Python
166 # module will be used/compiled in most cases.
167 "--no-compile",
168 # The prefix specified two lines above, thus
169 # target from config file or env var should be ignored
170 "--target",
171 "",
172 ]
173 if logger.getEffectiveLevel() <= logging.DEBUG:
174 args.append("-vv")
175 elif logger.getEffectiveLevel() <= VERBOSE:
176 args.append("-v")
177 for format_control in ("no_binary", "only_binary"):
178 formats = getattr(finder.format_control, format_control)
179 args.extend(
180 (
181 "--" + format_control.replace("_", "-"),
182 ",".join(sorted(formats or {":none:"})),
183 )
184 )
186 if finder.release_control is not None:
187 # Use ordered args to preserve the user's original command-line order
188 # This is important because later flags can override earlier ones
189 for attr_name, value in finder.release_control.get_ordered_args():
190 args.extend(("--" + attr_name.replace("_", "-"), value))
192 index_urls = finder.index_urls
193 if index_urls:
194 args.extend(["-i", index_urls[0]])
195 for extra_index in index_urls[1:]:
196 args.extend(["--extra-index-url", extra_index])
197 else:
198 args.append("--no-index")
199 for link in finder.find_links:
200 args.extend(["--find-links", link])
202 if finder.proxy:
203 args.extend(["--proxy", finder.proxy])
204 for host in finder.trusted_hosts:
205 args.extend(["--trusted-host", host])
206 if finder.custom_cert:
207 args.extend(["--cert", finder.custom_cert])
208 if finder.client_cert:
209 args.extend(["--client-cert", finder.client_cert])
210 if finder.prefer_binary:
211 args.append("--prefer-binary")
213 # Handle build constraints
214 if self._build_constraint_feature_enabled:
215 args.extend(["--use-feature", "build-constraint"])
217 if self._build_constraints:
218 # Build constraints must be passed as both constraints
219 # and build constraints, so that nested builds receive
220 # build constraints
221 for constraint_file in self._build_constraints:
222 args.extend(["--constraint", constraint_file])
223 args.extend(["--build-constraint", constraint_file])
225 extra_environ: ExtraEnviron = {}
226 if self._build_constraint_feature_enabled and not self._build_constraints:
227 # If there are no build constraints but the build constraints
228 # feature is enabled then we must ignore regular constraints
229 # in the isolated build environment
230 extra_environ = {"extra_environ": {"_PIP_IN_BUILD_IGNORE_CONSTRAINTS": "1"}}
232 if finder.uploaded_prior_to:
233 args.extend(["--uploaded-prior-to", finder.uploaded_prior_to.isoformat()])
234 args.append("--")
235 args.extend(requirements)
237 identify_requirement = (
238 f" for {for_req.name}" if for_req and for_req.name else ""
239 )
240 with open_spinner(f"Installing {kind}") as spinner:
241 call_subprocess(
242 args,
243 command_desc=f"installing {kind}{identify_requirement}",
244 spinner=spinner,
245 **extra_environ,
246 )
249class InprocessBuildEnvironmentInstaller:
250 """
251 Build dependency installer that runs in the same pip process.
253 This contains a stripped down version of the install command with
254 only the logic necessary for installing build dependencies. The
255 finder, session, build tracker, and wheel cache are reused, but new
256 instances of everything else are created as needed.
258 Options are inherited from the parent install command unless
259 they don't make sense for build dependencies (in which case, they
260 are hard-coded, see comments below).
261 """
263 def __init__(
264 self,
265 *,
266 finder: PackageFinder,
267 build_tracker: BuildTracker,
268 wheel_cache: WheelCache,
269 build_constraints: Sequence[InstallRequirement] = (),
270 verbosity: int = 0,
271 ) -> None:
272 from pip._internal.operations.prepare import RequirementPreparer
274 self._finder = finder
275 self._build_constraints = build_constraints
276 self._wheel_cache = wheel_cache
277 self._level = 0
279 build_dir = TempDirectory(kind="build-env-install", globally_managed=True)
280 self._preparer = RequirementPreparer(
281 build_isolation_installer=self,
282 # Inherited options or state.
283 finder=finder,
284 session=finder._link_collector.session,
285 build_dir=build_dir.path,
286 build_tracker=build_tracker,
287 verbosity=verbosity,
288 # This is irrelevant as it only applies to editable requirements.
289 src_dir="",
290 # Hard-coded options (that should NOT be inherited).
291 download_dir=None,
292 build_isolation=True,
293 check_build_deps=False,
294 progress_bar="off",
295 # TODO: hash-checking should be extended to build deps, but that is
296 # deferred for later as it'd be a breaking change.
297 require_hashes=False,
298 use_user_site=False,
299 lazy_wheel=False,
300 legacy_resolver=False,
301 )
303 def install(
304 self,
305 requirements: Iterable[str],
306 prefix: _Prefix,
307 *,
308 kind: str,
309 for_req: InstallRequirement | None,
310 ) -> None:
311 """Install entrypoint. Manages output capturing and error handling."""
312 capture_logs = not logger.isEnabledFor(VERBOSE) and self._level == 0
313 if capture_logs:
314 # Hide the logs from the installation of build dependencies.
315 # They will be shown only if an error occurs.
316 capture_ctx: ContextManager[StringIO] = capture_logging()
317 spinner: ContextManager[None] = open_rich_spinner(f"Installing {kind}")
318 else:
319 # Otherwise, pass-through all logs (with a header).
320 capture_ctx, spinner = nullcontext(StringIO()), nullcontext()
321 logger.info("Installing %s ...", kind)
323 try:
324 self._level += 1
325 with spinner, capture_ctx as stream:
326 self._install_impl(requirements, prefix)
328 except DiagnosticPipError as exc:
329 # Format similar to a nested subprocess error, where the
330 # causing error is shown first, followed by the build error.
331 logger.info(textwrap.dedent(stream.getvalue()))
332 logger.error("%s", exc, extra={"rich": True})
333 logger.info("")
334 raise BuildDependencyInstallError(
335 for_req, requirements, cause=exc, log_lines=None
336 )
338 except Exception as exc:
339 logs: list[str] | None = textwrap.dedent(stream.getvalue()).splitlines()
340 if not capture_logs:
341 # If logs aren't being captured, then display the error inline
342 # with the rest of the logs.
343 logs = None
344 if isinstance(exc, PipError):
345 logger.error("%s", exc)
346 else:
347 logger.exception("pip crashed unexpectedly")
348 raise BuildDependencyInstallError(
349 for_req, requirements, cause=exc, log_lines=logs
350 )
352 finally:
353 self._level -= 1
355 def _install_impl(self, requirements: Iterable[str], prefix: _Prefix) -> None:
356 """Core build dependency install logic."""
357 from pip._internal.commands.install import installed_packages_summary
358 from pip._internal.req import install_given_reqs
359 from pip._internal.req.constructors import install_req_from_line
360 from pip._internal.wheel_builder import build
362 ireqs = [install_req_from_line(req, user_supplied=True) for req in requirements]
363 ireqs.extend(self._build_constraints)
365 resolver = self._make_resolver()
366 resolved_set = resolver.resolve(ireqs, check_supported_wheels=True)
367 self._preparer.prepare_linked_requirements_more(
368 resolved_set.requirements.values()
369 )
371 reqs_to_build = [
372 r for r in resolved_set.requirements_to_install if not r.is_wheel
373 ]
374 _, build_failures = build(reqs_to_build, self._wheel_cache, verify=True)
375 if build_failures:
376 raise InstallWheelBuildError(build_failures)
378 installed = install_given_reqs(
379 resolver.get_installation_order(resolved_set),
380 prefix=prefix.path,
381 # Hard-coded options (that should NOT be inherited).
382 root=None,
383 home=None,
384 warn_script_location=False,
385 use_user_site=False,
386 # As the build environment is ephemeral, it's wasteful to
387 # pre-compile everything since not all modules will be used.
388 pycompile=False,
389 progress_bar="off",
390 )
392 env = get_environment(list(prefix.lib_dirs))
393 if summary := installed_packages_summary(installed, env):
394 logger.info(summary)
396 def _make_resolver(self) -> BaseResolver:
397 """Create a new resolver for one time use."""
398 # Legacy installer never used the legacy resolver so create a
399 # resolvelib resolver directly. Yuck.
400 from pip._internal.req.constructors import install_req_from_req_string
401 from pip._internal.resolution.resolvelib.resolver import Resolver
403 return Resolver(
404 make_install_req=install_req_from_req_string,
405 # Inherited state.
406 preparer=self._preparer,
407 finder=self._finder,
408 wheel_cache=self._wheel_cache,
409 # Hard-coded options (that should NOT be inherited).
410 ignore_requires_python=False,
411 use_user_site=False,
412 ignore_dependencies=False,
413 ignore_installed=True,
414 force_reinstall=False,
415 upgrade_strategy="to-satisfy-only",
416 py_version_info=None,
417 )
420class BuildEnvironment:
421 """Creates and manages an isolated environment to install build deps"""
423 def __init__(self, installer: BuildEnvironmentInstaller) -> None:
424 self.installer = installer
425 temp_dir = TempDirectory(kind=tempdir_kinds.BUILD_ENV, globally_managed=True)
427 self._prefixes = OrderedDict(
428 (name, _Prefix(os.path.join(temp_dir.path, name)))
429 for name in ("normal", "overlay")
430 )
432 self._bin_dirs: list[str] = []
433 self._lib_dirs: list[str] = []
434 for prefix in reversed(list(self._prefixes.values())):
435 self._bin_dirs.append(prefix.bin_dir)
436 self._lib_dirs.extend(prefix.lib_dirs)
438 # Customize site to:
439 # - ensure .pth files are honored
440 # - prevent access to system site packages
441 system_sites = _get_system_sitepackages()
443 self._site_dir = os.path.join(temp_dir.path, "site")
444 if not os.path.exists(self._site_dir):
445 os.mkdir(self._site_dir)
446 with open(
447 os.path.join(self._site_dir, "sitecustomize.py"), "w", encoding="utf-8"
448 ) as fp:
449 fp.write(textwrap.dedent("""
450 import os, site, sys
452 # First, discover all system-sites related paths.
453 original_sys_path = sys.path[:]
454 # Clear sys.path so addsitedir() will add system site paths and paths
455 # added by contained .pth files to sys.path reliably. This is necessary
456 # since Python 3.15, which notably no longer re-executes .pth files for
457 # known paths.
458 sys.path = []
459 known_paths = set()
460 for path in {system_sites!r}:
461 site.addsitedir(path, known_paths=known_paths)
462 system_paths = set(os.path.normcase(path) for path in sys.path)
464 # Drop discovered system-sites related paths.
465 original_sys_path = [
466 path for path in original_sys_path
467 if os.path.normcase(path) not in system_paths
468 ]
469 sys.path = original_sys_path
471 # Second, add lib directories.
472 # ensuring .pth file are processed.
473 for path in {lib_dirs!r}:
474 assert not path in sys.path
475 site.addsitedir(path)
476 """).format(system_sites=system_sites, lib_dirs=self._lib_dirs))
478 def __enter__(self) -> None:
479 self._save_env = {
480 name: os.environ.get(name, None)
481 for name in ("PATH", "PYTHONNOUSERSITE", "PYTHONPATH")
482 }
484 path = self._bin_dirs[:]
485 old_path = self._save_env["PATH"]
486 if old_path:
487 path.extend(old_path.split(os.pathsep))
489 pythonpath = [self._site_dir]
491 os.environ.update(
492 {
493 "PATH": os.pathsep.join(path),
494 "PYTHONNOUSERSITE": "1",
495 "PYTHONPATH": os.pathsep.join(pythonpath),
496 }
497 )
499 def __exit__(
500 self,
501 exc_type: type[BaseException] | None,
502 exc_val: BaseException | None,
503 exc_tb: TracebackType | None,
504 ) -> None:
505 for varname, old_value in self._save_env.items():
506 if old_value is None:
507 os.environ.pop(varname, None)
508 else:
509 os.environ[varname] = old_value
511 def check_requirements(
512 self, reqs: Iterable[str]
513 ) -> tuple[set[tuple[str, str]], set[str]]:
514 """Return 2 sets:
515 - conflicting requirements: set of (installed, wanted) reqs tuples
516 - missing requirements: set of reqs
517 """
518 missing = set()
519 conflicting = set()
520 if reqs:
521 env = (
522 get_environment(self._lib_dirs)
523 if hasattr(self, "_lib_dirs")
524 else get_default_environment()
525 )
526 for req_str in reqs:
527 req = get_requirement(req_str)
528 # We're explicitly evaluating with an empty extra value, since build
529 # environments are not provided any mechanism to select specific extras.
530 if req.marker is not None and not req.marker.evaluate({"extra": ""}):
531 continue
532 dist = env.get_distribution(req.name)
533 if not dist:
534 missing.add(req_str)
535 continue
536 if isinstance(dist.version, Version):
537 installed_req_str = f"{req.name}=={dist.version}"
538 else:
539 installed_req_str = f"{req.name}==={dist.version}"
540 if not req.specifier.contains(dist.version, prereleases=True):
541 conflicting.add((installed_req_str, req_str))
542 # FIXME: Consider direct URL?
543 return conflicting, missing
545 def install_requirements(
546 self,
547 requirements: Iterable[str],
548 prefix_as_string: str,
549 *,
550 kind: str,
551 for_req: InstallRequirement | None = None,
552 ) -> None:
553 prefix = self._prefixes[prefix_as_string]
554 assert not prefix.setup
555 prefix.setup = True
556 if not requirements:
557 return
558 self.installer.install(requirements, prefix, kind=kind, for_req=for_req)
561class NoOpBuildEnvironment(BuildEnvironment):
562 """A no-op drop-in replacement for BuildEnvironment"""
564 def __init__(self) -> None:
565 pass
567 def __enter__(self) -> None:
568 pass
570 def __exit__(
571 self,
572 exc_type: type[BaseException] | None,
573 exc_val: BaseException | None,
574 exc_tb: TracebackType | None,
575 ) -> None:
576 pass
578 def cleanup(self) -> None:
579 pass
581 def install_requirements(
582 self,
583 requirements: Iterable[str],
584 prefix_as_string: str,
585 *,
586 kind: str,
587 for_req: InstallRequirement | None = None,
588 ) -> None:
589 raise NotImplementedError()