1"""
2Load setuptools configuration from ``pyproject.toml`` files.
3
4**PRIVATE MODULE**: API reserved for setuptools internal usage only.
5
6To read project metadata, consider using
7``build.util.project_wheel_metadata`` (https://pypi.org/project/build/).
8For simple scenarios, you can also try parsing the file directly
9with the help of ``tomllib`` or ``tomli``.
10"""
11
12from __future__ import annotations
13
14import logging
15import os
16from contextlib import contextmanager
17from functools import partial
18from types import TracebackType
19from typing import TYPE_CHECKING, Any, Callable, Mapping
20
21from .._path import StrPath
22from ..errors import FileError, InvalidConfigError
23from ..warnings import SetuptoolsWarning
24from . import expand as _expand
25from ._apply_pyprojecttoml import _PREVIOUSLY_DEFINED, _MissingDynamic, apply as _apply
26
27if TYPE_CHECKING:
28 from typing_extensions import Self
29
30 from setuptools.dist import Distribution
31
32_logger = logging.getLogger(__name__)
33
34
35def load_file(filepath: StrPath) -> dict:
36 from ..compat.py310 import tomllib
37
38 with open(filepath, "rb") as file:
39 return tomllib.load(file)
40
41
42def validate(config: dict, filepath: StrPath) -> bool:
43 from . import _validate_pyproject as validator
44
45 trove_classifier = validator.FORMAT_FUNCTIONS.get("trove-classifier")
46 if hasattr(trove_classifier, "_disable_download"):
47 # Improve reproducibility by default. See issue 31 for validate-pyproject.
48 trove_classifier._disable_download() # type: ignore
49
50 try:
51 return validator.validate(config)
52 except validator.ValidationError as ex:
53 summary = f"configuration error: {ex.summary}"
54 if ex.name.strip("`") != "project":
55 # Probably it is just a field missing/misnamed, not worthy the verbosity...
56 _logger.debug(summary)
57 _logger.debug(ex.details)
58
59 error = f"invalid pyproject.toml config: {ex.name}."
60 raise ValueError(f"{error}\n{summary}") from None
61
62
63def apply_configuration(
64 dist: Distribution,
65 filepath: StrPath,
66 ignore_option_errors=False,
67) -> Distribution:
68 """Apply the configuration from a ``pyproject.toml`` file into an existing
69 distribution object.
70 """
71 config = read_configuration(filepath, True, ignore_option_errors, dist)
72 return _apply(dist, config, filepath)
73
74
75def read_configuration(
76 filepath: StrPath,
77 expand=True,
78 ignore_option_errors=False,
79 dist: Distribution | None = None,
80) -> dict[str, Any]:
81 """Read given configuration file and returns options from it as a dict.
82
83 :param str|unicode filepath: Path to configuration file in the ``pyproject.toml``
84 format.
85
86 :param bool expand: Whether to expand directives and other computed values
87 (i.e. post-process the given configuration)
88
89 :param bool ignore_option_errors: Whether to silently ignore
90 options, values of which could not be resolved (e.g. due to exceptions
91 in directives such as file:, attr:, etc.).
92 If False exceptions are propagated as expected.
93
94 :param Distribution|None: Distribution object to which the configuration refers.
95 If not given a dummy object will be created and discarded after the
96 configuration is read. This is used for auto-discovery of packages and in the
97 case a dynamic configuration (e.g. ``attr`` or ``cmdclass``) is expanded.
98 When ``expand=False`` this object is simply ignored.
99
100 :rtype: dict
101 """
102 filepath = os.path.abspath(filepath)
103
104 if not os.path.isfile(filepath):
105 raise FileError(f"Configuration file {filepath!r} does not exist.")
106
107 asdict = load_file(filepath) or {}
108 project_table = asdict.get("project", {})
109 tool_table = asdict.get("tool", {})
110 setuptools_table = tool_table.get("setuptools", {})
111 if not asdict or not (project_table or setuptools_table):
112 return {} # User is not using pyproject to configure setuptools
113
114 if "setuptools" in asdict.get("tools", {}):
115 # let the user know they probably have a typo in their metadata
116 _ToolsTypoInMetadata.emit()
117
118 if "distutils" in tool_table:
119 _ExperimentalConfiguration.emit(subject="[tool.distutils]")
120
121 # There is an overall sense in the community that making include_package_data=True
122 # the default would be an improvement.
123 # `ini2toml` backfills include_package_data=False when nothing is explicitly given,
124 # therefore setting a default here is backwards compatible.
125 if dist and dist.include_package_data is not None:
126 setuptools_table.setdefault("include-package-data", dist.include_package_data)
127 else:
128 setuptools_table.setdefault("include-package-data", True)
129 # Persist changes:
130 asdict["tool"] = tool_table
131 tool_table["setuptools"] = setuptools_table
132
133 if "ext-modules" in setuptools_table:
134 _ExperimentalConfiguration.emit(subject="[tool.setuptools.ext-modules]")
135
136 with _ignore_errors(ignore_option_errors):
137 # Don't complain about unrelated errors (e.g. tools not using the "tool" table)
138 subset = {"project": project_table, "tool": {"setuptools": setuptools_table}}
139 validate(subset, filepath)
140
141 if expand:
142 root_dir = os.path.dirname(filepath)
143 return expand_configuration(asdict, root_dir, ignore_option_errors, dist)
144
145 return asdict
146
147
148def expand_configuration(
149 config: dict,
150 root_dir: StrPath | None = None,
151 ignore_option_errors: bool = False,
152 dist: Distribution | None = None,
153) -> dict:
154 """Given a configuration with unresolved fields (e.g. dynamic, cmdclass, ...)
155 find their final values.
156
157 :param dict config: Dict containing the configuration for the distribution
158 :param str root_dir: Top-level directory for the distribution/project
159 (the same directory where ``pyproject.toml`` is place)
160 :param bool ignore_option_errors: see :func:`read_configuration`
161 :param Distribution|None: Distribution object to which the configuration refers.
162 If not given a dummy object will be created and discarded after the
163 configuration is read. Used in the case a dynamic configuration
164 (e.g. ``attr`` or ``cmdclass``).
165
166 :rtype: dict
167 """
168 return _ConfigExpander(config, root_dir, ignore_option_errors, dist).expand()
169
170
171class _ConfigExpander:
172 def __init__(
173 self,
174 config: dict,
175 root_dir: StrPath | None = None,
176 ignore_option_errors: bool = False,
177 dist: Distribution | None = None,
178 ):
179 self.config = config
180 self.root_dir = root_dir or os.getcwd()
181 self.project_cfg = config.get("project", {})
182 self.dynamic = self.project_cfg.get("dynamic", [])
183 self.setuptools_cfg = config.get("tool", {}).get("setuptools", {})
184 self.dynamic_cfg = self.setuptools_cfg.get("dynamic", {})
185 self.ignore_option_errors = ignore_option_errors
186 self._dist = dist
187 self._referenced_files: set[str] = set()
188
189 def _ensure_dist(self) -> Distribution:
190 from setuptools.dist import Distribution
191
192 attrs = {"src_root": self.root_dir, "name": self.project_cfg.get("name", None)}
193 return self._dist or Distribution(attrs)
194
195 def _process_field(self, container: dict, field: str, fn: Callable):
196 if field in container:
197 with _ignore_errors(self.ignore_option_errors):
198 container[field] = fn(container[field])
199
200 def _canonic_package_data(self, field="package-data"):
201 package_data = self.setuptools_cfg.get(field, {})
202 return _expand.canonic_package_data(package_data)
203
204 def expand(self):
205 self._expand_packages()
206 self._canonic_package_data()
207 self._canonic_package_data("exclude-package-data")
208
209 # A distribution object is required for discovering the correct package_dir
210 dist = self._ensure_dist()
211 ctx = _EnsurePackagesDiscovered(dist, self.project_cfg, self.setuptools_cfg)
212 with ctx as ensure_discovered:
213 package_dir = ensure_discovered.package_dir
214 self._expand_data_files()
215 self._expand_cmdclass(package_dir)
216 self._expand_all_dynamic(dist, package_dir)
217
218 dist._referenced_files.update(self._referenced_files)
219 return self.config
220
221 def _expand_packages(self):
222 packages = self.setuptools_cfg.get("packages")
223 if packages is None or isinstance(packages, (list, tuple)):
224 return
225
226 find = packages.get("find")
227 if isinstance(find, dict):
228 find["root_dir"] = self.root_dir
229 find["fill_package_dir"] = self.setuptools_cfg.setdefault("package-dir", {})
230 with _ignore_errors(self.ignore_option_errors):
231 self.setuptools_cfg["packages"] = _expand.find_packages(**find)
232
233 def _expand_data_files(self):
234 data_files = partial(_expand.canonic_data_files, root_dir=self.root_dir)
235 self._process_field(self.setuptools_cfg, "data-files", data_files)
236
237 def _expand_cmdclass(self, package_dir: Mapping[str, str]):
238 root_dir = self.root_dir
239 cmdclass = partial(_expand.cmdclass, package_dir=package_dir, root_dir=root_dir)
240 self._process_field(self.setuptools_cfg, "cmdclass", cmdclass)
241
242 def _expand_all_dynamic(self, dist: Distribution, package_dir: Mapping[str, str]):
243 special = ( # need special handling
244 "version",
245 "readme",
246 "entry-points",
247 "scripts",
248 "gui-scripts",
249 "classifiers",
250 "dependencies",
251 "optional-dependencies",
252 )
253 # `_obtain` functions are assumed to raise appropriate exceptions/warnings.
254 obtained_dynamic = {
255 field: self._obtain(dist, field, package_dir)
256 for field in self.dynamic
257 if field not in special
258 }
259 obtained_dynamic.update(
260 self._obtain_entry_points(dist, package_dir) or {},
261 version=self._obtain_version(dist, package_dir),
262 readme=self._obtain_readme(dist),
263 classifiers=self._obtain_classifiers(dist),
264 dependencies=self._obtain_dependencies(dist),
265 optional_dependencies=self._obtain_optional_dependencies(dist),
266 )
267 # `None` indicates there is nothing in `tool.setuptools.dynamic` but the value
268 # might have already been set by setup.py/extensions, so avoid overwriting.
269 updates = {k: v for k, v in obtained_dynamic.items() if v is not None}
270 self.project_cfg.update(updates)
271
272 def _ensure_previously_set(self, dist: Distribution, field: str):
273 previous = _PREVIOUSLY_DEFINED[field](dist)
274 if previous is None and not self.ignore_option_errors:
275 msg = (
276 f"No configuration found for dynamic {field!r}.\n"
277 "Some dynamic fields need to be specified via `tool.setuptools.dynamic`"
278 "\nothers must be specified via the equivalent attribute in `setup.py`."
279 )
280 raise InvalidConfigError(msg)
281
282 def _expand_directive(
283 self, specifier: str, directive, package_dir: Mapping[str, str]
284 ):
285 from more_itertools import always_iterable
286
287 with _ignore_errors(self.ignore_option_errors):
288 root_dir = self.root_dir
289 if "file" in directive:
290 self._referenced_files.update(always_iterable(directive["file"]))
291 return _expand.read_files(directive["file"], root_dir)
292 if "attr" in directive:
293 return _expand.read_attr(directive["attr"], package_dir, root_dir)
294 raise ValueError(f"invalid `{specifier}`: {directive!r}")
295 return None
296
297 def _obtain(self, dist: Distribution, field: str, package_dir: Mapping[str, str]):
298 if field in self.dynamic_cfg:
299 return self._expand_directive(
300 f"tool.setuptools.dynamic.{field}",
301 self.dynamic_cfg[field],
302 package_dir,
303 )
304 self._ensure_previously_set(dist, field)
305 return None
306
307 def _obtain_version(self, dist: Distribution, package_dir: Mapping[str, str]):
308 # Since plugins can set version, let's silently skip if it cannot be obtained
309 if "version" in self.dynamic and "version" in self.dynamic_cfg:
310 return _expand.version(
311 # We already do an early check for the presence of "version"
312 self._obtain(dist, "version", package_dir) # pyright: ignore[reportArgumentType]
313 )
314 return None
315
316 def _obtain_readme(self, dist: Distribution) -> dict[str, str] | None:
317 if "readme" not in self.dynamic:
318 return None
319
320 dynamic_cfg = self.dynamic_cfg
321 if "readme" in dynamic_cfg:
322 return {
323 # We already do an early check for the presence of "readme"
324 "text": self._obtain(dist, "readme", {}),
325 "content-type": dynamic_cfg["readme"].get("content-type", "text/x-rst"),
326 } # pyright: ignore[reportReturnType]
327
328 self._ensure_previously_set(dist, "readme")
329 return None
330
331 def _obtain_entry_points(
332 self, dist: Distribution, package_dir: Mapping[str, str]
333 ) -> dict[str, dict] | None:
334 fields = ("entry-points", "scripts", "gui-scripts")
335 if not any(field in self.dynamic for field in fields):
336 return None
337
338 text = self._obtain(dist, "entry-points", package_dir)
339 if text is None:
340 return None
341
342 groups = _expand.entry_points(text)
343 expanded = {"entry-points": groups}
344
345 def _set_scripts(field: str, group: str):
346 if group in groups:
347 value = groups.pop(group)
348 if field not in self.dynamic:
349 raise InvalidConfigError(_MissingDynamic.details(field, value))
350 expanded[field] = value
351
352 _set_scripts("scripts", "console_scripts")
353 _set_scripts("gui-scripts", "gui_scripts")
354
355 return expanded
356
357 def _obtain_classifiers(self, dist: Distribution):
358 if "classifiers" in self.dynamic:
359 value = self._obtain(dist, "classifiers", {})
360 if value:
361 return value.splitlines()
362 return None
363
364 def _obtain_dependencies(self, dist: Distribution):
365 if "dependencies" in self.dynamic:
366 value = self._obtain(dist, "dependencies", {})
367 if value:
368 return _parse_requirements_list(value)
369 return None
370
371 def _obtain_optional_dependencies(self, dist: Distribution):
372 if "optional-dependencies" not in self.dynamic:
373 return None
374 if "optional-dependencies" in self.dynamic_cfg:
375 optional_dependencies_map = self.dynamic_cfg["optional-dependencies"]
376 assert isinstance(optional_dependencies_map, dict)
377 return {
378 group: _parse_requirements_list(
379 self._expand_directive(
380 f"tool.setuptools.dynamic.optional-dependencies.{group}",
381 directive,
382 {},
383 )
384 )
385 for group, directive in optional_dependencies_map.items()
386 }
387 self._ensure_previously_set(dist, "optional-dependencies")
388 return None
389
390
391def _parse_requirements_list(value):
392 return [
393 line
394 for line in value.splitlines()
395 if line.strip() and not line.strip().startswith("#")
396 ]
397
398
399@contextmanager
400def _ignore_errors(ignore_option_errors: bool):
401 if not ignore_option_errors:
402 yield
403 return
404
405 try:
406 yield
407 except Exception as ex:
408 _logger.debug(f"ignored error: {ex.__class__.__name__} - {ex}")
409
410
411class _EnsurePackagesDiscovered(_expand.EnsurePackagesDiscovered):
412 def __init__(
413 self, distribution: Distribution, project_cfg: dict, setuptools_cfg: dict
414 ):
415 super().__init__(distribution)
416 self._project_cfg = project_cfg
417 self._setuptools_cfg = setuptools_cfg
418
419 def __enter__(self) -> Self:
420 """When entering the context, the values of ``packages``, ``py_modules`` and
421 ``package_dir`` that are missing in ``dist`` are copied from ``setuptools_cfg``.
422 """
423 dist, cfg = self._dist, self._setuptools_cfg
424 package_dir: dict[str, str] = cfg.setdefault("package-dir", {})
425 package_dir.update(dist.package_dir or {})
426 dist.package_dir = package_dir # needs to be the same object
427
428 dist.set_defaults._ignore_ext_modules() # pyproject.toml-specific behaviour
429
430 # Set `name`, `py_modules` and `packages` in dist to short-circuit
431 # auto-discovery, but avoid overwriting empty lists purposefully set by users.
432 if dist.metadata.name is None:
433 dist.metadata.name = self._project_cfg.get("name")
434 if dist.py_modules is None:
435 dist.py_modules = cfg.get("py-modules")
436 if dist.packages is None:
437 dist.packages = cfg.get("packages")
438
439 return super().__enter__()
440
441 def __exit__(
442 self,
443 exc_type: type[BaseException] | None,
444 exc_value: BaseException | None,
445 traceback: TracebackType | None,
446 ) -> None:
447 """When exiting the context, if values of ``packages``, ``py_modules`` and
448 ``package_dir`` are missing in ``setuptools_cfg``, copy from ``dist``.
449 """
450 # If anything was discovered set them back, so they count in the final config.
451 self._setuptools_cfg.setdefault("packages", self._dist.packages)
452 self._setuptools_cfg.setdefault("py-modules", self._dist.py_modules)
453 return super().__exit__(exc_type, exc_value, traceback)
454
455
456class _ExperimentalConfiguration(SetuptoolsWarning):
457 _SUMMARY = (
458 "`{subject}` in `pyproject.toml` is still *experimental* "
459 "and likely to change in future releases."
460 )
461
462
463class _ToolsTypoInMetadata(SetuptoolsWarning):
464 _SUMMARY = (
465 "Ignoring [tools.setuptools] in pyproject.toml, did you mean [tool.setuptools]?"
466 )