Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""Utility functions to expand configuration directives or special values
2(such glob patterns).
4We can split the process of interpreting configuration files into 2 steps:
61. The parsing the file contents from strings to value objects
7 that can be understand by Python (for example a string with a comma
8 separated list of keywords into an actual Python list of strings).
102. The expansion (or post-processing) of these values according to the
11 semantics ``setuptools`` assign to them (for example a configuration field
12 with the ``file:`` directive should be expanded from a list of file paths to
13 a single string with the contents of those files concatenated)
15This module focus on the second step, and therefore allow sharing the expansion
16functions among several configuration file formats.
18**PRIVATE MODULE**: API reserved for setuptools internal usage only.
19"""
21from __future__ import annotations
23import ast
24import importlib
25import os
26import pathlib
27import sys
28from configparser import ConfigParser
29from glob import iglob
30from importlib.machinery import ModuleSpec, all_suffixes
31from itertools import chain
32from pathlib import Path
33from types import ModuleType, TracebackType
34from typing import TYPE_CHECKING, Any, Callable, Iterable, Iterator, Mapping, TypeVar
36from .._path import StrPath, same_path as _same_path
37from ..discovery import find_package_path
38from ..warnings import SetuptoolsWarning
40from distutils.errors import DistutilsOptionError
42if TYPE_CHECKING:
43 from typing_extensions import Self
45 from setuptools.dist import Distribution
47_K = TypeVar("_K")
48_V = TypeVar("_V", covariant=True)
51class StaticModule:
52 """Proxy to a module object that avoids executing arbitrary code."""
54 def __init__(self, name: str, spec: ModuleSpec):
55 module = ast.parse(pathlib.Path(spec.origin).read_bytes()) # type: ignore[arg-type] # Let it raise an error on None
56 vars(self).update(locals())
57 del self.self
59 def _find_assignments(self) -> Iterator[tuple[ast.AST, ast.AST]]:
60 for statement in self.module.body:
61 if isinstance(statement, ast.Assign):
62 yield from ((target, statement.value) for target in statement.targets)
63 elif isinstance(statement, ast.AnnAssign) and statement.value:
64 yield (statement.target, statement.value)
66 def __getattr__(self, attr: str):
67 """Attempt to load an attribute "statically", via :func:`ast.literal_eval`."""
68 try:
69 return next(
70 ast.literal_eval(value)
71 for target, value in self._find_assignments()
72 if isinstance(target, ast.Name) and target.id == attr
73 )
74 except Exception as e:
75 raise AttributeError(f"{self.name} has no attribute {attr}") from e
78def glob_relative(
79 patterns: Iterable[str], root_dir: StrPath | None = None
80) -> list[str]:
81 """Expand the list of glob patterns, but preserving relative paths.
83 :param list[str] patterns: List of glob patterns
84 :param str root_dir: Path to which globs should be relative
85 (current directory by default)
86 :rtype: list
87 """
88 glob_characters = {'*', '?', '[', ']', '{', '}'}
89 expanded_values = []
90 root_dir = root_dir or os.getcwd()
91 for value in patterns:
92 # Has globby characters?
93 if any(char in value for char in glob_characters):
94 # then expand the glob pattern while keeping paths *relative*:
95 glob_path = os.path.abspath(os.path.join(root_dir, value))
96 expanded_values.extend(
97 sorted(
98 os.path.relpath(path, root_dir).replace(os.sep, "/")
99 for path in iglob(glob_path, recursive=True)
100 )
101 )
103 else:
104 # take the value as-is
105 path = os.path.relpath(value, root_dir).replace(os.sep, "/")
106 expanded_values.append(path)
108 return expanded_values
111def read_files(
112 filepaths: StrPath | Iterable[StrPath], root_dir: StrPath | None = None
113) -> str:
114 """Return the content of the files concatenated using ``\n`` as str
116 This function is sandboxed and won't reach anything outside ``root_dir``
118 (By default ``root_dir`` is the current directory).
119 """
120 from more_itertools import always_iterable
122 root_dir = os.path.abspath(root_dir or os.getcwd())
123 _filepaths = (os.path.join(root_dir, path) for path in always_iterable(filepaths))
124 return '\n'.join(
125 _read_file(path)
126 for path in _filter_existing_files(_filepaths)
127 if _assert_local(path, root_dir)
128 )
131def _filter_existing_files(filepaths: Iterable[StrPath]) -> Iterator[StrPath]:
132 for path in filepaths:
133 if os.path.isfile(path):
134 yield path
135 else:
136 SetuptoolsWarning.emit(f"File {path!r} cannot be found")
139def _read_file(filepath: bytes | StrPath) -> str:
140 with open(filepath, encoding='utf-8') as f:
141 return f.read()
144def _assert_local(filepath: StrPath, root_dir: str):
145 if Path(os.path.abspath(root_dir)) not in Path(os.path.abspath(filepath)).parents:
146 msg = f"Cannot access {filepath!r} (or anything outside {root_dir!r})"
147 raise DistutilsOptionError(msg)
149 return True
152def read_attr(
153 attr_desc: str,
154 package_dir: Mapping[str, str] | None = None,
155 root_dir: StrPath | None = None,
156) -> Any:
157 """Reads the value of an attribute from a module.
159 This function will try to read the attributed statically first
160 (via :func:`ast.literal_eval`), and only evaluate the module if it fails.
162 Examples:
163 read_attr("package.attr")
164 read_attr("package.module.attr")
166 :param str attr_desc: Dot-separated string describing how to reach the
167 attribute (see examples above)
168 :param dict[str, str] package_dir: Mapping of package names to their
169 location in disk (represented by paths relative to ``root_dir``).
170 :param str root_dir: Path to directory containing all the packages in
171 ``package_dir`` (current directory by default).
172 :rtype: str
173 """
174 root_dir = root_dir or os.getcwd()
175 attrs_path = attr_desc.strip().split('.')
176 attr_name = attrs_path.pop()
177 module_name = '.'.join(attrs_path)
178 module_name = module_name or '__init__'
179 path = _find_module(module_name, package_dir, root_dir)
180 spec = _find_spec(module_name, path)
182 try:
183 return getattr(StaticModule(module_name, spec), attr_name)
184 except Exception:
185 # fallback to evaluate module
186 module = _load_spec(spec, module_name)
187 return getattr(module, attr_name)
190def _find_spec(module_name: str, module_path: StrPath | None) -> ModuleSpec:
191 spec = importlib.util.spec_from_file_location(module_name, module_path)
192 spec = spec or importlib.util.find_spec(module_name)
194 if spec is None:
195 raise ModuleNotFoundError(module_name)
197 return spec
200def _load_spec(spec: ModuleSpec, module_name: str) -> ModuleType:
201 name = getattr(spec, "__name__", module_name)
202 if name in sys.modules:
203 return sys.modules[name]
204 module = importlib.util.module_from_spec(spec)
205 sys.modules[name] = module # cache (it also ensures `==` works on loaded items)
206 spec.loader.exec_module(module) # type: ignore
207 return module
210def _find_module(
211 module_name: str, package_dir: Mapping[str, str] | None, root_dir: StrPath
212) -> str | None:
213 """Find the path to the module named ``module_name``,
214 considering the ``package_dir`` in the build configuration and ``root_dir``.
216 >>> tmp = getfixture('tmpdir')
217 >>> _ = tmp.ensure("a/b/c.py")
218 >>> _ = tmp.ensure("a/b/d/__init__.py")
219 >>> r = lambda x: x.replace(str(tmp), "tmp").replace(os.sep, "/")
220 >>> r(_find_module("a.b.c", None, tmp))
221 'tmp/a/b/c.py'
222 >>> r(_find_module("f.g.h", {"": "1", "f": "2", "f.g": "3", "f.g.h": "a/b/d"}, tmp))
223 'tmp/a/b/d/__init__.py'
224 """
225 path_start = find_package_path(module_name, package_dir or {}, root_dir)
226 candidates = chain.from_iterable(
227 (f"{path_start}{ext}", os.path.join(path_start, f"__init__{ext}"))
228 for ext in all_suffixes()
229 )
230 return next((x for x in candidates if os.path.isfile(x)), None)
233def resolve_class(
234 qualified_class_name: str,
235 package_dir: Mapping[str, str] | None = None,
236 root_dir: StrPath | None = None,
237) -> Callable:
238 """Given a qualified class name, return the associated class object"""
239 root_dir = root_dir or os.getcwd()
240 idx = qualified_class_name.rfind('.')
241 class_name = qualified_class_name[idx + 1 :]
242 pkg_name = qualified_class_name[:idx]
244 path = _find_module(pkg_name, package_dir, root_dir)
245 module = _load_spec(_find_spec(pkg_name, path), pkg_name)
246 return getattr(module, class_name)
249def cmdclass(
250 values: dict[str, str],
251 package_dir: Mapping[str, str] | None = None,
252 root_dir: StrPath | None = None,
253) -> dict[str, Callable]:
254 """Given a dictionary mapping command names to strings for qualified class
255 names, apply :func:`resolve_class` to the dict values.
256 """
257 return {k: resolve_class(v, package_dir, root_dir) for k, v in values.items()}
260def find_packages(
261 *,
262 namespaces=True,
263 fill_package_dir: dict[str, str] | None = None,
264 root_dir: StrPath | None = None,
265 **kwargs,
266) -> list[str]:
267 """Works similarly to :func:`setuptools.find_packages`, but with all
268 arguments given as keyword arguments. Moreover, ``where`` can be given
269 as a list (the results will be simply concatenated).
271 When the additional keyword argument ``namespaces`` is ``True``, it will
272 behave like :func:`setuptools.find_namespace_packages`` (i.e. include
273 implicit namespaces as per :pep:`420`).
275 The ``where`` argument will be considered relative to ``root_dir`` (or the current
276 working directory when ``root_dir`` is not given).
278 If the ``fill_package_dir`` argument is passed, this function will consider it as a
279 similar data structure to the ``package_dir`` configuration parameter add fill-in
280 any missing package location.
282 :rtype: list
283 """
284 from more_itertools import always_iterable, unique_everseen
286 from setuptools.discovery import construct_package_dir
288 if namespaces:
289 from setuptools.discovery import PEP420PackageFinder as PackageFinder
290 else:
291 from setuptools.discovery import PackageFinder # type: ignore
293 root_dir = root_dir or os.curdir
294 where = kwargs.pop('where', ['.'])
295 packages: list[str] = []
296 fill_package_dir = {} if fill_package_dir is None else fill_package_dir
297 search = list(unique_everseen(always_iterable(where)))
299 if len(search) == 1 and all(not _same_path(search[0], x) for x in (".", root_dir)):
300 fill_package_dir.setdefault("", search[0])
302 for path in search:
303 package_path = _nest_path(root_dir, path)
304 pkgs = PackageFinder.find(package_path, **kwargs)
305 packages.extend(pkgs)
306 if pkgs and not (
307 fill_package_dir.get("") == path or os.path.samefile(package_path, root_dir)
308 ):
309 fill_package_dir.update(construct_package_dir(pkgs, path))
311 return packages
314def _nest_path(parent: StrPath, path: StrPath) -> str:
315 path = parent if path in {".", ""} else os.path.join(parent, path)
316 return os.path.normpath(path)
319def version(value: Callable | Iterable[str | int] | str) -> str:
320 """When getting the version directly from an attribute,
321 it should be normalised to string.
322 """
323 _value = value() if callable(value) else value
325 if isinstance(_value, str):
326 return _value
327 if hasattr(_value, '__iter__'):
328 return '.'.join(map(str, _value))
329 return '%s' % _value
332def canonic_package_data(package_data: dict) -> dict:
333 if "*" in package_data:
334 package_data[""] = package_data.pop("*")
335 return package_data
338def canonic_data_files(
339 data_files: list | dict, root_dir: StrPath | None = None
340) -> list[tuple[str, list[str]]]:
341 """For compatibility with ``setup.py``, ``data_files`` should be a list
342 of pairs instead of a dict.
344 This function also expands glob patterns.
345 """
346 if isinstance(data_files, list):
347 return data_files
349 return [
350 (dest, glob_relative(patterns, root_dir))
351 for dest, patterns in data_files.items()
352 ]
355def entry_points(text: str, text_source="entry-points") -> dict[str, dict]:
356 """Given the contents of entry-points file,
357 process it into a 2-level dictionary (``dict[str, dict[str, str]]``).
358 The first level keys are entry-point groups, the second level keys are
359 entry-point names, and the second level values are references to objects
360 (that correspond to the entry-point value).
361 """
362 parser = ConfigParser(default_section=None, delimiters=("=",)) # type: ignore
363 parser.optionxform = str # case sensitive
364 parser.read_string(text, text_source)
365 groups = {k: dict(v.items()) for k, v in parser.items()}
366 groups.pop(parser.default_section, None)
367 return groups
370class EnsurePackagesDiscovered:
371 """Some expand functions require all the packages to already be discovered before
372 they run, e.g. :func:`read_attr`, :func:`resolve_class`, :func:`cmdclass`.
374 Therefore in some cases we will need to run autodiscovery during the evaluation of
375 the configuration. However, it is better to postpone calling package discovery as
376 much as possible, because some parameters can influence it (e.g. ``package_dir``),
377 and those might not have been processed yet.
378 """
380 def __init__(self, distribution: Distribution):
381 self._dist = distribution
382 self._called = False
384 def __call__(self):
385 """Trigger the automatic package discovery, if it is still necessary."""
386 if not self._called:
387 self._called = True
388 self._dist.set_defaults(name=False) # Skip name, we can still be parsing
390 def __enter__(self) -> Self:
391 return self
393 def __exit__(
394 self,
395 exc_type: type[BaseException] | None,
396 exc_value: BaseException | None,
397 traceback: TracebackType | None,
398 ) -> None:
399 if self._called:
400 self._dist.set_defaults.analyse_name() # Now we can set a default name
402 def _get_package_dir(self) -> Mapping[str, str]:
403 self()
404 pkg_dir = self._dist.package_dir
405 return {} if pkg_dir is None else pkg_dir
407 @property
408 def package_dir(self) -> Mapping[str, str]:
409 """Proxy to ``package_dir`` that may trigger auto-discovery when used."""
410 return LazyMappingProxy(self._get_package_dir)
413class LazyMappingProxy(Mapping[_K, _V]):
414 """Mapping proxy that delays resolving the target object, until really needed.
416 >>> def obtain_mapping():
417 ... print("Running expensive function!")
418 ... return {"key": "value", "other key": "other value"}
419 >>> mapping = LazyMappingProxy(obtain_mapping)
420 >>> mapping["key"]
421 Running expensive function!
422 'value'
423 >>> mapping["other key"]
424 'other value'
425 """
427 def __init__(self, obtain_mapping_value: Callable[[], Mapping[_K, _V]]):
428 self._obtain = obtain_mapping_value
429 self._value: Mapping[_K, _V] | None = None
431 def _target(self) -> Mapping[_K, _V]:
432 if self._value is None:
433 self._value = self._obtain()
434 return self._value
436 def __getitem__(self, key: _K) -> _V:
437 return self._target()[key]
439 def __len__(self) -> int:
440 return len(self._target())
442 def __iter__(self) -> Iterator[_K]:
443 return iter(self._target())