Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/setuptools/config/expand.py: 29%

Shortcuts on this page

r m x   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

187 statements  

1"""Utility functions to expand configuration directives or special values 

2(such glob patterns). 

3 

4We can split the process of interpreting configuration files into 2 steps: 

5 

61. The parsing the file contents from strings to value objects 

7 that can be understand by Python (for example a string with a comma 

8 separated list of keywords into an actual Python list of strings). 

9 

102. The expansion (or post-processing) of these values according to the 

11 semantics ``setuptools`` assign to them (for example a configuration field 

12 with the ``file:`` directive should be expanded from a list of file paths to 

13 a single string with the contents of those files concatenated) 

14 

15This module focus on the second step, and therefore allow sharing the expansion 

16functions among several configuration file formats. 

17 

18**PRIVATE MODULE**: API reserved for setuptools internal usage only. 

19""" 

20 

21from __future__ import annotations 

22 

23import ast 

24import importlib 

25import os 

26import pathlib 

27import sys 

28from configparser import ConfigParser 

29from glob import iglob 

30from importlib.machinery import ModuleSpec, all_suffixes 

31from itertools import chain 

32from pathlib import Path 

33from types import ModuleType, TracebackType 

34from typing import TYPE_CHECKING, Any, Callable, Iterable, Iterator, Mapping, TypeVar 

35 

36from .._path import StrPath, same_path as _same_path 

37from ..discovery import find_package_path 

38from ..warnings import SetuptoolsWarning 

39 

40from distutils.errors import DistutilsOptionError 

41 

42if TYPE_CHECKING: 

43 from typing_extensions import Self 

44 

45 from setuptools.dist import Distribution 

46 

47_K = TypeVar("_K") 

48_V = TypeVar("_V", covariant=True) 

49 

50 

51class StaticModule: 

52 """Proxy to a module object that avoids executing arbitrary code.""" 

53 

54 def __init__(self, name: str, spec: ModuleSpec): 

55 module = ast.parse(pathlib.Path(spec.origin).read_bytes()) # type: ignore[arg-type] # Let it raise an error on None 

56 vars(self).update(locals()) 

57 del self.self 

58 

59 def _find_assignments(self) -> Iterator[tuple[ast.AST, ast.AST]]: 

60 for statement in self.module.body: 

61 if isinstance(statement, ast.Assign): 

62 yield from ((target, statement.value) for target in statement.targets) 

63 elif isinstance(statement, ast.AnnAssign) and statement.value: 

64 yield (statement.target, statement.value) 

65 

66 def __getattr__(self, attr: str): 

67 """Attempt to load an attribute "statically", via :func:`ast.literal_eval`.""" 

68 try: 

69 return next( 

70 ast.literal_eval(value) 

71 for target, value in self._find_assignments() 

72 if isinstance(target, ast.Name) and target.id == attr 

73 ) 

74 except Exception as e: 

75 raise AttributeError(f"{self.name} has no attribute {attr}") from e 

76 

77 

78def glob_relative( 

79 patterns: Iterable[str], root_dir: StrPath | None = None 

80) -> list[str]: 

81 """Expand the list of glob patterns, but preserving relative paths. 

82 

83 :param list[str] patterns: List of glob patterns 

84 :param str root_dir: Path to which globs should be relative 

85 (current directory by default) 

86 :rtype: list 

87 """ 

88 glob_characters = {'*', '?', '[', ']', '{', '}'} 

89 expanded_values = [] 

90 root_dir = root_dir or os.getcwd() 

91 for value in patterns: 

92 # Has globby characters? 

93 if any(char in value for char in glob_characters): 

94 # then expand the glob pattern while keeping paths *relative*: 

95 glob_path = os.path.abspath(os.path.join(root_dir, value)) 

96 expanded_values.extend( 

97 sorted( 

98 os.path.relpath(path, root_dir).replace(os.sep, "/") 

99 for path in iglob(glob_path, recursive=True) 

100 ) 

101 ) 

102 

103 else: 

104 # take the value as-is 

105 path = os.path.relpath(value, root_dir).replace(os.sep, "/") 

106 expanded_values.append(path) 

107 

108 return expanded_values 

109 

110 

111def read_files( 

112 filepaths: StrPath | Iterable[StrPath], root_dir: StrPath | None = None 

113) -> str: 

114 """Return the content of the files concatenated using ``\n`` as str 

115 

116 This function is sandboxed and won't reach anything outside ``root_dir`` 

117 

118 (By default ``root_dir`` is the current directory). 

119 """ 

120 from more_itertools import always_iterable 

121 

122 root_dir = os.path.abspath(root_dir or os.getcwd()) 

123 _filepaths = (os.path.join(root_dir, path) for path in always_iterable(filepaths)) 

124 return '\n'.join( 

125 _read_file(path) 

126 for path in _filter_existing_files(_filepaths) 

127 if _assert_local(path, root_dir) 

128 ) 

129 

130 

131def _filter_existing_files(filepaths: Iterable[StrPath]) -> Iterator[StrPath]: 

132 for path in filepaths: 

133 if os.path.isfile(path): 

134 yield path 

135 else: 

136 SetuptoolsWarning.emit(f"File {path!r} cannot be found") 

137 

138 

139def _read_file(filepath: bytes | StrPath) -> str: 

140 with open(filepath, encoding='utf-8') as f: 

141 return f.read() 

142 

143 

144def _assert_local(filepath: StrPath, root_dir: str): 

145 if Path(os.path.abspath(root_dir)) not in Path(os.path.abspath(filepath)).parents: 

146 msg = f"Cannot access {filepath!r} (or anything outside {root_dir!r})" 

147 raise DistutilsOptionError(msg) 

148 

149 return True 

150 

151 

152def read_attr( 

153 attr_desc: str, 

154 package_dir: Mapping[str, str] | None = None, 

155 root_dir: StrPath | None = None, 

156) -> Any: 

157 """Reads the value of an attribute from a module. 

158 

159 This function will try to read the attributed statically first 

160 (via :func:`ast.literal_eval`), and only evaluate the module if it fails. 

161 

162 Examples: 

163 read_attr("package.attr") 

164 read_attr("package.module.attr") 

165 

166 :param str attr_desc: Dot-separated string describing how to reach the 

167 attribute (see examples above) 

168 :param dict[str, str] package_dir: Mapping of package names to their 

169 location in disk (represented by paths relative to ``root_dir``). 

170 :param str root_dir: Path to directory containing all the packages in 

171 ``package_dir`` (current directory by default). 

172 :rtype: str 

173 """ 

174 root_dir = root_dir or os.getcwd() 

175 attrs_path = attr_desc.strip().split('.') 

176 attr_name = attrs_path.pop() 

177 module_name = '.'.join(attrs_path) 

178 module_name = module_name or '__init__' 

179 path = _find_module(module_name, package_dir, root_dir) 

180 spec = _find_spec(module_name, path) 

181 

182 try: 

183 return getattr(StaticModule(module_name, spec), attr_name) 

184 except Exception: 

185 # fallback to evaluate module 

186 module = _load_spec(spec, module_name) 

187 return getattr(module, attr_name) 

188 

189 

190def _find_spec(module_name: str, module_path: StrPath | None) -> ModuleSpec: 

191 spec = importlib.util.spec_from_file_location(module_name, module_path) 

192 spec = spec or importlib.util.find_spec(module_name) 

193 

194 if spec is None: 

195 raise ModuleNotFoundError(module_name) 

196 

197 return spec 

198 

199 

200def _load_spec(spec: ModuleSpec, module_name: str) -> ModuleType: 

201 name = getattr(spec, "__name__", module_name) 

202 if name in sys.modules: 

203 return sys.modules[name] 

204 module = importlib.util.module_from_spec(spec) 

205 sys.modules[name] = module # cache (it also ensures `==` works on loaded items) 

206 spec.loader.exec_module(module) # type: ignore 

207 return module 

208 

209 

210def _find_module( 

211 module_name: str, package_dir: Mapping[str, str] | None, root_dir: StrPath 

212) -> str | None: 

213 """Find the path to the module named ``module_name``, 

214 considering the ``package_dir`` in the build configuration and ``root_dir``. 

215 

216 >>> tmp = getfixture('tmpdir') 

217 >>> _ = tmp.ensure("a/b/c.py") 

218 >>> _ = tmp.ensure("a/b/d/__init__.py") 

219 >>> r = lambda x: x.replace(str(tmp), "tmp").replace(os.sep, "/") 

220 >>> r(_find_module("a.b.c", None, tmp)) 

221 'tmp/a/b/c.py' 

222 >>> r(_find_module("f.g.h", {"": "1", "f": "2", "f.g": "3", "f.g.h": "a/b/d"}, tmp)) 

223 'tmp/a/b/d/__init__.py' 

224 """ 

225 path_start = find_package_path(module_name, package_dir or {}, root_dir) 

226 candidates = chain.from_iterable( 

227 (f"{path_start}{ext}", os.path.join(path_start, f"__init__{ext}")) 

228 for ext in all_suffixes() 

229 ) 

230 return next((x for x in candidates if os.path.isfile(x)), None) 

231 

232 

233def resolve_class( 

234 qualified_class_name: str, 

235 package_dir: Mapping[str, str] | None = None, 

236 root_dir: StrPath | None = None, 

237) -> Callable: 

238 """Given a qualified class name, return the associated class object""" 

239 root_dir = root_dir or os.getcwd() 

240 idx = qualified_class_name.rfind('.') 

241 class_name = qualified_class_name[idx + 1 :] 

242 pkg_name = qualified_class_name[:idx] 

243 

244 path = _find_module(pkg_name, package_dir, root_dir) 

245 module = _load_spec(_find_spec(pkg_name, path), pkg_name) 

246 return getattr(module, class_name) 

247 

248 

249def cmdclass( 

250 values: dict[str, str], 

251 package_dir: Mapping[str, str] | None = None, 

252 root_dir: StrPath | None = None, 

253) -> dict[str, Callable]: 

254 """Given a dictionary mapping command names to strings for qualified class 

255 names, apply :func:`resolve_class` to the dict values. 

256 """ 

257 return {k: resolve_class(v, package_dir, root_dir) for k, v in values.items()} 

258 

259 

260def find_packages( 

261 *, 

262 namespaces=True, 

263 fill_package_dir: dict[str, str] | None = None, 

264 root_dir: StrPath | None = None, 

265 **kwargs, 

266) -> list[str]: 

267 """Works similarly to :func:`setuptools.find_packages`, but with all 

268 arguments given as keyword arguments. Moreover, ``where`` can be given 

269 as a list (the results will be simply concatenated). 

270 

271 When the additional keyword argument ``namespaces`` is ``True``, it will 

272 behave like :func:`setuptools.find_namespace_packages`` (i.e. include 

273 implicit namespaces as per :pep:`420`). 

274 

275 The ``where`` argument will be considered relative to ``root_dir`` (or the current 

276 working directory when ``root_dir`` is not given). 

277 

278 If the ``fill_package_dir`` argument is passed, this function will consider it as a 

279 similar data structure to the ``package_dir`` configuration parameter add fill-in 

280 any missing package location. 

281 

282 :rtype: list 

283 """ 

284 from more_itertools import always_iterable, unique_everseen 

285 

286 from setuptools.discovery import construct_package_dir 

287 

288 if namespaces: 

289 from setuptools.discovery import PEP420PackageFinder as PackageFinder 

290 else: 

291 from setuptools.discovery import PackageFinder # type: ignore 

292 

293 root_dir = root_dir or os.curdir 

294 where = kwargs.pop('where', ['.']) 

295 packages: list[str] = [] 

296 fill_package_dir = {} if fill_package_dir is None else fill_package_dir 

297 search = list(unique_everseen(always_iterable(where))) 

298 

299 if len(search) == 1 and all(not _same_path(search[0], x) for x in (".", root_dir)): 

300 fill_package_dir.setdefault("", search[0]) 

301 

302 for path in search: 

303 package_path = _nest_path(root_dir, path) 

304 pkgs = PackageFinder.find(package_path, **kwargs) 

305 packages.extend(pkgs) 

306 if pkgs and not ( 

307 fill_package_dir.get("") == path or os.path.samefile(package_path, root_dir) 

308 ): 

309 fill_package_dir.update(construct_package_dir(pkgs, path)) 

310 

311 return packages 

312 

313 

314def _nest_path(parent: StrPath, path: StrPath) -> str: 

315 path = parent if path in {".", ""} else os.path.join(parent, path) 

316 return os.path.normpath(path) 

317 

318 

319def version(value: Callable | Iterable[str | int] | str) -> str: 

320 """When getting the version directly from an attribute, 

321 it should be normalised to string. 

322 """ 

323 _value = value() if callable(value) else value 

324 

325 if isinstance(_value, str): 

326 return _value 

327 if hasattr(_value, '__iter__'): 

328 return '.'.join(map(str, _value)) 

329 return '%s' % _value 

330 

331 

332def canonic_package_data(package_data: dict) -> dict: 

333 if "*" in package_data: 

334 package_data[""] = package_data.pop("*") 

335 return package_data 

336 

337 

338def canonic_data_files( 

339 data_files: list | dict, root_dir: StrPath | None = None 

340) -> list[tuple[str, list[str]]]: 

341 """For compatibility with ``setup.py``, ``data_files`` should be a list 

342 of pairs instead of a dict. 

343 

344 This function also expands glob patterns. 

345 """ 

346 if isinstance(data_files, list): 

347 return data_files 

348 

349 return [ 

350 (dest, glob_relative(patterns, root_dir)) 

351 for dest, patterns in data_files.items() 

352 ] 

353 

354 

355def entry_points(text: str, text_source="entry-points") -> dict[str, dict]: 

356 """Given the contents of entry-points file, 

357 process it into a 2-level dictionary (``dict[str, dict[str, str]]``). 

358 The first level keys are entry-point groups, the second level keys are 

359 entry-point names, and the second level values are references to objects 

360 (that correspond to the entry-point value). 

361 """ 

362 parser = ConfigParser(default_section=None, delimiters=("=",)) # type: ignore 

363 parser.optionxform = str # case sensitive 

364 parser.read_string(text, text_source) 

365 groups = {k: dict(v.items()) for k, v in parser.items()} 

366 groups.pop(parser.default_section, None) 

367 return groups 

368 

369 

370class EnsurePackagesDiscovered: 

371 """Some expand functions require all the packages to already be discovered before 

372 they run, e.g. :func:`read_attr`, :func:`resolve_class`, :func:`cmdclass`. 

373 

374 Therefore in some cases we will need to run autodiscovery during the evaluation of 

375 the configuration. However, it is better to postpone calling package discovery as 

376 much as possible, because some parameters can influence it (e.g. ``package_dir``), 

377 and those might not have been processed yet. 

378 """ 

379 

380 def __init__(self, distribution: Distribution): 

381 self._dist = distribution 

382 self._called = False 

383 

384 def __call__(self): 

385 """Trigger the automatic package discovery, if it is still necessary.""" 

386 if not self._called: 

387 self._called = True 

388 self._dist.set_defaults(name=False) # Skip name, we can still be parsing 

389 

390 def __enter__(self) -> Self: 

391 return self 

392 

393 def __exit__( 

394 self, 

395 exc_type: type[BaseException] | None, 

396 exc_value: BaseException | None, 

397 traceback: TracebackType | None, 

398 ) -> None: 

399 if self._called: 

400 self._dist.set_defaults.analyse_name() # Now we can set a default name 

401 

402 def _get_package_dir(self) -> Mapping[str, str]: 

403 self() 

404 pkg_dir = self._dist.package_dir 

405 return {} if pkg_dir is None else pkg_dir 

406 

407 @property 

408 def package_dir(self) -> Mapping[str, str]: 

409 """Proxy to ``package_dir`` that may trigger auto-discovery when used.""" 

410 return LazyMappingProxy(self._get_package_dir) 

411 

412 

413class LazyMappingProxy(Mapping[_K, _V]): 

414 """Mapping proxy that delays resolving the target object, until really needed. 

415 

416 >>> def obtain_mapping(): 

417 ... print("Running expensive function!") 

418 ... return {"key": "value", "other key": "other value"} 

419 >>> mapping = LazyMappingProxy(obtain_mapping) 

420 >>> mapping["key"] 

421 Running expensive function! 

422 'value' 

423 >>> mapping["other key"] 

424 'other value' 

425 """ 

426 

427 def __init__(self, obtain_mapping_value: Callable[[], Mapping[_K, _V]]): 

428 self._obtain = obtain_mapping_value 

429 self._value: Mapping[_K, _V] | None = None 

430 

431 def _target(self) -> Mapping[_K, _V]: 

432 if self._value is None: 

433 self._value = self._obtain() 

434 return self._value 

435 

436 def __getitem__(self, key: _K) -> _V: 

437 return self._target()[key] 

438 

439 def __len__(self) -> int: 

440 return len(self._target()) 

441 

442 def __iter__(self) -> Iterator[_K]: 

443 return iter(self._target())