Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/astroid/modutils.py: 25%

1# Licensed under the LGPL: https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html

2# For details: https://github.com/pylint-dev/astroid/blob/main/LICENSE

3# Copyright (c) https://github.com/pylint-dev/astroid/blob/main/CONTRIBUTORS.txt

5"""Python modules manipulation utility functions.

7:type PY_SOURCE_EXTS: tuple(str)

8:var PY_SOURCE_EXTS: list of possible python source file extension

10:type STD_LIB_DIRS: set of str

11:var STD_LIB_DIRS: directories where standard modules are located

13:type BUILTIN_MODULES: dict

14:var BUILTIN_MODULES: dictionary with builtin module names has key

15"""

17from __future__ import annotations

19import importlib

20import importlib.machinery

21import importlib.util

22import io

23import itertools

24import logging

25import os

26import sys

27import sysconfig

28import types

29import warnings

30from collections.abc import Callable, Iterable, Sequence

31from contextlib import redirect_stderr, redirect_stdout

32from functools import lru_cache

33from pathlib import Path

35from astroid.const import IS_JYTHON, IS_PYPY, PY310_PLUS

36from astroid.interpreter._import import spec, util

38if PY310_PLUS:

39 from sys import stdlib_module_names

40else:

41 from astroid._backport_stdlib_names import stdlib_module_names

43logger = logging.getLogger(__name__)

46if sys.platform.startswith("win"):

47 PY_SOURCE_EXTS = ("py", "pyw", "pyi")

48 PY_COMPILED_EXTS = ("dll", "pyd")

49else:

50 PY_SOURCE_EXTS = ("py", "pyi")

51 PY_COMPILED_EXTS = ("so",)

54# TODO: Adding `platstdlib` is a fix for a workaround in virtualenv. At some point we should

55# revisit whether this is still necessary. See https://github.com/pylint-dev/astroid/pull/1323.

56STD_LIB_DIRS = {sysconfig.get_path("stdlib"), sysconfig.get_path("platstdlib")}

58if os.name == "nt":

59 STD_LIB_DIRS.add(os.path.join(sys.prefix, "dlls"))

60 try:

61 # real_prefix is defined when running inside virtual environments,

62 # created with the **virtualenv** library.

63 # Deprecated in virtualenv==16.7.9

64 # See: https://github.com/pypa/virtualenv/issues/1622

65 STD_LIB_DIRS.add(os.path.join(sys.real_prefix, "dlls")) # type: ignore[attr-defined]

66 except AttributeError:

67 # sys.base_exec_prefix is always defined, but in a virtual environment

68 # created with the stdlib **venv** module, it points to the original

69 # installation, if the virtual env is activated.

70 try:

71 STD_LIB_DIRS.add(os.path.join(sys.base_exec_prefix, "dlls"))

72 except AttributeError:

73 pass

75if IS_PYPY and sys.version_info < (3, 8):

76 # PyPy stores the stdlib in two places: sys.prefix/lib_pypy and sys.prefix/lib-python/3

77 # sysconfig.get_path on PyPy returns the first, but without an underscore so we patch this manually.

78 # Beginning with 3.8 the stdlib is only stored in: sys.prefix/pypy{py_version_short}

79 STD_LIB_DIRS.add(str(Path(sysconfig.get_path("stdlib")).parent / "lib_pypy"))

80 STD_LIB_DIRS.add(str(Path(sysconfig.get_path("stdlib")).parent / "lib-python/3"))

82 # TODO: This is a fix for a workaround in virtualenv. At some point we should revisit

83 # whether this is still necessary. See https://github.com/pylint-dev/astroid/pull/1324.

84 STD_LIB_DIRS.add(str(Path(sysconfig.get_path("platstdlib")).parent / "lib_pypy"))

85 STD_LIB_DIRS.add(

86 str(Path(sysconfig.get_path("platstdlib")).parent / "lib-python/3")

87 )

89if os.name == "posix":

90 # Need the real prefix if we're in a virtualenv, otherwise

91 # the usual one will do.

92 # Deprecated in virtualenv==16.7.9

93 # See: https://github.com/pypa/virtualenv/issues/1622

94 try:

95 prefix: str = sys.real_prefix # type: ignore[attr-defined]

96 except AttributeError:

97 prefix = sys.prefix

99 def _posix_path(path: str) -> str:

100 base_python = "python%d.%d" % sys.version_info[:2]

101 return os.path.join(prefix, path, base_python)

102

103 STD_LIB_DIRS.add(_posix_path("lib"))

104 if sys.maxsize > 2**32:

105 # This tries to fix a problem with /usr/lib64 builds,

106 # where systems are running both 32-bit and 64-bit code

107 # on the same machine, which reflects into the places where

108 # standard library could be found. More details can be found

109 # here http://bugs.python.org/issue1294959.

110 # An easy reproducing case would be

111 # https://github.com/pylint-dev/pylint/issues/712#issuecomment-163178753

112 STD_LIB_DIRS.add(_posix_path("lib64"))

113

114EXT_LIB_DIRS = {sysconfig.get_path("purelib"), sysconfig.get_path("platlib")}

115BUILTIN_MODULES = dict.fromkeys(sys.builtin_module_names, True)

116

117

118class NoSourceFile(Exception):

119 """Exception raised when we are not able to get a python

120 source file for a precompiled file.

121 """

122

123

124def _normalize_path(path: str) -> str:

125 """Resolve symlinks in path and convert to absolute path.

126

127 Note that environment variables and ~ in the path need to be expanded in

128 advance.

129

130 This can be cached by using _cache_normalize_path.

131 """

132 return os.path.normcase(os.path.realpath(path))

133

134

135def _path_from_filename(filename: str, is_jython: bool = IS_JYTHON) -> str:

136 if not is_jython:

137 return filename

138 head, has_pyclass, _ = filename.partition("$py.class")

139 if has_pyclass:

140 return head + ".py"

141 return filename

142

143

144def _handle_blacklist(

145 blacklist: Sequence[str], dirnames: list[str], filenames: list[str]

146) -> None:

147 """Remove files/directories in the black list.

148

149 dirnames/filenames are usually from os.walk

150 """

151 for norecurs in blacklist:

152 if norecurs in dirnames:

153 dirnames.remove(norecurs)

154 elif norecurs in filenames:

155 filenames.remove(norecurs)

156

157

158@lru_cache

159def _cache_normalize_path_(path: str) -> str:

160 return _normalize_path(path)

161

162

163def _cache_normalize_path(path: str) -> str:

164 """Normalize path with caching."""

165 # _module_file calls abspath on every path in sys.path every time it's

166 # called; on a larger codebase this easily adds up to half a second just

167 # assembling path components. This cache alleviates that.

168 if not path: # don't cache result for ''

169 return _normalize_path(path)

170 return _cache_normalize_path_(path)

171

172

173def load_module_from_name(dotted_name: str) -> types.ModuleType:

174 """Load a Python module from its name.

175

176 :type dotted_name: str

177 :param dotted_name: python name of a module or package

178

179 :raise ImportError: if the module or package is not found

180

181 :rtype: module

182 :return: the loaded module

183 """

184 try:

185 return sys.modules[dotted_name]

186 except KeyError:

187 pass

188

189 # Capture and log anything emitted during import to avoid

190 # contaminating JSON reports in pylint

191 with redirect_stderr(io.StringIO()) as stderr, redirect_stdout(

192 io.StringIO()

193 ) as stdout:

194 module = importlib.import_module(dotted_name)

195

196 stderr_value = stderr.getvalue()

197 if stderr_value:

198 logger.error(

199 "Captured stderr while importing %s:\n%s", dotted_name, stderr_value

200 )

201 stdout_value = stdout.getvalue()

202 if stdout_value:

203 logger.info(

204 "Captured stdout while importing %s:\n%s", dotted_name, stdout_value

205 )

206

207 return module

208

209

210def load_module_from_modpath(parts: Sequence[str]) -> types.ModuleType:

211 """Load a python module from its split name.

212

213 :param parts:

214 python name of a module or package split on '.'

215

216 :raise ImportError: if the module or package is not found

217

218 :return: the loaded module

219 """

220 return load_module_from_name(".".join(parts))

221

222

223def load_module_from_file(filepath: str) -> types.ModuleType:

224 """Load a Python module from it's path.

225

226 :type filepath: str

227 :param filepath: path to the python module or package

228

229 :raise ImportError: if the module or package is not found

230

231 :rtype: module

232 :return: the loaded module

233 """

234 modpath = modpath_from_file(filepath)

235 return load_module_from_modpath(modpath)

236

237

238def check_modpath_has_init(path: str, mod_path: list[str]) -> bool:

239 """Check there are some __init__.py all along the way."""

240 modpath: list[str] = []

241 for part in mod_path:

242 modpath.append(part)

243 path = os.path.join(path, part)

244 if not _has_init(path):

245 old_namespace = util.is_namespace(".".join(modpath))

246 if not old_namespace:

247 return False

248 return True

249

250

251def _get_relative_base_path(filename: str, path_to_check: str) -> list[str] | None:

252 """Extracts the relative mod path of the file to import from.

253

254 Check if a file is within the passed in path and if so, returns the

255 relative mod path from the one passed in.

256

257 If the filename is no in path_to_check, returns None

258

259 Note this function will look for both abs and realpath of the file,

260 this allows to find the relative base path even if the file is a

261 symlink of a file in the passed in path

262

263 Examples:

264 _get_relative_base_path("/a/b/c/d.py", "/a/b") -> ["c","d"]

265 _get_relative_base_path("/a/b/c/d.py", "/dev") -> None

266 """

267 importable_path = None

268 path_to_check = os.path.normcase(path_to_check)

269 abs_filename = os.path.abspath(filename)

270 if os.path.normcase(abs_filename).startswith(path_to_check):

271 importable_path = abs_filename

272

273 real_filename = os.path.realpath(filename)

274 if os.path.normcase(real_filename).startswith(path_to_check):

275 importable_path = real_filename

276

277 if importable_path:

278 base_path = os.path.splitext(importable_path)[0]

279 relative_base_path = base_path[len(path_to_check) :]

280 return [pkg for pkg in relative_base_path.split(os.sep) if pkg]

281

282 return None

283

284

285def modpath_from_file_with_callback(

286 filename: str,

287 path: Sequence[str] | None = None,

288 is_package_cb: Callable[[str, list[str]], bool] | None = None,

289) -> list[str]:

290 filename = os.path.expanduser(_path_from_filename(filename))

291 paths_to_check = sys.path.copy()

292 if path:

293 paths_to_check += path

294 for pathname in itertools.chain(

295 paths_to_check, map(_cache_normalize_path, paths_to_check)

296 ):

297 if not pathname:

298 continue

299 modpath = _get_relative_base_path(filename, pathname)

300 if not modpath:

301 continue

302 assert is_package_cb is not None

303 if is_package_cb(pathname, modpath[:-1]):

304 return modpath

305

306 raise ImportError(

307 "Unable to find module for {} in {}".format(filename, ", \n".join(sys.path))

308 )

309

310

311def modpath_from_file(filename: str, path: Sequence[str] | None = None) -> list[str]:

312 """Get the corresponding split module's name from a filename.

313

314 This function will return the name of a module or package split on `.`.

315

316 :type filename: str

317 :param filename: file's path for which we want the module's name

318

319 :type Optional[List[str]] path:

320 Optional list of path where the module or package should be

321 searched (use sys.path if nothing or None is given)

322

323 :raise ImportError:

324 if the corresponding module's name has not been found

325

326 :rtype: list(str)

327 :return: the corresponding split module's name

328 """

329 return modpath_from_file_with_callback(filename, path, check_modpath_has_init)

330

331

332def file_from_modpath(

333 modpath: list[str],

334 path: Sequence[str] | None = None,

335 context_file: str | None = None,

336) -> str | None:

337 return file_info_from_modpath(modpath, path, context_file).location

338

339

340def file_info_from_modpath(

341 modpath: list[str],

342 path: Sequence[str] | None = None,

343 context_file: str | None = None,

344) -> spec.ModuleSpec:

345 """Given a mod path (i.e. split module / package name), return the

346 corresponding file.

347

348 Giving priority to source file over precompiled file if it exists.

349

350 :param modpath:

351 split module's name (i.e name of a module or package split

352 on '.')

353 (this means explicit relative imports that start with dots have

354 empty strings in this list!)

355

356 :param path:

357 optional list of path where the module or package should be

358 searched (use sys.path if nothing or None is given)

359

360 :param context_file:

361 context file to consider, necessary if the identifier has been

362 introduced using a relative import unresolvable in the actual

363 context (i.e. modutils)

364

365 :raise ImportError: if there is no such module in the directory

366

367 :return:

368 the path to the module's file or None if it's an integrated

369 builtin module such as 'sys'

370 """

371 if context_file is not None:

372 context: str | None = os.path.dirname(context_file)

373 else:

374 context = context_file

375 if modpath[0] == "xml":

376 # handle _xmlplus

377 try:

378 return _spec_from_modpath(["_xmlplus"] + modpath[1:], path, context)

379 except ImportError:

380 return _spec_from_modpath(modpath, path, context)

381 elif modpath == ["os", "path"]:

382 # FIXME: currently ignoring search_path...

383 return spec.ModuleSpec(

384 name="os.path",

385 location=os.path.__file__,

386 type=spec.ModuleType.PY_SOURCE,

387 )

388 return _spec_from_modpath(modpath, path, context)

389

390

391def get_module_part(dotted_name: str, context_file: str | None = None) -> str:

392 """Given a dotted name return the module part of the name :

393

394 >>> get_module_part('astroid.as_string.dump')

395 'astroid.as_string'

396

397 :param dotted_name: full name of the identifier we are interested in

398

399 :param context_file:

400 context file to consider, necessary if the identifier has been

401 introduced using a relative import unresolvable in the actual

402 context (i.e. modutils)

403

404 :raise ImportError: if there is no such module in the directory

405

406 :return:

407 the module part of the name or None if we have not been able at

408 all to import the given name

409

410 XXX: deprecated, since it doesn't handle package precedence over module

411 (see #10066)

412 """

413 # os.path trick

414 if dotted_name.startswith("os.path"):

415 return "os.path"

416 parts = dotted_name.split(".")

417 if context_file is not None:

418 # first check for builtin module which won't be considered latter

419 # in that case (path != None)

420 if parts[0] in BUILTIN_MODULES:

421 if len(parts) > 2:

422 raise ImportError(dotted_name)

423 return parts[0]

424 # don't use += or insert, we want a new list to be created !

425 path: list[str] | None = None

426 starti = 0

427 if parts[0] == "":

428 assert (

429 context_file is not None

430 ), "explicit relative import, but no context_file?"

431 path = [] # prevent resolving the import non-relatively

432 starti = 1

433 while parts[starti] == "": # for all further dots: change context

434 starti += 1

435 assert (

436 context_file is not None

437 ), "explicit relative import, but no context_file?"

438 context_file = os.path.dirname(context_file)

439 for i in range(starti, len(parts)):

440 try:

441 file_from_modpath(

442 parts[starti : i + 1], path=path, context_file=context_file

443 )

444 except ImportError:

445 if i < max(1, len(parts) - 2):

446 raise

447 return ".".join(parts[:i])

448 return dotted_name

449

450

451def get_module_files(

452 src_directory: str, blacklist: Sequence[str], list_all: bool = False

453) -> list[str]:

454 """Given a package directory return a list of all available python

455 module's files in the package and its subpackages.

456

457 :param src_directory:

458 path of the directory corresponding to the package

459

460 :param blacklist: iterable

461 list of files or directories to ignore.

462

463 :param list_all:

464 get files from all paths, including ones without __init__.py

465

466 :return:

467 the list of all available python module's files in the package and

468 its subpackages

469 """

470 files: list[str] = []

471 for directory, dirnames, filenames in os.walk(src_directory):

472 if directory in blacklist:

473 continue

474 _handle_blacklist(blacklist, dirnames, filenames)

475 # check for __init__.py

476 if not list_all and {"__init__.py", "__init__.pyi"}.isdisjoint(filenames):

477 dirnames[:] = ()

478 continue

479 for filename in filenames:

480 if _is_python_file(filename):

481 src = os.path.join(directory, filename)

482 files.append(src)

483 return files

484

485

486def get_source_file(filename: str, include_no_ext: bool = False) -> str:

487 """Given a python module's file name return the matching source file

488 name (the filename will be returned identically if it's already an

489 absolute path to a python source file).

490

491 :param filename: python module's file name

492

493 :raise NoSourceFile: if no source file exists on the file system

494

495 :return: the absolute path of the source file if it exists

496 """

497 filename = os.path.abspath(_path_from_filename(filename))

498 base, orig_ext = os.path.splitext(filename)

499 if orig_ext == ".pyi" and os.path.exists(f"{base}{orig_ext}"):

500 return f"{base}{orig_ext}"

501 for ext in PY_SOURCE_EXTS:

502 source_path = f"{base}.{ext}"

503 if os.path.exists(source_path):

504 return source_path

505 if include_no_ext and not orig_ext and os.path.exists(base):

506 return base

507 raise NoSourceFile(filename)

508

509

510def is_python_source(filename: str | None) -> bool:

511 """Return: True if the filename is a python source file."""

512 if not filename:

513 return False

514 return os.path.splitext(filename)[1][1:] in PY_SOURCE_EXTS

515

516

517def is_stdlib_module(modname: str) -> bool:

518 """Return: True if the modname is in the standard library"""

519 return modname.split(".")[0] in stdlib_module_names

520

521

522def module_in_path(modname: str, path: str | Iterable[str]) -> bool:

523 """Try to determine if a module is imported from one of the specified paths

524

525 :param modname: name of the module

526

527 :param path: paths to consider

528

529 :return:

530 true if the module:

531 - is located on the path listed in one of the directory in `paths`

532 """

533

534 modname = modname.split(".")[0]

535 try:

536 filename = file_from_modpath([modname])

537 except ImportError:

538 # Import failed, we can't check path if we don't know it

539 return False

540

541 if filename is None:

542 # No filename likely means it's compiled in, or potentially a namespace

543 return False

544 filename = _normalize_path(filename)

545

546 if isinstance(path, str):

547 return filename.startswith(_cache_normalize_path(path))

548

549 return any(filename.startswith(_cache_normalize_path(entry)) for entry in path)

550

551

552def is_standard_module(modname: str, std_path: Iterable[str] | None = None) -> bool:

553 """Try to guess if a module is a standard python module (by default,

554 see `std_path` parameter's description).

555

556 :param modname: name of the module we are interested in

557

558 :param std_path: list of path considered has standard

559

560 :return:

561 true if the module:

562 - is located on the path listed in one of the directory in `std_path`

563 - is a built-in module

564 """

565 warnings.warn(

566 "is_standard_module() is deprecated. Use, is_stdlib_module() or module_in_path() instead",

567 DeprecationWarning,

568 stacklevel=2,

569 )

570

571 modname = modname.split(".")[0]

572 try:

573 filename = file_from_modpath([modname])

574 except ImportError:

575 # import failed, i'm probably not so wrong by supposing it's

576 # not standard...

577 return False

578 # modules which are not living in a file are considered standard

579 # (sys and __builtin__ for instance)

580 if filename is None:

581 # we assume there are no namespaces in stdlib

582 return not util.is_namespace(modname)

583 filename = _normalize_path(filename)

584 for path in EXT_LIB_DIRS:

585 if filename.startswith(_cache_normalize_path(path)):

586 return False

587 if std_path is None:

588 std_path = STD_LIB_DIRS

589

590 return any(filename.startswith(_cache_normalize_path(path)) for path in std_path)

591

592

593def is_relative(modname: str, from_file: str) -> bool:

594 """Return true if the given module name is relative to the given

595 file name.

596

597 :param modname: name of the module we are interested in

598

599 :param from_file:

600 path of the module from which modname has been imported

601

602 :return:

603 true if the module has been imported relatively to `from_file`

604 """

605 if not os.path.isdir(from_file):

606 from_file = os.path.dirname(from_file)

607 if from_file in sys.path:

608 return False

609 return bool(

610 importlib.machinery.PathFinder.find_spec(

611 modname.split(".", maxsplit=1)[0], [from_file]

612 )

613 )

614

615

616# internal only functions #####################################################

617

618

619def _spec_from_modpath(

620 modpath: list[str],

621 path: Sequence[str] | None = None,

622 context: str | None = None,

623) -> spec.ModuleSpec:

624 """Given a mod path (i.e. split module / package name), return the

625 corresponding spec.

626

627 this function is used internally, see `file_from_modpath`'s

628 documentation for more information

629 """

630 assert modpath

631 location = None

632 if context is not None:

633 try:

634 found_spec = spec.find_spec(modpath, [context])

635 location = found_spec.location

636 except ImportError:

637 found_spec = spec.find_spec(modpath, path)

638 location = found_spec.location

639 else:

640 found_spec = spec.find_spec(modpath, path)

641 if found_spec.type == spec.ModuleType.PY_COMPILED:

642 try:

643 assert found_spec.location is not None

644 location = get_source_file(found_spec.location)

645 return found_spec._replace(

646 location=location, type=spec.ModuleType.PY_SOURCE

647 )

648 except NoSourceFile:

649 return found_spec._replace(location=location)

650 elif found_spec.type == spec.ModuleType.C_BUILTIN:

651 # integrated builtin module

652 return found_spec._replace(location=None)

653 elif found_spec.type == spec.ModuleType.PKG_DIRECTORY:

654 assert found_spec.location is not None

655 location = _has_init(found_spec.location)

656 return found_spec._replace(location=location, type=spec.ModuleType.PY_SOURCE)

657 return found_spec

658

659

660def _is_python_file(filename: str) -> bool:

661 """Return true if the given filename should be considered as a python file.

662

663 .pyc and .pyo are ignored

664 """

665 return filename.endswith((".py", ".pyi", ".so", ".pyd", ".pyw"))

666

667

668def _has_init(directory: str) -> str | None:

669 """If the given directory has a valid __init__ file, return its path,

670 else return None.

671 """

672 mod_or_pack = os.path.join(directory, "__init__")

673 for ext in (*PY_SOURCE_EXTS, "pyc", "pyo"):

674 if os.path.exists(mod_or_pack + "." + ext):

675 return mod_or_pack + "." + ext

676 return None

677

678

679def is_namespace(specobj: spec.ModuleSpec) -> bool:

680 return specobj.type == spec.ModuleType.PY_NAMESPACE

681

682

683def is_directory(specobj: spec.ModuleSpec) -> bool:

684 return specobj.type == spec.ModuleType.PKG_DIRECTORY

685

686

687def is_module_name_part_of_extension_package_whitelist(

688 module_name: str, package_whitelist: set[str]

689) -> bool:

690 """

691 Returns True if one part of the module name is in the package whitelist.

692

693 >>> is_module_name_part_of_extension_package_whitelist('numpy.core.umath', {'numpy'})

694 True

695 """

696 parts = module_name.split(".")

697 return any(

698 ".".join(parts[:x]) in package_whitelist for x in range(1, len(parts) + 1)

699 )