Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/jsonpickle/util.py: 47%

5# This software is licensed as described in the file COPYING, which

6# you should have received as part of this distribution.

8"""Helper functions for pickling and unpickling. Most functions assist in

9determining the type of an object.

10"""

12import base64

13import binascii

14import collections

15import inspect

16import io

17import operator

18import sys

19import time

20import types

21import warnings

22from collections.abc import Callable, Iterable, Iterator

23from typing import Any, TypeVar

25from . import tags

27# key

28K = TypeVar("K")

29# value

30V = TypeVar("V")

31# type

32T = TypeVar("T")

34_ITERATOR_TYPE: type = type(iter(""))

35# Protocol used when asking objects to reduce themselves. 4 is the lowest

36# protocol that encodes datetime's PEP 495 fold bit; 2 silently drops it.

37PICKLE_PROTOCOL: int = 4

38SEQUENCES: tuple[type] = (list, set, tuple) # type: ignore[assignment]

39SEQUENCES_SET: set[type] = {list, set, tuple}

40PRIMITIVES: set[type] = {str, bool, int, float, type(None)}

41FUNCTION_TYPES: set[type] = {

42 types.FunctionType,

43 types.MethodType,

44 types.LambdaType,

45 types.BuiltinFunctionType,

46 types.BuiltinMethodType,

47}

48# Internal set for NON_REDUCIBLE_TYPES that excludes MethodType to allow method round-trip

49_NON_REDUCIBLE_FUNCTION_TYPES: set[type] = FUNCTION_TYPES - {types.MethodType}

50NON_REDUCIBLE_TYPES: set[type] = (

51 {

52 list,

53 dict,

54 set,

55 tuple,

56 object,

57 bytes,

58 }

59 | PRIMITIVES

60 | _NON_REDUCIBLE_FUNCTION_TYPES

61)

62NON_CLASS_TYPES: set[type] = {

63 list,

64 dict,

65 set,

66 tuple,

67 bytes,

68} | PRIMITIVES

69_TYPES_IMPORTABLE_NAMES: dict[type | Callable[..., Any], str] = {

70 getattr(types, name): f"types.{name}"

71 for name in types.__all__

72 if name.endswith("Type")

73}

74# for module lookup

75_MISSING = object()

76_MAIN_MODULES = ("__main__", "__mp_main__")

77_GLOBAL_MODULES: dict[tuple[int, str], str] = {}

80def _is_type(obj: Any) -> bool:

81 """Returns True is obj is a reference to a type.

83 >>> _is_type(1)

84 False

86 >>> _is_type(object)

87 True

89 >>> class Klass: pass

90 >>> _is_type(Klass)

91 True

92 """

93 # use "isinstance" and not "is" to allow for metaclasses

94 return isinstance(obj, type)

97def has_method(obj: Any, name: str) -> bool:

98 # false if attribute doesn't exist

99 if not hasattr(obj, name):

100 return False

101 func = getattr(obj, name)

102

103 # builtin descriptors like __getnewargs__

104 if isinstance(func, types.BuiltinMethodType):

105 return True

106

107 # note that FunctionType has a different meaning in py2/py3

108 if not isinstance(func, (types.MethodType, types.FunctionType)):

109 return False

110

111 # need to go through __dict__'s since in py3

112 # methods are essentially descriptors

113

114 # __class__ for old-style classes

115 base_type = obj if _is_type(obj) else obj.__class__

116 original = None

117 # there is no .mro() for old-style classes

118 for subtype in inspect.getmro(base_type):

119 original = vars(subtype).get(name)

120 if original is not None:

121 break

122

123 # name not found in the mro

124 if original is None:

125 return False

126

127 # static methods are always fine

128 if isinstance(original, staticmethod):

129 return True

130

131 # at this point, the method has to be an instancemthod or a classmethod

132 if not isinstance(func, types.MethodType):

133 return False

134 bound_to = func.__self__

135

136 # class methods

137 if isinstance(original, classmethod):

138 return isinstance(bound_to, type) and issubclass(base_type, bound_to)

139

140 # bound methods

141 return isinstance(obj, type(bound_to))

142

143

144def _is_object(obj: Any) -> bool:

145 """Returns True is obj is a reference to an object instance.

146

147 >>> _is_object(1)

148 True

149

150 >>> _is_object(object())

151 True

152

153 >>> _is_object(lambda x: 1)

154 False

155 """

156 return isinstance(obj, object) and not isinstance(

157 obj, (type, types.FunctionType, types.BuiltinFunctionType)

158 )

159

160

161def _is_not_class(obj: Any) -> bool:

162 """Determines if the object is not a class or a class instance.

163 Used for serializing properties.

164 """

165 return type(obj) in NON_CLASS_TYPES

166

167

168def _is_primitive(obj: Any) -> bool:

169 """Helper method to see if the object is a basic data type. Unicode strings,

170 integers, longs, floats, booleans, and None are considered primitive

171 and will return True when passed into *_is_primitive()*

172

173 >>> _is_primitive(3)

174 True

175 >>> _is_primitive([4,4])

176 False

177 """

178 return type(obj) in PRIMITIVES

179

180

181def _is_enum(obj: Any) -> bool:

182 """Is the object an enum?"""

183 return "enum" in sys.modules and isinstance(obj, sys.modules["enum"].Enum)

184

185

186def _is_dictionary_subclass(obj: Any) -> bool:

187 """Returns True if *obj* is a subclass of the dict type. *obj* must be

188 a subclass and not the actual builtin dict.

189

190 >>> class Temp(dict): pass

191 >>> _is_dictionary_subclass(Temp())

192 True

193 """

194 # TODO: add UserDict

195 return (

196 hasattr(obj, "__class__")

197 and issubclass(obj.__class__, dict)

198 and type(obj) is not dict

199 )

200

201

202def _is_sequence_subclass(obj: Any) -> bool:

203 """Returns True if *obj* is a subclass of list, set or tuple.

204

205 *obj* must be a subclass and not the actual builtin, such

206 as list, set, tuple, etc..

207

208 >>> class Temp(list): pass

209 >>> _is_sequence_subclass(Temp())

210 True

211 """

212 return (

213 hasattr(obj, "__class__")

214 and issubclass(obj.__class__, SEQUENCES)

215 and type(obj) not in SEQUENCES_SET

216 )

217

218

219def _is_noncomplex(obj: Any) -> bool:

220 """Returns True if *obj* is a special (weird) class, that is more complex

221 than primitive data types, but is not a full object. Including:

222

223 * :class:`~time.struct_time`

224 """

225 return type(obj) is time.struct_time

226

227

228def _is_function(obj: Any) -> bool:

229 """Returns true if passed a function

230

231 >>> _is_function(lambda x: 1)

232 True

233

234 >>> _is_function(locals)

235 True

236

237 >>> def method(): pass

238 >>> _is_function(method)

239 True

240

241 >>> _is_function(1)

242 False

243 """

244 return type(obj) in FUNCTION_TYPES

245

246

247def _is_module_function(obj: Any) -> bool:

248 """Return True if `obj` is a module-global function

249

250 >>> import os

251 >>> _is_module_function(os.path.exists)

252 True

253

254 >>> _is_module_function(lambda: None)

255 False

256

257 """

258

259 return (

260 hasattr(obj, "__class__")

261 and isinstance(obj, (types.FunctionType, types.BuiltinFunctionType))

262 and hasattr(obj, "__module__")

263 and hasattr(obj, "__name__")

264 and obj.__name__ != "<lambda>"

265 ) or _is_cython_function(obj)

266

267

268def _is_picklable(name: str, value: types.FunctionType) -> bool:

269 """Return True if an object can be pickled

270

271 >>> import os

272 >>> _is_picklable('os', os)

273 True

274

275 >>> def foo(): pass

276 >>> _is_picklable('foo', foo)

277 True

278

279 >>> _is_picklable('foo', lambda: None)

280 False

281

282 """

283 if name in tags.RESERVED:

284 return False

285 return _is_module_function(value) or not _is_function(value)

286

287

288def _is_installed(module: str) -> bool:

289 """Tests to see if ``module`` is available on the sys.path

290

291 >>> _is_installed('sys')

292 True

293 >>> _is_installed('hopefullythisisnotarealmodule')

294 False

295

296 """

297 try:

298 __import__(module)

299 return True

300 except ImportError:

301 return False

302

303

304def _is_list_like(obj: Any) -> bool:

305 return hasattr(obj, "__getitem__") and hasattr(obj, "append")

306

307

308def _is_iterator(obj: Any) -> bool:

309 return isinstance(obj, Iterator) and not isinstance(obj, io.IOBase)

310

311

312def _is_collections(obj: Any) -> bool:

313 try:

314 return type(obj).__module__ == "collections"

315 except Exception: # ruff: ignore[BLE001]

316 return False

317

318

319def _is_reducible_sequence_subclass(obj: Any) -> bool:

320 return hasattr(obj, "__class__") and issubclass(obj.__class__, SEQUENCES)

321

322

323def _is_reducible(obj: Any) -> bool:

324 """

325 Returns false if of a type which have special casing,

326 and should not have their __reduce__ methods used

327 """

328 # defaultdicts may contain functions which we cannot serialise

329 if _is_collections(obj) and not isinstance(obj, collections.defaultdict):

330 return True

331 return not (

332 type(obj) in NON_REDUCIBLE_TYPES

333 or obj is object

334 or _is_dictionary_subclass(obj)

335 or isinstance(obj, types.ModuleType)

336 or _is_reducible_sequence_subclass(obj)

337 or _is_list_like(obj)

338 or isinstance(getattr(obj, "__slots__", None), _ITERATOR_TYPE)

339 or (_is_type(obj) and obj.__module__ == "datetime")

340 )

341

342

343def _is_cython_function(obj: Any) -> bool:

344 """Returns true if the object is a reference to a Cython function"""

345 return (

346 callable(obj)

347 and hasattr(obj, "__repr__")

348 and repr(obj).startswith("<cyfunction ")

349 )

350

351

352def _is_readonly(obj: Any, attr: str, value: Any) -> bool:

353 # CPython 3.11+ has 0-cost try/except, please use up-to-date versions!

354 try:

355 setattr(obj, attr, value)

356 return False

357 except AttributeError:

358 # this is okay, it means the attribute couldn't be set

359 return True

360 except TypeError:

361 # this should only be happening when obj is a dict

362 # as these errors happen when attr isn't a str

363 return True

364

365

366def in_dict(obj: Any, key: str, default: bool = False) -> bool:

367 """

368 Returns true if key exists in obj.__dict__; false if not in.

369 If obj.__dict__ is absent, return default

370 """

371 return (key in obj.__dict__) if getattr(obj, "__dict__", None) else default

372

373

374def in_slots(obj: Any, key: str, default: bool = False) -> bool:

375 """

376 Returns true if key exists in obj.__slots__; false if not in.

377 If obj.__slots__ is absent, return default

378 """

379 return (key in obj.__slots__) if getattr(obj, "__slots__", None) else default

380

381

382def has_reduce(obj: Any) -> tuple[bool, bool]:

383 """

384 Tests if __reduce__ or __reduce_ex__ exists in the object dict or

385 in the class dicts of every class in the MRO *except object*.

386

387 Returns a tuple of booleans (has_reduce, has_reduce_ex)

388 """

389

390 if not _is_reducible(obj) or _is_type(obj):

391 return (False, False)

392

393 # in this case, reduce works and is desired

394 # notwithstanding depending on default object

395 # reduce

396 if _is_noncomplex(obj):

397 return (False, True)

398

399 has_reduce = False

400 has_reduce_ex = False

401

402 REDUCE = "__reduce__"

403 REDUCE_EX = "__reduce_ex__"

404

405 # For object instance

406 has_reduce = in_dict(obj, REDUCE) or in_slots(obj, REDUCE)

407 has_reduce_ex = in_dict(obj, REDUCE_EX) or in_slots(obj, REDUCE_EX)

408

409 # turn to the MRO

410 for base in type(obj).__mro__:

411 if _is_reducible(base):

412 has_reduce = has_reduce or in_dict(base, REDUCE)

413 has_reduce_ex = has_reduce_ex or in_dict(base, REDUCE_EX)

414 if has_reduce and has_reduce_ex:

415 return (has_reduce, has_reduce_ex)

416

417 # for things that don't have a proper dict but can be

418 # getattred (rare, but includes some builtins)

419 cls = type(obj)

420 object_reduce = getattr(object, REDUCE)

421 object_reduce_ex = getattr(object, REDUCE_EX)

422 if not has_reduce:

423 has_reduce_cls = getattr(cls, REDUCE, False)

424 if has_reduce_cls is not object_reduce:

425 has_reduce = has_reduce_cls

426

427 if not has_reduce_ex:

428 has_reduce_ex_cls = getattr(cls, REDUCE_EX, False)

429 if has_reduce_ex_cls is not object_reduce_ex:

430 has_reduce_ex = has_reduce_ex_cls

431

432 return (has_reduce, has_reduce_ex)

433

434

435def translate_module_name(module: str) -> str:

436 """Rename builtin modules to a consistent module name.

437

438 Prefer the more modern naming.

439

440 This is used so that references to Python's `builtins` module can

441 be loaded in both Python 2 and 3. We remap to the "__builtin__"

442 name and unmap it when importing.

443

444 Map the Python2 `exceptions` module to `builtins` because

445 `builtins` is a superset and contains everything that is

446 available in `exceptions`, which makes the translation simpler.

447

448 See untranslate_module_name() for the reverse operation.

449 """

450 lookup = {"__builtin__": "builtins", "exceptions": "builtins"}

451 return lookup.get(module, module)

452

453

454def _0_9_6_compat_untranslate(module: str) -> str:

455 """Provide compatibility for pickles created with jsonpickle 0.9.6 and

456 earlier, remapping `exceptions` and `__builtin__` to `builtins`.

457 """

458 lookup = {"__builtin__": "builtins", "exceptions": "builtins"}

459 return lookup.get(module, module)

460

461

462def untranslate_module_name(module: str) -> str:

463 """Rename module names mention in JSON to names that we can import

464

465 This reverses the translation applied by translate_module_name() to

466 a module name available to the current version of Python.

467

468 """

469 return _0_9_6_compat_untranslate(module)

470

471

472def importable_name(cls: type | Callable[..., Any]) -> str:

473 """

474 >>> class Example(object):

475 ... pass

476

477 >>> ex = Example()

478 >>> importable_name(ex.__class__) == 'jsonpickle.util.Example'

479 True

480 >>> importable_name(type(25)) == 'builtins.int'

481 True

482 >>> importable_name(object().__str__.__class__) == 'types.MethodWrapperType'

483 True

484 >>> importable_name(False.__class__) == 'builtins.bool'

485 True

486 >>> importable_name(AttributeError) == 'builtins.AttributeError'

487 True

488 >>> import argparse

489 >>> importable_name(type(argparse.ArgumentParser().add_argument)) == 'types.MethodType'

490 True

491

492 """

493 types_importable_name = _TYPES_IMPORTABLE_NAMES.get(cls)

494 if types_importable_name is not None:

495 return types_importable_name

496

497 # Use the fully-qualified name if available (Python >= 3.3)

498 name = getattr(cls, "__qualname__", cls.__name__)

499 module = translate_module_name(cls.__module__)

500 if not module and hasattr(cls, "__self__"):

501 if hasattr(cls.__self__, "__module__"):

502 module = cls.__self__.__module__

503 else:

504 module = cls.__self__.__class__.__module__

505 return f"{module}.{name}"

506

507

508def _getattr_path(obj: Any, path: str) -> Any:

509 """Follow a dotted attribute path, returning _MISSING if any part is missing"""

510 for name in path.split("."):

511 try:

512 obj = getattr(obj, name)

513 except Exception: # ruff: ignore[BLE001]

514 # a module's __getattr__ can raise anything

515 return _MISSING

516 return obj

517

518

519def _find_global_module(obj: Any, name: str) -> str | None:

520 """Find the module that obj can be imported from as name

521

522 This is similar to pickle.whichmodule except that it prefers the public package that

523 re-exports obj over the module obj is defined in. That module is often

524 private and gets renamed between releases (scipy.special.expit is in

525 scipy.special._ufuncs before scipy 1.15 and scipy.special._special_ufuncs

526 after), which would make the output unreadable by other versions.

527 """

528 key = (id(obj), name)

529 module = _GLOBAL_MODULES.get(key)

530 # ids are reused once an object is freed, so check the cached module still has obj

531 if module is not None:

532 module_obj = sys.modules.get(module)

533 if module_obj is not None and _getattr_path(module_obj, name) is obj:

534 return module

535 holders = [

536 module_name

537 for module_name, module_obj in list(sys.modules.items())

538 if module_obj is not None and _getattr_path(module_obj, name) is obj

539 ]

540 if not holders:

541 return None

542

543 def rank(module_name: str) -> tuple[bool, bool, bool, int, str]:

544 return (

545 # the running script, which other processes can't import, goes last

546 module_name in _MAIN_MODULES,

547 # then public modules before private ones

548 any(part.startswith("_") for part in module_name.split(".")),

549 # then the library's own packages before modules that only import obj

550 not any(holder.startswith(f"{module_name}.") for holder in holders),

551 module_name.count("."),

552 module_name,

553 )

554

555 module = min(holders, key=rank)

556 if len(_GLOBAL_MODULES) >= 256:

557 _GLOBAL_MODULES.clear()

558 _GLOBAL_MODULES[key] = module

559 return module

560

561

562def b64encode(data: bytes) -> str:

563 """

564 Encode binary data to ascii text in base64. Data must be bytes.

565 """

566 return base64.b64encode(data).decode("ascii")

567

568

569def b64decode(payload: str) -> bytes:

570 """

571 Decode payload - must be ascii text.

572 """

573 try:

574 return base64.b64decode(payload, validate=True)

575 except (TypeError, binascii.Error) as error:

576 # we can't tell the difference between an empty result and an empty payload

577 # so we warn rather than discarding the data silently

578 warnings.warn(f"jsonpickle could not decode base64 payload: {error}")

579 return b""

580

581

582def b85encode(data: bytes) -> str:

583 """

584 Encode binary data to ascii text in base85. Data must be bytes.

585 """

586 return base64.b85encode(data).decode("ascii")

587

588

589def b85decode(payload: bytes) -> bytes:

590 """

591 Decode payload - must be ascii text.

592 """

593 try:

594 return base64.b85decode(payload)

595 except (TypeError, ValueError) as error:

596 # we can't tell the difference between an empty result and an empty payload

597 # so we warn rather than discarding the data silently

598 warnings.warn(f"jsonpickle could not decode base85 payload: {error}")

599 return b""

600

601

602def itemgetter(

603 obj: Any,

604 getter: Callable[[Any], Any] = operator.itemgetter(0),

605) -> str:

606 return str(getter(obj))

607

608

609def items(

610 obj: dict[Any, Any],

611 exclude: Iterable[Any] = (),

612) -> Iterator[tuple[Any, Any]]:

613 """

614 This can't be easily replaced by dict.items() because this has the exclude parameter.

615 Keep it for now.

616 """

617 for k, v in obj.items():

618 if k in exclude:

619 continue

620 yield k, v

621

622

623def loadclass(

624 module_and_name: str, classes: dict[str, type] | None = None

625) -> Any | None:

626 """Loads the module and returns the class.

627

628 >>> cls = loadclass('datetime.datetime')

629 >>> cls.__name__

630 'datetime'

631

632 >>> loadclass('does.not.exist')

633

634 >>> loadclass('builtins.int')()

635 0

636

637 """

638 # Check if the class exists in a caller-provided scope

639 if classes:

640 try:

641 return classes[module_and_name]

642 except KeyError:

643 # maybe they didn't provide a fully qualified path

644 try:

645 return classes[module_and_name.rsplit(".", 1)[-1]]

646 except KeyError:

647 pass

648 # Otherwise, load classes from globally-accessible imports

649 names = module_and_name.split(".")

650 # First assume that everything up to the last dot is the module name,

651 # then try other splits to handle classes that are defined within

652 # classes

653 for up_to in range(len(names) - 1, 0, -1):

654 module = untranslate_module_name(".".join(names[:up_to]))

655 try:

656 __import__(module)

657 obj = sys.modules[module]

658 for class_name in names[up_to:]:

659 obj = getattr(obj, class_name)

660 return obj

661 except (AttributeError, ImportError, ValueError):

662 continue

663 # NoneType is a special case and can not be imported/created

664 if module_and_name == "builtins.NoneType":

665 return type(None)

666 return None