Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/jsonpickle/util.py: 50%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# Copyright (C) 2008 John Paulett (john -at- paulett.org)
2# Copyright (C) 2009-2018 David Aguilar (davvid -at- gmail.com)
3# All rights reserved.
4#
5# This software is licensed as described in the file COPYING, which
6# you should have received as part of this distribution.
8"""Helper functions for pickling and unpickling. Most functions assist in
9determining the type of an object.
10"""
12import base64
13import binascii
14import collections
15import inspect
16import io
17import operator
18import sys
19import time
20import types
21import warnings
22from collections.abc import Callable, Iterable, Iterator
23from typing import Any, TypeVar
25from . import tags
27# key
28K = TypeVar("K")
29# value
30V = TypeVar("V")
31# type
32T = TypeVar("T")
34_ITERATOR_TYPE: type = type(iter(""))
35# Protocol used when asking objects to reduce themselves. 4 is the lowest
36# protocol that encodes datetime's PEP 495 fold bit; 2 silently drops it.
37PICKLE_PROTOCOL: int = 4
38SEQUENCES: tuple[type] = (list, set, tuple) # type: ignore[assignment]
39SEQUENCES_SET: set[type] = {list, set, tuple}
40PRIMITIVES: set[type] = {str, bool, int, float, type(None)}
41FUNCTION_TYPES: set[type] = {
42 types.FunctionType,
43 types.MethodType,
44 types.LambdaType,
45 types.BuiltinFunctionType,
46 types.BuiltinMethodType,
47}
48# Internal set for NON_REDUCIBLE_TYPES that excludes MethodType to allow method round-trip
49_NON_REDUCIBLE_FUNCTION_TYPES: set[type] = FUNCTION_TYPES - {types.MethodType}
50NON_REDUCIBLE_TYPES: set[type] = (
51 {
52 list,
53 dict,
54 set,
55 tuple,
56 object,
57 bytes,
58 }
59 | PRIMITIVES
60 | _NON_REDUCIBLE_FUNCTION_TYPES
61)
62NON_CLASS_TYPES: set[type] = {
63 list,
64 dict,
65 set,
66 tuple,
67 bytes,
68} | PRIMITIVES
69_TYPES_IMPORTABLE_NAMES: dict[type | Callable[..., Any], str] = {
70 getattr(types, name): f"types.{name}"
71 for name in types.__all__
72 if name.endswith("Type")
73}
76def _is_type(obj: Any) -> bool:
77 """Returns True is obj is a reference to a type.
79 >>> _is_type(1)
80 False
82 >>> _is_type(object)
83 True
85 >>> class Klass: pass
86 >>> _is_type(Klass)
87 True
88 """
89 # use "isinstance" and not "is" to allow for metaclasses
90 return isinstance(obj, type)
93def has_method(obj: Any, name: str) -> bool:
94 # false if attribute doesn't exist
95 if not hasattr(obj, name):
96 return False
97 func = getattr(obj, name)
99 # builtin descriptors like __getnewargs__
100 if isinstance(func, types.BuiltinMethodType):
101 return True
103 # note that FunctionType has a different meaning in py2/py3
104 if not isinstance(func, (types.MethodType, types.FunctionType)):
105 return False
107 # need to go through __dict__'s since in py3
108 # methods are essentially descriptors
110 # __class__ for old-style classes
111 base_type = obj if _is_type(obj) else obj.__class__
112 original = None
113 # there is no .mro() for old-style classes
114 for subtype in inspect.getmro(base_type):
115 original = vars(subtype).get(name)
116 if original is not None:
117 break
119 # name not found in the mro
120 if original is None:
121 return False
123 # static methods are always fine
124 if isinstance(original, staticmethod):
125 return True
127 # at this point, the method has to be an instancemthod or a classmethod
128 if not isinstance(func, types.MethodType):
129 return False
130 bound_to = func.__self__
132 # class methods
133 if isinstance(original, classmethod):
134 return isinstance(bound_to, type) and issubclass(base_type, bound_to)
136 # bound methods
137 return isinstance(obj, type(bound_to))
140def _is_object(obj: Any) -> bool:
141 """Returns True is obj is a reference to an object instance.
143 >>> _is_object(1)
144 True
146 >>> _is_object(object())
147 True
149 >>> _is_object(lambda x: 1)
150 False
151 """
152 return isinstance(obj, object) and not isinstance(
153 obj, (type, types.FunctionType, types.BuiltinFunctionType)
154 )
157def _is_not_class(obj: Any) -> bool:
158 """Determines if the object is not a class or a class instance.
159 Used for serializing properties.
160 """
161 return type(obj) in NON_CLASS_TYPES
164def _is_primitive(obj: Any) -> bool:
165 """Helper method to see if the object is a basic data type. Unicode strings,
166 integers, longs, floats, booleans, and None are considered primitive
167 and will return True when passed into *_is_primitive()*
169 >>> _is_primitive(3)
170 True
171 >>> _is_primitive([4,4])
172 False
173 """
174 return type(obj) in PRIMITIVES
177def _is_enum(obj: Any) -> bool:
178 """Is the object an enum?"""
179 return "enum" in sys.modules and isinstance(obj, sys.modules["enum"].Enum)
182def _is_dictionary_subclass(obj: Any) -> bool:
183 """Returns True if *obj* is a subclass of the dict type. *obj* must be
184 a subclass and not the actual builtin dict.
186 >>> class Temp(dict): pass
187 >>> _is_dictionary_subclass(Temp())
188 True
189 """
190 # TODO: add UserDict
191 return (
192 hasattr(obj, "__class__")
193 and issubclass(obj.__class__, dict)
194 and type(obj) is not dict
195 )
198def _is_sequence_subclass(obj: Any) -> bool:
199 """Returns True if *obj* is a subclass of list, set or tuple.
201 *obj* must be a subclass and not the actual builtin, such
202 as list, set, tuple, etc..
204 >>> class Temp(list): pass
205 >>> _is_sequence_subclass(Temp())
206 True
207 """
208 return (
209 hasattr(obj, "__class__")
210 and issubclass(obj.__class__, SEQUENCES)
211 and type(obj) not in SEQUENCES_SET
212 )
215def _is_noncomplex(obj: Any) -> bool:
216 """Returns True if *obj* is a special (weird) class, that is more complex
217 than primitive data types, but is not a full object. Including:
219 * :class:`~time.struct_time`
220 """
221 return type(obj) is time.struct_time
224def _is_function(obj: Any) -> bool:
225 """Returns true if passed a function
227 >>> _is_function(lambda x: 1)
228 True
230 >>> _is_function(locals)
231 True
233 >>> def method(): pass
234 >>> _is_function(method)
235 True
237 >>> _is_function(1)
238 False
239 """
240 return type(obj) in FUNCTION_TYPES
243def _is_module_function(obj: Any) -> bool:
244 """Return True if `obj` is a module-global function
246 >>> import os
247 >>> _is_module_function(os.path.exists)
248 True
250 >>> _is_module_function(lambda: None)
251 False
253 """
255 return (
256 hasattr(obj, "__class__")
257 and isinstance(obj, (types.FunctionType, types.BuiltinFunctionType))
258 and hasattr(obj, "__module__")
259 and hasattr(obj, "__name__")
260 and obj.__name__ != "<lambda>"
261 ) or _is_cython_function(obj)
264def _is_picklable(name: str, value: types.FunctionType) -> bool:
265 """Return True if an object can be pickled
267 >>> import os
268 >>> _is_picklable('os', os)
269 True
271 >>> def foo(): pass
272 >>> _is_picklable('foo', foo)
273 True
275 >>> _is_picklable('foo', lambda: None)
276 False
278 """
279 if name in tags.RESERVED:
280 return False
281 return _is_module_function(value) or not _is_function(value)
284def _is_installed(module: str) -> bool:
285 """Tests to see if ``module`` is available on the sys.path
287 >>> _is_installed('sys')
288 True
289 >>> _is_installed('hopefullythisisnotarealmodule')
290 False
292 """
293 try:
294 __import__(module)
295 return True
296 except ImportError:
297 return False
300def _is_list_like(obj: Any) -> bool:
301 return hasattr(obj, "__getitem__") and hasattr(obj, "append")
304def _is_iterator(obj: Any) -> bool:
305 return isinstance(obj, Iterator) and not isinstance(obj, io.IOBase)
308def _is_collections(obj: Any) -> bool:
309 try:
310 return type(obj).__module__ == "collections"
311 except Exception: # ruff: ignore[BLE001]
312 return False
315def _is_reducible_sequence_subclass(obj: Any) -> bool:
316 return hasattr(obj, "__class__") and issubclass(obj.__class__, SEQUENCES)
319def _is_reducible(obj: Any) -> bool:
320 """
321 Returns false if of a type which have special casing,
322 and should not have their __reduce__ methods used
323 """
324 # defaultdicts may contain functions which we cannot serialise
325 if _is_collections(obj) and not isinstance(obj, collections.defaultdict):
326 return True
327 return not (
328 type(obj) in NON_REDUCIBLE_TYPES
329 or obj is object
330 or _is_dictionary_subclass(obj)
331 or isinstance(obj, types.ModuleType)
332 or _is_reducible_sequence_subclass(obj)
333 or _is_list_like(obj)
334 or isinstance(getattr(obj, "__slots__", None), _ITERATOR_TYPE)
335 or (_is_type(obj) and obj.__module__ == "datetime")
336 )
339def _is_cython_function(obj: Any) -> bool:
340 """Returns true if the object is a reference to a Cython function"""
341 return (
342 callable(obj)
343 and hasattr(obj, "__repr__")
344 and repr(obj).startswith("<cyfunction ")
345 )
348def _is_readonly(obj: Any, attr: str, value: Any) -> bool:
349 # CPython 3.11+ has 0-cost try/except, please use up-to-date versions!
350 try:
351 setattr(obj, attr, value)
352 return False
353 except AttributeError:
354 # this is okay, it means the attribute couldn't be set
355 return True
356 except TypeError:
357 # this should only be happening when obj is a dict
358 # as these errors happen when attr isn't a str
359 return True
362def in_dict(obj: Any, key: str, default: bool = False) -> bool:
363 """
364 Returns true if key exists in obj.__dict__; false if not in.
365 If obj.__dict__ is absent, return default
366 """
367 return (key in obj.__dict__) if getattr(obj, "__dict__", None) else default
370def in_slots(obj: Any, key: str, default: bool = False) -> bool:
371 """
372 Returns true if key exists in obj.__slots__; false if not in.
373 If obj.__slots__ is absent, return default
374 """
375 return (key in obj.__slots__) if getattr(obj, "__slots__", None) else default
378def has_reduce(obj: Any) -> tuple[bool, bool]:
379 """
380 Tests if __reduce__ or __reduce_ex__ exists in the object dict or
381 in the class dicts of every class in the MRO *except object*.
383 Returns a tuple of booleans (has_reduce, has_reduce_ex)
384 """
386 if not _is_reducible(obj) or _is_type(obj):
387 return (False, False)
389 # in this case, reduce works and is desired
390 # notwithstanding depending on default object
391 # reduce
392 if _is_noncomplex(obj):
393 return (False, True)
395 has_reduce = False
396 has_reduce_ex = False
398 REDUCE = "__reduce__"
399 REDUCE_EX = "__reduce_ex__"
401 # For object instance
402 has_reduce = in_dict(obj, REDUCE) or in_slots(obj, REDUCE)
403 has_reduce_ex = in_dict(obj, REDUCE_EX) or in_slots(obj, REDUCE_EX)
405 # turn to the MRO
406 for base in type(obj).__mro__:
407 if _is_reducible(base):
408 has_reduce = has_reduce or in_dict(base, REDUCE)
409 has_reduce_ex = has_reduce_ex or in_dict(base, REDUCE_EX)
410 if has_reduce and has_reduce_ex:
411 return (has_reduce, has_reduce_ex)
413 # for things that don't have a proper dict but can be
414 # getattred (rare, but includes some builtins)
415 cls = type(obj)
416 object_reduce = getattr(object, REDUCE)
417 object_reduce_ex = getattr(object, REDUCE_EX)
418 if not has_reduce:
419 has_reduce_cls = getattr(cls, REDUCE, False)
420 if has_reduce_cls is not object_reduce:
421 has_reduce = has_reduce_cls
423 if not has_reduce_ex:
424 has_reduce_ex_cls = getattr(cls, REDUCE_EX, False)
425 if has_reduce_ex_cls is not object_reduce_ex:
426 has_reduce_ex = has_reduce_ex_cls
428 return (has_reduce, has_reduce_ex)
431def translate_module_name(module: str) -> str:
432 """Rename builtin modules to a consistent module name.
434 Prefer the more modern naming.
436 This is used so that references to Python's `builtins` module can
437 be loaded in both Python 2 and 3. We remap to the "__builtin__"
438 name and unmap it when importing.
440 Map the Python2 `exceptions` module to `builtins` because
441 `builtins` is a superset and contains everything that is
442 available in `exceptions`, which makes the translation simpler.
444 See untranslate_module_name() for the reverse operation.
445 """
446 lookup = {"__builtin__": "builtins", "exceptions": "builtins"}
447 return lookup.get(module, module)
450def _0_9_6_compat_untranslate(module: str) -> str:
451 """Provide compatibility for pickles created with jsonpickle 0.9.6 and
452 earlier, remapping `exceptions` and `__builtin__` to `builtins`.
453 """
454 lookup = {"__builtin__": "builtins", "exceptions": "builtins"}
455 return lookup.get(module, module)
458def untranslate_module_name(module: str) -> str:
459 """Rename module names mention in JSON to names that we can import
461 This reverses the translation applied by translate_module_name() to
462 a module name available to the current version of Python.
464 """
465 return _0_9_6_compat_untranslate(module)
468def importable_name(cls: type | Callable[..., Any]) -> str:
469 """
470 >>> class Example(object):
471 ... pass
473 >>> ex = Example()
474 >>> importable_name(ex.__class__) == 'jsonpickle.util.Example'
475 True
476 >>> importable_name(type(25)) == 'builtins.int'
477 True
478 >>> importable_name(object().__str__.__class__) == 'types.MethodWrapperType'
479 True
480 >>> importable_name(False.__class__) == 'builtins.bool'
481 True
482 >>> importable_name(AttributeError) == 'builtins.AttributeError'
483 True
484 >>> import argparse
485 >>> importable_name(type(argparse.ArgumentParser().add_argument)) == 'types.MethodType'
486 True
488 """
489 types_importable_name = _TYPES_IMPORTABLE_NAMES.get(cls)
490 if types_importable_name is not None:
491 return types_importable_name
493 # Use the fully-qualified name if available (Python >= 3.3)
494 name = getattr(cls, "__qualname__", cls.__name__)
495 module = translate_module_name(cls.__module__)
496 if not module and hasattr(cls, "__self__"):
497 if hasattr(cls.__self__, "__module__"):
498 module = cls.__self__.__module__
499 else:
500 module = cls.__self__.__class__.__module__
501 return f"{module}.{name}"
504def b64encode(data: bytes) -> str:
505 """
506 Encode binary data to ascii text in base64. Data must be bytes.
507 """
508 return base64.b64encode(data).decode("ascii")
511def b64decode(payload: str) -> bytes:
512 """
513 Decode payload - must be ascii text.
514 """
515 try:
516 return base64.b64decode(payload, validate=True)
517 except (TypeError, binascii.Error) as error:
518 # we can't tell the difference between an empty result and an empty payload
519 # so we warn rather than discarding the data silently
520 warnings.warn(f"jsonpickle could not decode base64 payload: {error}")
521 return b""
524def b85encode(data: bytes) -> str:
525 """
526 Encode binary data to ascii text in base85. Data must be bytes.
527 """
528 return base64.b85encode(data).decode("ascii")
531def b85decode(payload: bytes) -> bytes:
532 """
533 Decode payload - must be ascii text.
534 """
535 try:
536 return base64.b85decode(payload)
537 except (TypeError, ValueError) as error:
538 # we can't tell the difference between an empty result and an empty payload
539 # so we warn rather than discarding the data silently
540 warnings.warn(f"jsonpickle could not decode base85 payload: {error}")
541 return b""
544def itemgetter(
545 obj: Any,
546 getter: Callable[[Any], Any] = operator.itemgetter(0),
547) -> str:
548 return str(getter(obj))
551def items(
552 obj: dict[Any, Any],
553 exclude: Iterable[Any] = (),
554) -> Iterator[tuple[Any, Any]]:
555 """
556 This can't be easily replaced by dict.items() because this has the exclude parameter.
557 Keep it for now.
558 """
559 for k, v in obj.items():
560 if k in exclude:
561 continue
562 yield k, v
565def loadclass(
566 module_and_name: str, classes: dict[str, type] | None = None
567) -> Any | None:
568 """Loads the module and returns the class.
570 >>> cls = loadclass('datetime.datetime')
571 >>> cls.__name__
572 'datetime'
574 >>> loadclass('does.not.exist')
576 >>> loadclass('builtins.int')()
577 0
579 """
580 # Check if the class exists in a caller-provided scope
581 if classes:
582 try:
583 return classes[module_and_name]
584 except KeyError:
585 # maybe they didn't provide a fully qualified path
586 try:
587 return classes[module_and_name.rsplit(".", 1)[-1]]
588 except KeyError:
589 pass
590 # Otherwise, load classes from globally-accessible imports
591 names = module_and_name.split(".")
592 # First assume that everything up to the last dot is the module name,
593 # then try other splits to handle classes that are defined within
594 # classes
595 for up_to in range(len(names) - 1, 0, -1):
596 module = untranslate_module_name(".".join(names[:up_to]))
597 try:
598 __import__(module)
599 obj = sys.modules[module]
600 for class_name in names[up_to:]:
601 obj = getattr(obj, class_name)
602 return obj
603 except (AttributeError, ImportError, ValueError):
604 continue
605 # NoneType is a special case and can not be imported/created
606 if module_and_name == "builtins.NoneType":
607 return type(None)
608 return None