Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/jsonpickle/util.py: 50%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# Copyright (C) 2008 John Paulett (john -at- paulett.org)
2# Copyright (C) 2009-2018 David Aguilar (davvid -at- gmail.com)
3# All rights reserved.
4#
5# This software is licensed as described in the file COPYING, which
6# you should have received as part of this distribution.
8"""Helper functions for pickling and unpickling. Most functions assist in
9determining the type of an object.
10"""
12import base64
13import binascii
14import collections
15import inspect
16import io
17import operator
18import sys
19import time
20import types
21from collections.abc import Callable, Iterable, Iterator
22from typing import Any, TypeVar
24from . import tags
26# key
27K = TypeVar("K")
28# value
29V = TypeVar("V")
30# type
31T = TypeVar("T")
33_ITERATOR_TYPE: type = type(iter(""))
34# Protocol used when asking objects to reduce themselves. 4 is the lowest
35# protocol that encodes datetime's PEP 495 fold bit; 2 silently drops it.
36PICKLE_PROTOCOL: int = 4
37SEQUENCES: tuple[type] = (list, set, tuple) # type: ignore[assignment]
38SEQUENCES_SET: set[type] = {list, set, tuple}
39PRIMITIVES: set[type] = {str, bool, int, float, type(None)}
40FUNCTION_TYPES: set[type] = {
41 types.FunctionType,
42 types.MethodType,
43 types.LambdaType,
44 types.BuiltinFunctionType,
45 types.BuiltinMethodType,
46}
47# Internal set for NON_REDUCIBLE_TYPES that excludes MethodType to allow method round-trip
48_NON_REDUCIBLE_FUNCTION_TYPES: set[type] = FUNCTION_TYPES - {types.MethodType}
49NON_REDUCIBLE_TYPES: set[type] = (
50 {
51 list,
52 dict,
53 set,
54 tuple,
55 object,
56 bytes,
57 }
58 | PRIMITIVES
59 | _NON_REDUCIBLE_FUNCTION_TYPES
60)
61NON_CLASS_TYPES: set[type] = {
62 list,
63 dict,
64 set,
65 tuple,
66 bytes,
67} | PRIMITIVES
68_TYPES_IMPORTABLE_NAMES: dict[type | Callable[..., Any], str] = {
69 getattr(types, name): f"types.{name}"
70 for name in types.__all__
71 if name.endswith("Type")
72}
75def _is_type(obj: Any) -> bool:
76 """Returns True is obj is a reference to a type.
78 >>> _is_type(1)
79 False
81 >>> _is_type(object)
82 True
84 >>> class Klass: pass
85 >>> _is_type(Klass)
86 True
87 """
88 # use "isinstance" and not "is" to allow for metaclasses
89 return isinstance(obj, type)
92def has_method(obj: Any, name: str) -> bool:
93 # false if attribute doesn't exist
94 if not hasattr(obj, name):
95 return False
96 func = getattr(obj, name)
98 # builtin descriptors like __getnewargs__
99 if isinstance(func, types.BuiltinMethodType):
100 return True
102 # note that FunctionType has a different meaning in py2/py3
103 if not isinstance(func, (types.MethodType, types.FunctionType)):
104 return False
106 # need to go through __dict__'s since in py3
107 # methods are essentially descriptors
109 # __class__ for old-style classes
110 base_type = obj if _is_type(obj) else obj.__class__
111 original = None
112 # there is no .mro() for old-style classes
113 for subtype in inspect.getmro(base_type):
114 original = vars(subtype).get(name)
115 if original is not None:
116 break
118 # name not found in the mro
119 if original is None:
120 return False
122 # static methods are always fine
123 if isinstance(original, staticmethod):
124 return True
126 # at this point, the method has to be an instancemthod or a classmethod
127 if not isinstance(func, types.MethodType):
128 return False
129 bound_to = func.__self__
131 # class methods
132 if isinstance(original, classmethod):
133 return isinstance(bound_to, type) and issubclass(base_type, bound_to)
135 # bound methods
136 return isinstance(obj, type(bound_to))
139def _is_object(obj: Any) -> bool:
140 """Returns True is obj is a reference to an object instance.
142 >>> _is_object(1)
143 True
145 >>> _is_object(object())
146 True
148 >>> _is_object(lambda x: 1)
149 False
150 """
151 return isinstance(obj, object) and not isinstance(
152 obj, (type, types.FunctionType, types.BuiltinFunctionType)
153 )
156def _is_not_class(obj: Any) -> bool:
157 """Determines if the object is not a class or a class instance.
158 Used for serializing properties.
159 """
160 return type(obj) in NON_CLASS_TYPES
163def _is_primitive(obj: Any) -> bool:
164 """Helper method to see if the object is a basic data type. Unicode strings,
165 integers, longs, floats, booleans, and None are considered primitive
166 and will return True when passed into *_is_primitive()*
168 >>> _is_primitive(3)
169 True
170 >>> _is_primitive([4,4])
171 False
172 """
173 return type(obj) in PRIMITIVES
176def _is_enum(obj: Any) -> bool:
177 """Is the object an enum?"""
178 return "enum" in sys.modules and isinstance(obj, sys.modules["enum"].Enum)
181def _is_dictionary_subclass(obj: Any) -> bool:
182 """Returns True if *obj* is a subclass of the dict type. *obj* must be
183 a subclass and not the actual builtin dict.
185 >>> class Temp(dict): pass
186 >>> _is_dictionary_subclass(Temp())
187 True
188 """
189 # TODO: add UserDict
190 return (
191 hasattr(obj, "__class__")
192 and issubclass(obj.__class__, dict)
193 and type(obj) is not dict
194 )
197def _is_sequence_subclass(obj: Any) -> bool:
198 """Returns True if *obj* is a subclass of list, set or tuple.
200 *obj* must be a subclass and not the actual builtin, such
201 as list, set, tuple, etc..
203 >>> class Temp(list): pass
204 >>> _is_sequence_subclass(Temp())
205 True
206 """
207 return (
208 hasattr(obj, "__class__")
209 and issubclass(obj.__class__, SEQUENCES)
210 and type(obj) not in SEQUENCES_SET
211 )
214def _is_noncomplex(obj: Any) -> bool:
215 """Returns True if *obj* is a special (weird) class, that is more complex
216 than primitive data types, but is not a full object. Including:
218 * :class:`~time.struct_time`
219 """
220 return type(obj) is time.struct_time
223def _is_function(obj: Any) -> bool:
224 """Returns true if passed a function
226 >>> _is_function(lambda x: 1)
227 True
229 >>> _is_function(locals)
230 True
232 >>> def method(): pass
233 >>> _is_function(method)
234 True
236 >>> _is_function(1)
237 False
238 """
239 return type(obj) in FUNCTION_TYPES
242def _is_module_function(obj: Any) -> bool:
243 """Return True if `obj` is a module-global function
245 >>> import os
246 >>> _is_module_function(os.path.exists)
247 True
249 >>> _is_module_function(lambda: None)
250 False
252 """
254 return (
255 hasattr(obj, "__class__")
256 and isinstance(obj, (types.FunctionType, types.BuiltinFunctionType))
257 and hasattr(obj, "__module__")
258 and hasattr(obj, "__name__")
259 and obj.__name__ != "<lambda>"
260 ) or _is_cython_function(obj)
263def _is_picklable(name: str, value: types.FunctionType) -> bool:
264 """Return True if an object can be pickled
266 >>> import os
267 >>> _is_picklable('os', os)
268 True
270 >>> def foo(): pass
271 >>> _is_picklable('foo', foo)
272 True
274 >>> _is_picklable('foo', lambda: None)
275 False
277 """
278 if name in tags.RESERVED:
279 return False
280 return _is_module_function(value) or not _is_function(value)
283def _is_installed(module: str) -> bool:
284 """Tests to see if ``module`` is available on the sys.path
286 >>> _is_installed('sys')
287 True
288 >>> _is_installed('hopefullythisisnotarealmodule')
289 False
291 """
292 try:
293 __import__(module)
294 return True
295 except ImportError:
296 return False
299def _is_list_like(obj: Any) -> bool:
300 return hasattr(obj, "__getitem__") and hasattr(obj, "append")
303def _is_iterator(obj: Any) -> bool:
304 return isinstance(obj, Iterator) and not isinstance(obj, io.IOBase)
307def _is_collections(obj: Any) -> bool:
308 try:
309 return type(obj).__module__ == "collections"
310 except Exception: # ruff: ignore[BLE001]
311 return False
314def _is_reducible_sequence_subclass(obj: Any) -> bool:
315 return hasattr(obj, "__class__") and issubclass(obj.__class__, SEQUENCES)
318def _is_reducible(obj: Any) -> bool:
319 """
320 Returns false if of a type which have special casing,
321 and should not have their __reduce__ methods used
322 """
323 # defaultdicts may contain functions which we cannot serialise
324 if _is_collections(obj) and not isinstance(obj, collections.defaultdict):
325 return True
326 return not (
327 type(obj) in NON_REDUCIBLE_TYPES
328 or obj is object
329 or _is_dictionary_subclass(obj)
330 or isinstance(obj, types.ModuleType)
331 or _is_reducible_sequence_subclass(obj)
332 or _is_list_like(obj)
333 or isinstance(getattr(obj, "__slots__", None), _ITERATOR_TYPE)
334 or (_is_type(obj) and obj.__module__ == "datetime")
335 )
338def _is_cython_function(obj: Any) -> bool:
339 """Returns true if the object is a reference to a Cython function"""
340 return (
341 callable(obj)
342 and hasattr(obj, "__repr__")
343 and repr(obj).startswith("<cyfunction ")
344 )
347def _is_readonly(obj: Any, attr: str, value: Any) -> bool:
348 # CPython 3.11+ has 0-cost try/except, please use up-to-date versions!
349 try:
350 setattr(obj, attr, value)
351 return False
352 except AttributeError:
353 # this is okay, it means the attribute couldn't be set
354 return True
355 except TypeError:
356 # this should only be happening when obj is a dict
357 # as these errors happen when attr isn't a str
358 return True
361def in_dict(obj: Any, key: str, default: bool = False) -> bool:
362 """
363 Returns true if key exists in obj.__dict__; false if not in.
364 If obj.__dict__ is absent, return default
365 """
366 return (key in obj.__dict__) if getattr(obj, "__dict__", None) else default
369def in_slots(obj: Any, key: str, default: bool = False) -> bool:
370 """
371 Returns true if key exists in obj.__slots__; false if not in.
372 If obj.__slots__ is absent, return default
373 """
374 return (key in obj.__slots__) if getattr(obj, "__slots__", None) else default
377def has_reduce(obj: Any) -> tuple[bool, bool]:
378 """
379 Tests if __reduce__ or __reduce_ex__ exists in the object dict or
380 in the class dicts of every class in the MRO *except object*.
382 Returns a tuple of booleans (has_reduce, has_reduce_ex)
383 """
385 if not _is_reducible(obj) or _is_type(obj):
386 return (False, False)
388 # in this case, reduce works and is desired
389 # notwithstanding depending on default object
390 # reduce
391 if _is_noncomplex(obj):
392 return (False, True)
394 has_reduce = False
395 has_reduce_ex = False
397 REDUCE = "__reduce__"
398 REDUCE_EX = "__reduce_ex__"
400 # For object instance
401 has_reduce = in_dict(obj, REDUCE) or in_slots(obj, REDUCE)
402 has_reduce_ex = in_dict(obj, REDUCE_EX) or in_slots(obj, REDUCE_EX)
404 # turn to the MRO
405 for base in type(obj).__mro__:
406 if _is_reducible(base):
407 has_reduce = has_reduce or in_dict(base, REDUCE)
408 has_reduce_ex = has_reduce_ex or in_dict(base, REDUCE_EX)
409 if has_reduce and has_reduce_ex:
410 return (has_reduce, has_reduce_ex)
412 # for things that don't have a proper dict but can be
413 # getattred (rare, but includes some builtins)
414 cls = type(obj)
415 object_reduce = getattr(object, REDUCE)
416 object_reduce_ex = getattr(object, REDUCE_EX)
417 if not has_reduce:
418 has_reduce_cls = getattr(cls, REDUCE, False)
419 if has_reduce_cls is not object_reduce:
420 has_reduce = has_reduce_cls
422 if not has_reduce_ex:
423 has_reduce_ex_cls = getattr(cls, REDUCE_EX, False)
424 if has_reduce_ex_cls is not object_reduce_ex:
425 has_reduce_ex = has_reduce_ex_cls
427 return (has_reduce, has_reduce_ex)
430def translate_module_name(module: str) -> str:
431 """Rename builtin modules to a consistent module name.
433 Prefer the more modern naming.
435 This is used so that references to Python's `builtins` module can
436 be loaded in both Python 2 and 3. We remap to the "__builtin__"
437 name and unmap it when importing.
439 Map the Python2 `exceptions` module to `builtins` because
440 `builtins` is a superset and contains everything that is
441 available in `exceptions`, which makes the translation simpler.
443 See untranslate_module_name() for the reverse operation.
444 """
445 lookup = {"__builtin__": "builtins", "exceptions": "builtins"}
446 return lookup.get(module, module)
449def _0_9_6_compat_untranslate(module: str) -> str:
450 """Provide compatibility for pickles created with jsonpickle 0.9.6 and
451 earlier, remapping `exceptions` and `__builtin__` to `builtins`.
452 """
453 lookup = {"__builtin__": "builtins", "exceptions": "builtins"}
454 return lookup.get(module, module)
457def untranslate_module_name(module: str) -> str:
458 """Rename module names mention in JSON to names that we can import
460 This reverses the translation applied by translate_module_name() to
461 a module name available to the current version of Python.
463 """
464 return _0_9_6_compat_untranslate(module)
467def importable_name(cls: type | Callable[..., Any]) -> str:
468 """
469 >>> class Example(object):
470 ... pass
472 >>> ex = Example()
473 >>> importable_name(ex.__class__) == 'jsonpickle.util.Example'
474 True
475 >>> importable_name(type(25)) == 'builtins.int'
476 True
477 >>> importable_name(object().__str__.__class__) == 'types.MethodWrapperType'
478 True
479 >>> importable_name(False.__class__) == 'builtins.bool'
480 True
481 >>> importable_name(AttributeError) == 'builtins.AttributeError'
482 True
483 >>> import argparse
484 >>> importable_name(type(argparse.ArgumentParser().add_argument)) == 'types.MethodType'
485 True
487 """
488 types_importable_name = _TYPES_IMPORTABLE_NAMES.get(cls)
489 if types_importable_name is not None:
490 return types_importable_name
492 # Use the fully-qualified name if available (Python >= 3.3)
493 name = getattr(cls, "__qualname__", cls.__name__)
494 module = translate_module_name(cls.__module__)
495 if not module and hasattr(cls, "__self__"):
496 if hasattr(cls.__self__, "__module__"):
497 module = cls.__self__.__module__
498 else:
499 module = cls.__self__.__class__.__module__
500 return f"{module}.{name}"
503def b64encode(data: bytes) -> str:
504 """
505 Encode binary data to ascii text in base64. Data must be bytes.
506 """
507 return base64.b64encode(data).decode("ascii")
510def b64decode(payload: str) -> bytes:
511 """
512 Decode payload - must be ascii text.
513 """
514 try:
515 return base64.b64decode(payload)
516 except (TypeError, binascii.Error):
517 return b""
520def b85encode(data: bytes) -> str:
521 """
522 Encode binary data to ascii text in base85. Data must be bytes.
523 """
524 return base64.b85encode(data).decode("ascii")
527def b85decode(payload: bytes) -> bytes:
528 """
529 Decode payload - must be ascii text.
530 """
531 try:
532 return base64.b85decode(payload)
533 except (TypeError, ValueError):
534 return b""
537def itemgetter(
538 obj: Any,
539 getter: Callable[[Any], Any] = operator.itemgetter(0),
540) -> str:
541 return str(getter(obj))
544def items(
545 obj: dict[Any, Any],
546 exclude: Iterable[Any] = (),
547) -> Iterator[tuple[Any, Any]]:
548 """
549 This can't be easily replaced by dict.items() because this has the exclude parameter.
550 Keep it for now.
551 """
552 for k, v in obj.items():
553 if k in exclude:
554 continue
555 yield k, v
558def loadclass(
559 module_and_name: str, classes: dict[str, type] | None = None
560) -> Any | None:
561 """Loads the module and returns the class.
563 >>> cls = loadclass('datetime.datetime')
564 >>> cls.__name__
565 'datetime'
567 >>> loadclass('does.not.exist')
569 >>> loadclass('builtins.int')()
570 0
572 """
573 # Check if the class exists in a caller-provided scope
574 if classes:
575 try:
576 return classes[module_and_name]
577 except KeyError:
578 # maybe they didn't provide a fully qualified path
579 try:
580 return classes[module_and_name.rsplit(".", 1)[-1]]
581 except KeyError:
582 pass
583 # Otherwise, load classes from globally-accessible imports
584 names = module_and_name.split(".")
585 # First assume that everything up to the last dot is the module name,
586 # then try other splits to handle classes that are defined within
587 # classes
588 for up_to in range(len(names) - 1, 0, -1):
589 module = untranslate_module_name(".".join(names[:up_to]))
590 try:
591 __import__(module)
592 obj = sys.modules[module]
593 for class_name in names[up_to:]:
594 obj = getattr(obj, class_name)
595 return obj
596 except (AttributeError, ImportError, ValueError):
597 continue
598 # NoneType is a special case and can not be imported/created
599 if module_and_name == "builtins.NoneType":
600 return type(None)
601 return None