Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/joblib/externals/cloudpickle/cloudpickle_fast.py: 26%
327 statements
« prev ^ index » next coverage.py v7.3.2, created at 2023-12-12 06:31 +0000
1"""
2New, fast version of the CloudPickler.
4This new CloudPickler class can now extend the fast C Pickler instead of the
5previous Python implementation of the Pickler class. Because this functionality
6is only available for Python versions 3.8+, a lot of backward-compatibility
7code is also removed.
9Note that the C Pickler subclassing API is CPython-specific. Therefore, some
10guards present in cloudpickle.py that were written to handle PyPy specificities
11are not present in cloudpickle_fast.py
12"""
13import _collections_abc
14import abc
15import copyreg
16import io
17import itertools
18import logging
19import sys
20import struct
21import types
22import weakref
23import typing
25from enum import Enum
26from collections import ChainMap, OrderedDict
28from .compat import pickle, Pickler
29from .cloudpickle import (
30 _extract_code_globals, _BUILTIN_TYPE_NAMES, DEFAULT_PROTOCOL,
31 _find_imported_submodules, _get_cell_contents, _should_pickle_by_reference,
32 _builtin_type, _get_or_create_tracker_id, _make_skeleton_class,
33 _make_skeleton_enum, _extract_class_dict, dynamic_subimport, subimport,
34 _typevar_reduce, _get_bases, _make_cell, _make_empty_cell, CellType,
35 _is_parametrized_type_hint, PYPY, cell_set,
36 parametrized_type_hint_getinitargs, _create_parametrized_type_hint,
37 builtin_code_type,
38 _make_dict_keys, _make_dict_values, _make_dict_items, _make_function,
39)
42if pickle.HIGHEST_PROTOCOL >= 5:
43 # Shorthands similar to pickle.dump/pickle.dumps
45 def dump(obj, file, protocol=None, buffer_callback=None):
46 """Serialize obj as bytes streamed into file
48 protocol defaults to cloudpickle.DEFAULT_PROTOCOL which is an alias to
49 pickle.HIGHEST_PROTOCOL. This setting favors maximum communication
50 speed between processes running the same Python version.
52 Set protocol=pickle.DEFAULT_PROTOCOL instead if you need to ensure
53 compatibility with older versions of Python.
54 """
55 CloudPickler(
56 file, protocol=protocol, buffer_callback=buffer_callback
57 ).dump(obj)
59 def dumps(obj, protocol=None, buffer_callback=None):
60 """Serialize obj as a string of bytes allocated in memory
62 protocol defaults to cloudpickle.DEFAULT_PROTOCOL which is an alias to
63 pickle.HIGHEST_PROTOCOL. This setting favors maximum communication
64 speed between processes running the same Python version.
66 Set protocol=pickle.DEFAULT_PROTOCOL instead if you need to ensure
67 compatibility with older versions of Python.
68 """
69 with io.BytesIO() as file:
70 cp = CloudPickler(
71 file, protocol=protocol, buffer_callback=buffer_callback
72 )
73 cp.dump(obj)
74 return file.getvalue()
76else:
77 # Shorthands similar to pickle.dump/pickle.dumps
78 def dump(obj, file, protocol=None):
79 """Serialize obj as bytes streamed into file
81 protocol defaults to cloudpickle.DEFAULT_PROTOCOL which is an alias to
82 pickle.HIGHEST_PROTOCOL. This setting favors maximum communication
83 speed between processes running the same Python version.
85 Set protocol=pickle.DEFAULT_PROTOCOL instead if you need to ensure
86 compatibility with older versions of Python.
87 """
88 CloudPickler(file, protocol=protocol).dump(obj)
90 def dumps(obj, protocol=None):
91 """Serialize obj as a string of bytes allocated in memory
93 protocol defaults to cloudpickle.DEFAULT_PROTOCOL which is an alias to
94 pickle.HIGHEST_PROTOCOL. This setting favors maximum communication
95 speed between processes running the same Python version.
97 Set protocol=pickle.DEFAULT_PROTOCOL instead if you need to ensure
98 compatibility with older versions of Python.
99 """
100 with io.BytesIO() as file:
101 cp = CloudPickler(file, protocol=protocol)
102 cp.dump(obj)
103 return file.getvalue()
106load, loads = pickle.load, pickle.loads
109# COLLECTION OF OBJECTS __getnewargs__-LIKE METHODS
110# -------------------------------------------------
112def _class_getnewargs(obj):
113 type_kwargs = {}
114 if "__slots__" in obj.__dict__:
115 type_kwargs["__slots__"] = obj.__slots__
117 __dict__ = obj.__dict__.get('__dict__', None)
118 if isinstance(__dict__, property):
119 type_kwargs['__dict__'] = __dict__
121 return (type(obj), obj.__name__, _get_bases(obj), type_kwargs,
122 _get_or_create_tracker_id(obj), None)
125def _enum_getnewargs(obj):
126 members = {e.name: e.value for e in obj}
127 return (obj.__bases__, obj.__name__, obj.__qualname__, members,
128 obj.__module__, _get_or_create_tracker_id(obj), None)
131# COLLECTION OF OBJECTS RECONSTRUCTORS
132# ------------------------------------
133def _file_reconstructor(retval):
134 return retval
137# COLLECTION OF OBJECTS STATE GETTERS
138# -----------------------------------
139def _function_getstate(func):
140 # - Put func's dynamic attributes (stored in func.__dict__) in state. These
141 # attributes will be restored at unpickling time using
142 # f.__dict__.update(state)
143 # - Put func's members into slotstate. Such attributes will be restored at
144 # unpickling time by iterating over slotstate and calling setattr(func,
145 # slotname, slotvalue)
146 slotstate = {
147 "__name__": func.__name__,
148 "__qualname__": func.__qualname__,
149 "__annotations__": func.__annotations__,
150 "__kwdefaults__": func.__kwdefaults__,
151 "__defaults__": func.__defaults__,
152 "__module__": func.__module__,
153 "__doc__": func.__doc__,
154 "__closure__": func.__closure__,
155 }
157 f_globals_ref = _extract_code_globals(func.__code__)
158 f_globals = {k: func.__globals__[k] for k in f_globals_ref if k in
159 func.__globals__}
161 closure_values = (
162 list(map(_get_cell_contents, func.__closure__))
163 if func.__closure__ is not None else ()
164 )
166 # Extract currently-imported submodules used by func. Storing these modules
167 # in a smoke _cloudpickle_subimports attribute of the object's state will
168 # trigger the side effect of importing these modules at unpickling time
169 # (which is necessary for func to work correctly once depickled)
170 slotstate["_cloudpickle_submodules"] = _find_imported_submodules(
171 func.__code__, itertools.chain(f_globals.values(), closure_values))
172 slotstate["__globals__"] = f_globals
174 state = func.__dict__
175 return state, slotstate
178def _class_getstate(obj):
179 clsdict = _extract_class_dict(obj)
180 clsdict.pop('__weakref__', None)
182 if issubclass(type(obj), abc.ABCMeta):
183 # If obj is an instance of an ABCMeta subclass, don't pickle the
184 # cache/negative caches populated during isinstance/issubclass
185 # checks, but pickle the list of registered subclasses of obj.
186 clsdict.pop('_abc_cache', None)
187 clsdict.pop('_abc_negative_cache', None)
188 clsdict.pop('_abc_negative_cache_version', None)
189 registry = clsdict.pop('_abc_registry', None)
190 if registry is None:
191 # in Python3.7+, the abc caches and registered subclasses of a
192 # class are bundled into the single _abc_impl attribute
193 clsdict.pop('_abc_impl', None)
194 (registry, _, _, _) = abc._get_dump(obj)
196 clsdict["_abc_impl"] = [subclass_weakref()
197 for subclass_weakref in registry]
198 else:
199 # In the above if clause, registry is a set of weakrefs -- in
200 # this case, registry is a WeakSet
201 clsdict["_abc_impl"] = [type_ for type_ in registry]
203 if "__slots__" in clsdict:
204 # pickle string length optimization: member descriptors of obj are
205 # created automatically from obj's __slots__ attribute, no need to
206 # save them in obj's state
207 if isinstance(obj.__slots__, str):
208 clsdict.pop(obj.__slots__)
209 else:
210 for k in obj.__slots__:
211 clsdict.pop(k, None)
213 clsdict.pop('__dict__', None) # unpicklable property object
215 return (clsdict, {})
218def _enum_getstate(obj):
219 clsdict, slotstate = _class_getstate(obj)
221 members = {e.name: e.value for e in obj}
222 # Cleanup the clsdict that will be passed to _rehydrate_skeleton_class:
223 # Those attributes are already handled by the metaclass.
224 for attrname in ["_generate_next_value_", "_member_names_",
225 "_member_map_", "_member_type_",
226 "_value2member_map_"]:
227 clsdict.pop(attrname, None)
228 for member in members:
229 clsdict.pop(member)
230 # Special handling of Enum subclasses
231 return clsdict, slotstate
234# COLLECTIONS OF OBJECTS REDUCERS
235# -------------------------------
236# A reducer is a function taking a single argument (obj), and that returns a
237# tuple with all the necessary data to re-construct obj. Apart from a few
238# exceptions (list, dict, bytes, int, etc.), a reducer is necessary to
239# correctly pickle an object.
240# While many built-in objects (Exceptions objects, instances of the "object"
241# class, etc), are shipped with their own built-in reducer (invoked using
242# obj.__reduce__), some do not. The following methods were created to "fill
243# these holes".
245def _code_reduce(obj):
246 """codeobject reducer"""
247 # If you are not sure about the order of arguments, take a look at help
248 # of the specific type from types, for example:
249 # >>> from types import CodeType
250 # >>> help(CodeType)
251 if hasattr(obj, "co_exceptiontable"): # pragma: no branch
252 # Python 3.11 and later: there are some new attributes
253 # related to the enhanced exceptions.
254 args = (
255 obj.co_argcount, obj.co_posonlyargcount,
256 obj.co_kwonlyargcount, obj.co_nlocals, obj.co_stacksize,
257 obj.co_flags, obj.co_code, obj.co_consts, obj.co_names,
258 obj.co_varnames, obj.co_filename, obj.co_name, obj.co_qualname,
259 obj.co_firstlineno, obj.co_linetable, obj.co_exceptiontable,
260 obj.co_freevars, obj.co_cellvars,
261 )
262 elif hasattr(obj, "co_linetable"): # pragma: no branch
263 # Python 3.10 and later: obj.co_lnotab is deprecated and constructor
264 # expects obj.co_linetable instead.
265 args = (
266 obj.co_argcount, obj.co_posonlyargcount,
267 obj.co_kwonlyargcount, obj.co_nlocals, obj.co_stacksize,
268 obj.co_flags, obj.co_code, obj.co_consts, obj.co_names,
269 obj.co_varnames, obj.co_filename, obj.co_name,
270 obj.co_firstlineno, obj.co_linetable, obj.co_freevars,
271 obj.co_cellvars
272 )
273 elif hasattr(obj, "co_nmeta"): # pragma: no cover
274 # "nogil" Python: modified attributes from 3.9
275 args = (
276 obj.co_argcount, obj.co_posonlyargcount,
277 obj.co_kwonlyargcount, obj.co_nlocals, obj.co_framesize,
278 obj.co_ndefaultargs, obj.co_nmeta,
279 obj.co_flags, obj.co_code, obj.co_consts,
280 obj.co_varnames, obj.co_filename, obj.co_name,
281 obj.co_firstlineno, obj.co_lnotab, obj.co_exc_handlers,
282 obj.co_jump_table, obj.co_freevars, obj.co_cellvars,
283 obj.co_free2reg, obj.co_cell2reg
284 )
285 elif hasattr(obj, "co_posonlyargcount"):
286 # Backward compat for 3.9 and older
287 args = (
288 obj.co_argcount, obj.co_posonlyargcount,
289 obj.co_kwonlyargcount, obj.co_nlocals, obj.co_stacksize,
290 obj.co_flags, obj.co_code, obj.co_consts, obj.co_names,
291 obj.co_varnames, obj.co_filename, obj.co_name,
292 obj.co_firstlineno, obj.co_lnotab, obj.co_freevars,
293 obj.co_cellvars
294 )
295 else:
296 # Backward compat for even older versions of Python
297 args = (
298 obj.co_argcount, obj.co_kwonlyargcount, obj.co_nlocals,
299 obj.co_stacksize, obj.co_flags, obj.co_code, obj.co_consts,
300 obj.co_names, obj.co_varnames, obj.co_filename,
301 obj.co_name, obj.co_firstlineno, obj.co_lnotab,
302 obj.co_freevars, obj.co_cellvars
303 )
304 return types.CodeType, args
307def _cell_reduce(obj):
308 """Cell (containing values of a function's free variables) reducer"""
309 try:
310 obj.cell_contents
311 except ValueError: # cell is empty
312 return _make_empty_cell, ()
313 else:
314 return _make_cell, (obj.cell_contents, )
317def _classmethod_reduce(obj):
318 orig_func = obj.__func__
319 return type(obj), (orig_func,)
322def _file_reduce(obj):
323 """Save a file"""
324 import io
326 if not hasattr(obj, "name") or not hasattr(obj, "mode"):
327 raise pickle.PicklingError(
328 "Cannot pickle files that do not map to an actual file"
329 )
330 if obj is sys.stdout:
331 return getattr, (sys, "stdout")
332 if obj is sys.stderr:
333 return getattr, (sys, "stderr")
334 if obj is sys.stdin:
335 raise pickle.PicklingError("Cannot pickle standard input")
336 if obj.closed:
337 raise pickle.PicklingError("Cannot pickle closed files")
338 if hasattr(obj, "isatty") and obj.isatty():
339 raise pickle.PicklingError(
340 "Cannot pickle files that map to tty objects"
341 )
342 if "r" not in obj.mode and "+" not in obj.mode:
343 raise pickle.PicklingError(
344 "Cannot pickle files that are not opened for reading: %s"
345 % obj.mode
346 )
348 name = obj.name
350 retval = io.StringIO()
352 try:
353 # Read the whole file
354 curloc = obj.tell()
355 obj.seek(0)
356 contents = obj.read()
357 obj.seek(curloc)
358 except IOError as e:
359 raise pickle.PicklingError(
360 "Cannot pickle file %s as it cannot be read" % name
361 ) from e
362 retval.write(contents)
363 retval.seek(curloc)
365 retval.name = name
366 return _file_reconstructor, (retval,)
369def _getset_descriptor_reduce(obj):
370 return getattr, (obj.__objclass__, obj.__name__)
373def _mappingproxy_reduce(obj):
374 return types.MappingProxyType, (dict(obj),)
377def _memoryview_reduce(obj):
378 return bytes, (obj.tobytes(),)
381def _module_reduce(obj):
382 if _should_pickle_by_reference(obj):
383 return subimport, (obj.__name__,)
384 else:
385 # Some external libraries can populate the "__builtins__" entry of a
386 # module's `__dict__` with unpicklable objects (see #316). For that
387 # reason, we do not attempt to pickle the "__builtins__" entry, and
388 # restore a default value for it at unpickling time.
389 state = obj.__dict__.copy()
390 state.pop('__builtins__', None)
391 return dynamic_subimport, (obj.__name__, state)
394def _method_reduce(obj):
395 return (types.MethodType, (obj.__func__, obj.__self__))
398def _logger_reduce(obj):
399 return logging.getLogger, (obj.name,)
402def _root_logger_reduce(obj):
403 return logging.getLogger, ()
406def _property_reduce(obj):
407 return property, (obj.fget, obj.fset, obj.fdel, obj.__doc__)
410def _weakset_reduce(obj):
411 return weakref.WeakSet, (list(obj),)
414def _dynamic_class_reduce(obj):
415 """
416 Save a class that can't be stored as module global.
418 This method is used to serialize classes that are defined inside
419 functions, or that otherwise can't be serialized as attribute lookups
420 from global modules.
421 """
422 if Enum is not None and issubclass(obj, Enum):
423 return (
424 _make_skeleton_enum, _enum_getnewargs(obj), _enum_getstate(obj),
425 None, None, _class_setstate
426 )
427 else:
428 return (
429 _make_skeleton_class, _class_getnewargs(obj), _class_getstate(obj),
430 None, None, _class_setstate
431 )
434def _class_reduce(obj):
435 """Select the reducer depending on the dynamic nature of the class obj"""
436 if obj is type(None): # noqa
437 return type, (None,)
438 elif obj is type(Ellipsis):
439 return type, (Ellipsis,)
440 elif obj is type(NotImplemented):
441 return type, (NotImplemented,)
442 elif obj in _BUILTIN_TYPE_NAMES:
443 return _builtin_type, (_BUILTIN_TYPE_NAMES[obj],)
444 elif not _should_pickle_by_reference(obj):
445 return _dynamic_class_reduce(obj)
446 return NotImplemented
449def _dict_keys_reduce(obj):
450 # Safer not to ship the full dict as sending the rest might
451 # be unintended and could potentially cause leaking of
452 # sensitive information
453 return _make_dict_keys, (list(obj), )
456def _dict_values_reduce(obj):
457 # Safer not to ship the full dict as sending the rest might
458 # be unintended and could potentially cause leaking of
459 # sensitive information
460 return _make_dict_values, (list(obj), )
463def _dict_items_reduce(obj):
464 return _make_dict_items, (dict(obj), )
467def _odict_keys_reduce(obj):
468 # Safer not to ship the full dict as sending the rest might
469 # be unintended and could potentially cause leaking of
470 # sensitive information
471 return _make_dict_keys, (list(obj), True)
474def _odict_values_reduce(obj):
475 # Safer not to ship the full dict as sending the rest might
476 # be unintended and could potentially cause leaking of
477 # sensitive information
478 return _make_dict_values, (list(obj), True)
481def _odict_items_reduce(obj):
482 return _make_dict_items, (dict(obj), True)
485# COLLECTIONS OF OBJECTS STATE SETTERS
486# ------------------------------------
487# state setters are called at unpickling time, once the object is created and
488# it has to be updated to how it was at unpickling time.
491def _function_setstate(obj, state):
492 """Update the state of a dynamic function.
494 As __closure__ and __globals__ are readonly attributes of a function, we
495 cannot rely on the native setstate routine of pickle.load_build, that calls
496 setattr on items of the slotstate. Instead, we have to modify them inplace.
497 """
498 state, slotstate = state
499 obj.__dict__.update(state)
501 obj_globals = slotstate.pop("__globals__")
502 obj_closure = slotstate.pop("__closure__")
503 # _cloudpickle_subimports is a set of submodules that must be loaded for
504 # the pickled function to work correctly at unpickling time. Now that these
505 # submodules are depickled (hence imported), they can be removed from the
506 # object's state (the object state only served as a reference holder to
507 # these submodules)
508 slotstate.pop("_cloudpickle_submodules")
510 obj.__globals__.update(obj_globals)
511 obj.__globals__["__builtins__"] = __builtins__
513 if obj_closure is not None:
514 for i, cell in enumerate(obj_closure):
515 try:
516 value = cell.cell_contents
517 except ValueError: # cell is empty
518 continue
519 cell_set(obj.__closure__[i], value)
521 for k, v in slotstate.items():
522 setattr(obj, k, v)
525def _class_setstate(obj, state):
526 state, slotstate = state
527 registry = None
528 for attrname, attr in state.items():
529 if attrname == "_abc_impl":
530 registry = attr
531 else:
532 setattr(obj, attrname, attr)
533 if registry is not None:
534 for subclass in registry:
535 obj.register(subclass)
537 return obj
540class CloudPickler(Pickler):
541 # set of reducers defined and used by cloudpickle (private)
542 _dispatch_table = {}
543 _dispatch_table[classmethod] = _classmethod_reduce
544 _dispatch_table[io.TextIOWrapper] = _file_reduce
545 _dispatch_table[logging.Logger] = _logger_reduce
546 _dispatch_table[logging.RootLogger] = _root_logger_reduce
547 _dispatch_table[memoryview] = _memoryview_reduce
548 _dispatch_table[property] = _property_reduce
549 _dispatch_table[staticmethod] = _classmethod_reduce
550 _dispatch_table[CellType] = _cell_reduce
551 _dispatch_table[types.CodeType] = _code_reduce
552 _dispatch_table[types.GetSetDescriptorType] = _getset_descriptor_reduce
553 _dispatch_table[types.ModuleType] = _module_reduce
554 _dispatch_table[types.MethodType] = _method_reduce
555 _dispatch_table[types.MappingProxyType] = _mappingproxy_reduce
556 _dispatch_table[weakref.WeakSet] = _weakset_reduce
557 _dispatch_table[typing.TypeVar] = _typevar_reduce
558 _dispatch_table[_collections_abc.dict_keys] = _dict_keys_reduce
559 _dispatch_table[_collections_abc.dict_values] = _dict_values_reduce
560 _dispatch_table[_collections_abc.dict_items] = _dict_items_reduce
561 _dispatch_table[type(OrderedDict().keys())] = _odict_keys_reduce
562 _dispatch_table[type(OrderedDict().values())] = _odict_values_reduce
563 _dispatch_table[type(OrderedDict().items())] = _odict_items_reduce
564 _dispatch_table[abc.abstractmethod] = _classmethod_reduce
565 _dispatch_table[abc.abstractclassmethod] = _classmethod_reduce
566 _dispatch_table[abc.abstractstaticmethod] = _classmethod_reduce
567 _dispatch_table[abc.abstractproperty] = _property_reduce
569 dispatch_table = ChainMap(_dispatch_table, copyreg.dispatch_table)
571 # function reducers are defined as instance methods of CloudPickler
572 # objects, as they rely on a CloudPickler attribute (globals_ref)
573 def _dynamic_function_reduce(self, func):
574 """Reduce a function that is not pickleable via attribute lookup."""
575 newargs = self._function_getnewargs(func)
576 state = _function_getstate(func)
577 return (_make_function, newargs, state, None, None,
578 _function_setstate)
580 def _function_reduce(self, obj):
581 """Reducer for function objects.
583 If obj is a top-level attribute of a file-backed module, this
584 reducer returns NotImplemented, making the CloudPickler fallback to
585 traditional _pickle.Pickler routines to save obj. Otherwise, it reduces
586 obj using a custom cloudpickle reducer designed specifically to handle
587 dynamic functions.
589 As opposed to cloudpickle.py, There no special handling for builtin
590 pypy functions because cloudpickle_fast is CPython-specific.
591 """
592 if _should_pickle_by_reference(obj):
593 return NotImplemented
594 else:
595 return self._dynamic_function_reduce(obj)
597 def _function_getnewargs(self, func):
598 code = func.__code__
600 # base_globals represents the future global namespace of func at
601 # unpickling time. Looking it up and storing it in
602 # CloudpiPickler.globals_ref allow functions sharing the same globals
603 # at pickling time to also share them once unpickled, at one condition:
604 # since globals_ref is an attribute of a CloudPickler instance, and
605 # that a new CloudPickler is created each time pickle.dump or
606 # pickle.dumps is called, functions also need to be saved within the
607 # same invocation of cloudpickle.dump/cloudpickle.dumps (for example:
608 # cloudpickle.dumps([f1, f2])). There is no such limitation when using
609 # CloudPickler.dump, as long as the multiple invocations are bound to
610 # the same CloudPickler.
611 base_globals = self.globals_ref.setdefault(id(func.__globals__), {})
613 if base_globals == {}:
614 # Add module attributes used to resolve relative imports
615 # instructions inside func.
616 for k in ["__package__", "__name__", "__path__", "__file__"]:
617 if k in func.__globals__:
618 base_globals[k] = func.__globals__[k]
620 # Do not bind the free variables before the function is created to
621 # avoid infinite recursion.
622 if func.__closure__ is None:
623 closure = None
624 else:
625 closure = tuple(
626 _make_empty_cell() for _ in range(len(code.co_freevars)))
628 return code, base_globals, None, None, closure
630 def dump(self, obj):
631 try:
632 return Pickler.dump(self, obj)
633 except RuntimeError as e:
634 if "recursion" in e.args[0]:
635 msg = (
636 "Could not pickle object as excessively deep recursion "
637 "required."
638 )
639 raise pickle.PicklingError(msg) from e
640 else:
641 raise
643 if pickle.HIGHEST_PROTOCOL >= 5:
644 def __init__(self, file, protocol=None, buffer_callback=None):
645 if protocol is None:
646 protocol = DEFAULT_PROTOCOL
647 Pickler.__init__(
648 self, file, protocol=protocol, buffer_callback=buffer_callback
649 )
650 # map functions __globals__ attribute ids, to ensure that functions
651 # sharing the same global namespace at pickling time also share
652 # their global namespace at unpickling time.
653 self.globals_ref = {}
654 self.proto = int(protocol)
655 else:
656 def __init__(self, file, protocol=None):
657 if protocol is None:
658 protocol = DEFAULT_PROTOCOL
659 Pickler.__init__(self, file, protocol=protocol)
660 # map functions __globals__ attribute ids, to ensure that functions
661 # sharing the same global namespace at pickling time also share
662 # their global namespace at unpickling time.
663 self.globals_ref = {}
664 assert hasattr(self, 'proto')
666 if pickle.HIGHEST_PROTOCOL >= 5 and not PYPY:
667 # Pickler is the C implementation of the CPython pickler and therefore
668 # we rely on reduce_override method to customize the pickler behavior.
670 # `CloudPickler.dispatch` is only left for backward compatibility - note
671 # that when using protocol 5, `CloudPickler.dispatch` is not an
672 # extension of `Pickler.dispatch` dictionary, because CloudPickler
673 # subclasses the C-implemented Pickler, which does not expose a
674 # `dispatch` attribute. Earlier versions of the protocol 5 CloudPickler
675 # used `CloudPickler.dispatch` as a class-level attribute storing all
676 # reducers implemented by cloudpickle, but the attribute name was not a
677 # great choice given the meaning of `CloudPickler.dispatch` when
678 # `CloudPickler` extends the pure-python pickler.
679 dispatch = dispatch_table
681 # Implementation of the reducer_override callback, in order to
682 # efficiently serialize dynamic functions and classes by subclassing
683 # the C-implemented Pickler.
684 # TODO: decorrelate reducer_override (which is tied to CPython's
685 # implementation - would it make sense to backport it to pypy? - and
686 # pickle's protocol 5 which is implementation agnostic. Currently, the
687 # availability of both notions coincide on CPython's pickle and the
688 # pickle5 backport, but it may not be the case anymore when pypy
689 # implements protocol 5
691 def reducer_override(self, obj):
692 """Type-agnostic reducing callback for function and classes.
694 For performance reasons, subclasses of the C _pickle.Pickler class
695 cannot register custom reducers for functions and classes in the
696 dispatch_table. Reducer for such types must instead implemented in
697 the special reducer_override method.
699 Note that method will be called for any object except a few
700 builtin-types (int, lists, dicts etc.), which differs from reducers
701 in the Pickler's dispatch_table, each of them being invoked for
702 objects of a specific type only.
704 This property comes in handy for classes: although most classes are
705 instances of the ``type`` metaclass, some of them can be instances
706 of other custom metaclasses (such as enum.EnumMeta for example). In
707 particular, the metaclass will likely not be known in advance, and
708 thus cannot be special-cased using an entry in the dispatch_table.
709 reducer_override, among other things, allows us to register a
710 reducer that will be called for any class, independently of its
711 type.
714 Notes:
716 * reducer_override has the priority over dispatch_table-registered
717 reducers.
718 * reducer_override can be used to fix other limitations of
719 cloudpickle for other types that suffered from type-specific
720 reducers, such as Exceptions. See
721 https://github.com/cloudpipe/cloudpickle/issues/248
722 """
723 if sys.version_info[:2] < (3, 7) and _is_parametrized_type_hint(obj): # noqa # pragma: no branch
724 return (
725 _create_parametrized_type_hint,
726 parametrized_type_hint_getinitargs(obj)
727 )
728 t = type(obj)
729 try:
730 is_anyclass = issubclass(t, type)
731 except TypeError: # t is not a class (old Boost; see SF #502085)
732 is_anyclass = False
734 if is_anyclass:
735 return _class_reduce(obj)
736 elif isinstance(obj, types.FunctionType):
737 return self._function_reduce(obj)
738 else:
739 # fallback to save_global, including the Pickler's
740 # dispatch_table
741 return NotImplemented
743 else:
744 # When reducer_override is not available, hack the pure-Python
745 # Pickler's types.FunctionType and type savers. Note: the type saver
746 # must override Pickler.save_global, because pickle.py contains a
747 # hard-coded call to save_global when pickling meta-classes.
748 dispatch = Pickler.dispatch.copy()
750 def _save_reduce_pickle5(self, func, args, state=None, listitems=None,
751 dictitems=None, state_setter=None, obj=None):
752 save = self.save
753 write = self.write
754 self.save_reduce(
755 func, args, state=None, listitems=listitems,
756 dictitems=dictitems, obj=obj
757 )
758 # backport of the Python 3.8 state_setter pickle operations
759 save(state_setter)
760 save(obj) # simple BINGET opcode as obj is already memoized.
761 save(state)
762 write(pickle.TUPLE2)
763 # Trigger a state_setter(obj, state) function call.
764 write(pickle.REDUCE)
765 # The purpose of state_setter is to carry-out an
766 # inplace modification of obj. We do not care about what the
767 # method might return, so its output is eventually removed from
768 # the stack.
769 write(pickle.POP)
771 def save_global(self, obj, name=None, pack=struct.pack):
772 """
773 Save a "global".
775 The name of this method is somewhat misleading: all types get
776 dispatched here.
777 """
778 if obj is type(None): # noqa
779 return self.save_reduce(type, (None,), obj=obj)
780 elif obj is type(Ellipsis):
781 return self.save_reduce(type, (Ellipsis,), obj=obj)
782 elif obj is type(NotImplemented):
783 return self.save_reduce(type, (NotImplemented,), obj=obj)
784 elif obj in _BUILTIN_TYPE_NAMES:
785 return self.save_reduce(
786 _builtin_type, (_BUILTIN_TYPE_NAMES[obj],), obj=obj)
788 if sys.version_info[:2] < (3, 7) and _is_parametrized_type_hint(obj): # noqa # pragma: no branch
789 # Parametrized typing constructs in Python < 3.7 are not
790 # compatible with type checks and ``isinstance`` semantics. For
791 # this reason, it is easier to detect them using a
792 # duck-typing-based check (``_is_parametrized_type_hint``) than
793 # to populate the Pickler's dispatch with type-specific savers.
794 self.save_reduce(
795 _create_parametrized_type_hint,
796 parametrized_type_hint_getinitargs(obj),
797 obj=obj
798 )
799 elif name is not None:
800 Pickler.save_global(self, obj, name=name)
801 elif not _should_pickle_by_reference(obj, name=name):
802 self._save_reduce_pickle5(*_dynamic_class_reduce(obj), obj=obj)
803 else:
804 Pickler.save_global(self, obj, name=name)
805 dispatch[type] = save_global
807 def save_function(self, obj, name=None):
808 """ Registered with the dispatch to handle all function types.
810 Determines what kind of function obj is (e.g. lambda, defined at
811 interactive prompt, etc) and handles the pickling appropriately.
812 """
813 if _should_pickle_by_reference(obj, name=name):
814 return Pickler.save_global(self, obj, name=name)
815 elif PYPY and isinstance(obj.__code__, builtin_code_type):
816 return self.save_pypy_builtin_func(obj)
817 else:
818 return self._save_reduce_pickle5(
819 *self._dynamic_function_reduce(obj), obj=obj
820 )
822 def save_pypy_builtin_func(self, obj):
823 """Save pypy equivalent of builtin functions.
824 PyPy does not have the concept of builtin-functions. Instead,
825 builtin-functions are simple function instances, but with a
826 builtin-code attribute.
827 Most of the time, builtin functions should be pickled by attribute.
828 But PyPy has flaky support for __qualname__, so some builtin
829 functions such as float.__new__ will be classified as dynamic. For
830 this reason only, we created this special routine. Because
831 builtin-functions are not expected to have closure or globals,
832 there is no additional hack (compared the one already implemented
833 in pickle) to protect ourselves from reference cycles. A simple
834 (reconstructor, newargs, obj.__dict__) tuple is save_reduced. Note
835 also that PyPy improved their support for __qualname__ in v3.6, so
836 this routing should be removed when cloudpickle supports only PyPy
837 3.6 and later.
838 """
839 rv = (types.FunctionType, (obj.__code__, {}, obj.__name__,
840 obj.__defaults__, obj.__closure__),
841 obj.__dict__)
842 self.save_reduce(*rv, obj=obj)
844 dispatch[types.FunctionType] = save_function