Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/joblib/externals/cloudpickle/cloudpickle.py: 24%
374 statements
« prev ^ index » next coverage.py v7.3.2, created at 2023-12-12 06:31 +0000
1"""
2This class is defined to override standard pickle functionality
4The goals of it follow:
5-Serialize lambdas and nested functions to compiled byte code
6-Deal with main module correctly
7-Deal with other non-serializable objects
9It does not include an unpickler, as standard python unpickling suffices.
11This module was extracted from the `cloud` package, developed by `PiCloud, Inc.
12<https://web.archive.org/web/20140626004012/http://www.picloud.com/>`_.
14Copyright (c) 2012, Regents of the University of California.
15Copyright (c) 2009 `PiCloud, Inc. <https://web.archive.org/web/20140626004012/http://www.picloud.com/>`_.
16All rights reserved.
18Redistribution and use in source and binary forms, with or without
19modification, are permitted provided that the following conditions
20are met:
21 * Redistributions of source code must retain the above copyright
22 notice, this list of conditions and the following disclaimer.
23 * Redistributions in binary form must reproduce the above copyright
24 notice, this list of conditions and the following disclaimer in the
25 documentation and/or other materials provided with the distribution.
26 * Neither the name of the University of California, Berkeley nor the
27 names of its contributors may be used to endorse or promote
28 products derived from this software without specific prior written
29 permission.
31THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
32"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
33LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
34A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
35HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
36SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
37TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
38PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
39LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
40NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
41SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
42"""
44import builtins
45import dis
46import opcode
47import platform
48import sys
49import types
50import weakref
51import uuid
52import threading
53import typing
54import warnings
56from .compat import pickle
57from collections import OrderedDict
58from typing import ClassVar, Generic, Union, Tuple, Callable
59from pickle import _getattribute
60from importlib._bootstrap import _find_spec
62try: # pragma: no branch
63 import typing_extensions as _typing_extensions
64 from typing_extensions import Literal, Final
65except ImportError:
66 _typing_extensions = Literal = Final = None
68if sys.version_info >= (3, 8):
69 from types import CellType
70else:
71 def f():
72 a = 1
74 def g():
75 return a
76 return g
77 CellType = type(f().__closure__[0])
80# cloudpickle is meant for inter process communication: we expect all
81# communicating processes to run the same Python version hence we favor
82# communication speed over compatibility:
83DEFAULT_PROTOCOL = pickle.HIGHEST_PROTOCOL
85# Names of modules whose resources should be treated as dynamic.
86_PICKLE_BY_VALUE_MODULES = set()
88# Track the provenance of reconstructed dynamic classes to make it possible to
89# reconstruct instances from the matching singleton class definition when
90# appropriate and preserve the usual "isinstance" semantics of Python objects.
91_DYNAMIC_CLASS_TRACKER_BY_CLASS = weakref.WeakKeyDictionary()
92_DYNAMIC_CLASS_TRACKER_BY_ID = weakref.WeakValueDictionary()
93_DYNAMIC_CLASS_TRACKER_LOCK = threading.Lock()
95PYPY = platform.python_implementation() == "PyPy"
97builtin_code_type = None
98if PYPY:
99 # builtin-code objects only exist in pypy
100 builtin_code_type = type(float.__new__.__code__)
102_extract_code_globals_cache = weakref.WeakKeyDictionary()
105def _get_or_create_tracker_id(class_def):
106 with _DYNAMIC_CLASS_TRACKER_LOCK:
107 class_tracker_id = _DYNAMIC_CLASS_TRACKER_BY_CLASS.get(class_def)
108 if class_tracker_id is None:
109 class_tracker_id = uuid.uuid4().hex
110 _DYNAMIC_CLASS_TRACKER_BY_CLASS[class_def] = class_tracker_id
111 _DYNAMIC_CLASS_TRACKER_BY_ID[class_tracker_id] = class_def
112 return class_tracker_id
115def _lookup_class_or_track(class_tracker_id, class_def):
116 if class_tracker_id is not None:
117 with _DYNAMIC_CLASS_TRACKER_LOCK:
118 class_def = _DYNAMIC_CLASS_TRACKER_BY_ID.setdefault(
119 class_tracker_id, class_def)
120 _DYNAMIC_CLASS_TRACKER_BY_CLASS[class_def] = class_tracker_id
121 return class_def
124def register_pickle_by_value(module):
125 """Register a module to make it functions and classes picklable by value.
127 By default, functions and classes that are attributes of an importable
128 module are to be pickled by reference, that is relying on re-importing
129 the attribute from the module at load time.
131 If `register_pickle_by_value(module)` is called, all its functions and
132 classes are subsequently to be pickled by value, meaning that they can
133 be loaded in Python processes where the module is not importable.
135 This is especially useful when developing a module in a distributed
136 execution environment: restarting the client Python process with the new
137 source code is enough: there is no need to re-install the new version
138 of the module on all the worker nodes nor to restart the workers.
140 Note: this feature is considered experimental. See the cloudpickle
141 README.md file for more details and limitations.
142 """
143 if not isinstance(module, types.ModuleType):
144 raise ValueError(
145 f"Input should be a module object, got {str(module)} instead"
146 )
147 # In the future, cloudpickle may need a way to access any module registered
148 # for pickling by value in order to introspect relative imports inside
149 # functions pickled by value. (see
150 # https://github.com/cloudpipe/cloudpickle/pull/417#issuecomment-873684633).
151 # This access can be ensured by checking that module is present in
152 # sys.modules at registering time and assuming that it will still be in
153 # there when accessed during pickling. Another alternative would be to
154 # store a weakref to the module. Even though cloudpickle does not implement
155 # this introspection yet, in order to avoid a possible breaking change
156 # later, we still enforce the presence of module inside sys.modules.
157 if module.__name__ not in sys.modules:
158 raise ValueError(
159 f"{module} was not imported correctly, have you used an "
160 f"`import` statement to access it?"
161 )
162 _PICKLE_BY_VALUE_MODULES.add(module.__name__)
165def unregister_pickle_by_value(module):
166 """Unregister that the input module should be pickled by value."""
167 if not isinstance(module, types.ModuleType):
168 raise ValueError(
169 f"Input should be a module object, got {str(module)} instead"
170 )
171 if module.__name__ not in _PICKLE_BY_VALUE_MODULES:
172 raise ValueError(f"{module} is not registered for pickle by value")
173 else:
174 _PICKLE_BY_VALUE_MODULES.remove(module.__name__)
177def list_registry_pickle_by_value():
178 return _PICKLE_BY_VALUE_MODULES.copy()
181def _is_registered_pickle_by_value(module):
182 module_name = module.__name__
183 if module_name in _PICKLE_BY_VALUE_MODULES:
184 return True
185 while True:
186 parent_name = module_name.rsplit(".", 1)[0]
187 if parent_name == module_name:
188 break
189 if parent_name in _PICKLE_BY_VALUE_MODULES:
190 return True
191 module_name = parent_name
192 return False
195def _whichmodule(obj, name):
196 """Find the module an object belongs to.
198 This function differs from ``pickle.whichmodule`` in two ways:
199 - it does not mangle the cases where obj's module is __main__ and obj was
200 not found in any module.
201 - Errors arising during module introspection are ignored, as those errors
202 are considered unwanted side effects.
203 """
204 if sys.version_info[:2] < (3, 7) and isinstance(obj, typing.TypeVar): # pragma: no branch # noqa
205 # Workaround bug in old Python versions: prior to Python 3.7,
206 # T.__module__ would always be set to "typing" even when the TypeVar T
207 # would be defined in a different module.
208 if name is not None and getattr(typing, name, None) is obj:
209 # Built-in TypeVar defined in typing such as AnyStr
210 return 'typing'
211 else:
212 # User defined or third-party TypeVar: __module__ attribute is
213 # irrelevant, thus trigger a exhaustive search for obj in all
214 # modules.
215 module_name = None
216 else:
217 module_name = getattr(obj, '__module__', None)
219 if module_name is not None:
220 return module_name
221 # Protect the iteration by using a copy of sys.modules against dynamic
222 # modules that trigger imports of other modules upon calls to getattr or
223 # other threads importing at the same time.
224 for module_name, module in sys.modules.copy().items():
225 # Some modules such as coverage can inject non-module objects inside
226 # sys.modules
227 if (
228 module_name == '__main__' or
229 module is None or
230 not isinstance(module, types.ModuleType)
231 ):
232 continue
233 try:
234 if _getattribute(module, name)[0] is obj:
235 return module_name
236 except Exception:
237 pass
238 return None
241def _should_pickle_by_reference(obj, name=None):
242 """Test whether an function or a class should be pickled by reference
244 Pickling by reference means by that the object (typically a function or a
245 class) is an attribute of a module that is assumed to be importable in the
246 target Python environment. Loading will therefore rely on importing the
247 module and then calling `getattr` on it to access the function or class.
249 Pickling by reference is the only option to pickle functions and classes
250 in the standard library. In cloudpickle the alternative option is to
251 pickle by value (for instance for interactively or locally defined
252 functions and classes or for attributes of modules that have been
253 explicitly registered to be pickled by value.
254 """
255 if isinstance(obj, types.FunctionType) or issubclass(type(obj), type):
256 module_and_name = _lookup_module_and_qualname(obj, name=name)
257 if module_and_name is None:
258 return False
259 module, name = module_and_name
260 return not _is_registered_pickle_by_value(module)
262 elif isinstance(obj, types.ModuleType):
263 # We assume that sys.modules is primarily used as a cache mechanism for
264 # the Python import machinery. Checking if a module has been added in
265 # is sys.modules therefore a cheap and simple heuristic to tell us
266 # whether we can assume that a given module could be imported by name
267 # in another Python process.
268 if _is_registered_pickle_by_value(obj):
269 return False
270 return obj.__name__ in sys.modules
271 else:
272 raise TypeError(
273 "cannot check importability of {} instances".format(
274 type(obj).__name__)
275 )
278def _lookup_module_and_qualname(obj, name=None):
279 if name is None:
280 name = getattr(obj, '__qualname__', None)
281 if name is None: # pragma: no cover
282 # This used to be needed for Python 2.7 support but is probably not
283 # needed anymore. However we keep the __name__ introspection in case
284 # users of cloudpickle rely on this old behavior for unknown reasons.
285 name = getattr(obj, '__name__', None)
287 module_name = _whichmodule(obj, name)
289 if module_name is None:
290 # In this case, obj.__module__ is None AND obj was not found in any
291 # imported module. obj is thus treated as dynamic.
292 return None
294 if module_name == "__main__":
295 return None
297 # Note: if module_name is in sys.modules, the corresponding module is
298 # assumed importable at unpickling time. See #357
299 module = sys.modules.get(module_name, None)
300 if module is None:
301 # The main reason why obj's module would not be imported is that this
302 # module has been dynamically created, using for example
303 # types.ModuleType. The other possibility is that module was removed
304 # from sys.modules after obj was created/imported. But this case is not
305 # supported, as the standard pickle does not support it either.
306 return None
308 try:
309 obj2, parent = _getattribute(module, name)
310 except AttributeError:
311 # obj was not found inside the module it points to
312 return None
313 if obj2 is not obj:
314 return None
315 return module, name
318def _extract_code_globals(co):
319 """
320 Find all globals names read or written to by codeblock co
321 """
322 out_names = _extract_code_globals_cache.get(co)
323 if out_names is None:
324 # We use a dict with None values instead of a set to get a
325 # deterministic order (assuming Python 3.6+) and avoid introducing
326 # non-deterministic pickle bytes as a results.
327 out_names = {name: None for name in _walk_global_ops(co)}
329 # Declaring a function inside another one using the "def ..."
330 # syntax generates a constant code object corresponding to the one
331 # of the nested function's As the nested function may itself need
332 # global variables, we need to introspect its code, extract its
333 # globals, (look for code object in it's co_consts attribute..) and
334 # add the result to code_globals
335 if co.co_consts:
336 for const in co.co_consts:
337 if isinstance(const, types.CodeType):
338 out_names.update(_extract_code_globals(const))
340 _extract_code_globals_cache[co] = out_names
342 return out_names
345def _find_imported_submodules(code, top_level_dependencies):
346 """
347 Find currently imported submodules used by a function.
349 Submodules used by a function need to be detected and referenced for the
350 function to work correctly at depickling time. Because submodules can be
351 referenced as attribute of their parent package (``package.submodule``), we
352 need a special introspection technique that does not rely on GLOBAL-related
353 opcodes to find references of them in a code object.
355 Example:
356 ```
357 import concurrent.futures
358 import cloudpickle
359 def func():
360 x = concurrent.futures.ThreadPoolExecutor
361 if __name__ == '__main__':
362 cloudpickle.dumps(func)
363 ```
364 The globals extracted by cloudpickle in the function's state include the
365 concurrent package, but not its submodule (here, concurrent.futures), which
366 is the module used by func. Find_imported_submodules will detect the usage
367 of concurrent.futures. Saving this module alongside with func will ensure
368 that calling func once depickled does not fail due to concurrent.futures
369 not being imported
370 """
372 subimports = []
373 # check if any known dependency is an imported package
374 for x in top_level_dependencies:
375 if (isinstance(x, types.ModuleType) and
376 hasattr(x, '__package__') and x.__package__):
377 # check if the package has any currently loaded sub-imports
378 prefix = x.__name__ + '.'
379 # A concurrent thread could mutate sys.modules,
380 # make sure we iterate over a copy to avoid exceptions
381 for name in list(sys.modules):
382 # Older versions of pytest will add a "None" module to
383 # sys.modules.
384 if name is not None and name.startswith(prefix):
385 # check whether the function can address the sub-module
386 tokens = set(name[len(prefix):].split('.'))
387 if not tokens - set(code.co_names):
388 subimports.append(sys.modules[name])
389 return subimports
392def cell_set(cell, value):
393 """Set the value of a closure cell.
395 The point of this function is to set the cell_contents attribute of a cell
396 after its creation. This operation is necessary in case the cell contains a
397 reference to the function the cell belongs to, as when calling the
398 function's constructor
399 ``f = types.FunctionType(code, globals, name, argdefs, closure)``,
400 closure will not be able to contain the yet-to-be-created f.
402 In Python3.7, cell_contents is writeable, so setting the contents of a cell
403 can be done simply using
404 >>> cell.cell_contents = value
406 In earlier Python3 versions, the cell_contents attribute of a cell is read
407 only, but this limitation can be worked around by leveraging the Python 3
408 ``nonlocal`` keyword.
410 In Python2 however, this attribute is read only, and there is no
411 ``nonlocal`` keyword. For this reason, we need to come up with more
412 complicated hacks to set this attribute.
414 The chosen approach is to create a function with a STORE_DEREF opcode,
415 which sets the content of a closure variable. Typically:
417 >>> def inner(value):
418 ... lambda: cell # the lambda makes cell a closure
419 ... cell = value # cell is a closure, so this triggers a STORE_DEREF
421 (Note that in Python2, A STORE_DEREF can never be triggered from an inner
422 function. The function g for example here
423 >>> def f(var):
424 ... def g():
425 ... var += 1
426 ... return g
428 will not modify the closure variable ``var```inplace, but instead try to
429 load a local variable var and increment it. As g does not assign the local
430 variable ``var`` any initial value, calling f(1)() will fail at runtime.)
432 Our objective is to set the value of a given cell ``cell``. So we need to
433 somewhat reference our ``cell`` object into the ``inner`` function so that
434 this object (and not the smoke cell of the lambda function) gets affected
435 by the STORE_DEREF operation.
437 In inner, ``cell`` is referenced as a cell variable (an enclosing variable
438 that is referenced by the inner function). If we create a new function
439 cell_set with the exact same code as ``inner``, but with ``cell`` marked as
440 a free variable instead, the STORE_DEREF will be applied on its closure -
441 ``cell``, which we can specify explicitly during construction! The new
442 cell_set variable thus actually sets the contents of a specified cell!
444 Note: we do not make use of the ``nonlocal`` keyword to set the contents of
445 a cell in early python3 versions to limit possible syntax errors in case
446 test and checker libraries decide to parse the whole file.
447 """
449 if sys.version_info[:2] >= (3, 7): # pragma: no branch
450 cell.cell_contents = value
451 else:
452 _cell_set = types.FunctionType(
453 _cell_set_template_code, {}, '_cell_set', (), (cell,),)
454 _cell_set(value)
457def _make_cell_set_template_code():
458 def _cell_set_factory(value):
459 lambda: cell
460 cell = value
462 co = _cell_set_factory.__code__
464 _cell_set_template_code = types.CodeType(
465 co.co_argcount,
466 co.co_kwonlyargcount, # Python 3 only argument
467 co.co_nlocals,
468 co.co_stacksize,
469 co.co_flags,
470 co.co_code,
471 co.co_consts,
472 co.co_names,
473 co.co_varnames,
474 co.co_filename,
475 co.co_name,
476 co.co_firstlineno,
477 co.co_lnotab,
478 co.co_cellvars, # co_freevars is initialized with co_cellvars
479 (), # co_cellvars is made empty
480 )
481 return _cell_set_template_code
484if sys.version_info[:2] < (3, 7):
485 _cell_set_template_code = _make_cell_set_template_code()
487# relevant opcodes
488STORE_GLOBAL = opcode.opmap['STORE_GLOBAL']
489DELETE_GLOBAL = opcode.opmap['DELETE_GLOBAL']
490LOAD_GLOBAL = opcode.opmap['LOAD_GLOBAL']
491GLOBAL_OPS = (STORE_GLOBAL, DELETE_GLOBAL, LOAD_GLOBAL)
492HAVE_ARGUMENT = dis.HAVE_ARGUMENT
493EXTENDED_ARG = dis.EXTENDED_ARG
496_BUILTIN_TYPE_NAMES = {}
497for k, v in types.__dict__.items():
498 if type(v) is type:
499 _BUILTIN_TYPE_NAMES[v] = k
502def _builtin_type(name):
503 if name == "ClassType": # pragma: no cover
504 # Backward compat to load pickle files generated with cloudpickle
505 # < 1.3 even if loading pickle files from older versions is not
506 # officially supported.
507 return type
508 return getattr(types, name)
511def _walk_global_ops(code):
512 """
513 Yield referenced name for all global-referencing instructions in *code*.
514 """
515 for instr in dis.get_instructions(code):
516 op = instr.opcode
517 if op in GLOBAL_OPS:
518 yield instr.argval
521def _extract_class_dict(cls):
522 """Retrieve a copy of the dict of a class without the inherited methods"""
523 clsdict = dict(cls.__dict__) # copy dict proxy to a dict
524 if len(cls.__bases__) == 1:
525 inherited_dict = cls.__bases__[0].__dict__
526 else:
527 inherited_dict = {}
528 for base in reversed(cls.__bases__):
529 inherited_dict.update(base.__dict__)
530 to_remove = []
531 for name, value in clsdict.items():
532 try:
533 base_value = inherited_dict[name]
534 if value is base_value:
535 to_remove.append(name)
536 except KeyError:
537 pass
538 for name in to_remove:
539 clsdict.pop(name)
540 return clsdict
543if sys.version_info[:2] < (3, 7): # pragma: no branch
544 def _is_parametrized_type_hint(obj):
545 # This is very cheap but might generate false positives. So try to
546 # narrow it down is good as possible.
547 type_module = getattr(type(obj), '__module__', None)
548 from_typing_extensions = type_module == 'typing_extensions'
549 from_typing = type_module == 'typing'
551 # general typing Constructs
552 is_typing = getattr(obj, '__origin__', None) is not None
554 # typing_extensions.Literal
555 is_literal = (
556 (getattr(obj, '__values__', None) is not None)
557 and from_typing_extensions
558 )
560 # typing_extensions.Final
561 is_final = (
562 (getattr(obj, '__type__', None) is not None)
563 and from_typing_extensions
564 )
566 # typing.ClassVar
567 is_classvar = (
568 (getattr(obj, '__type__', None) is not None) and from_typing
569 )
571 # typing.Union/Tuple for old Python 3.5
572 is_union = getattr(obj, '__union_params__', None) is not None
573 is_tuple = getattr(obj, '__tuple_params__', None) is not None
574 is_callable = (
575 getattr(obj, '__result__', None) is not None and
576 getattr(obj, '__args__', None) is not None
577 )
578 return any((is_typing, is_literal, is_final, is_classvar, is_union,
579 is_tuple, is_callable))
581 def _create_parametrized_type_hint(origin, args):
582 return origin[args]
583else:
584 _is_parametrized_type_hint = None
585 _create_parametrized_type_hint = None
588def parametrized_type_hint_getinitargs(obj):
589 # The distorted type check sematic for typing construct becomes:
590 # ``type(obj) is type(TypeHint)``, which means "obj is a
591 # parametrized TypeHint"
592 if type(obj) is type(Literal): # pragma: no branch
593 initargs = (Literal, obj.__values__)
594 elif type(obj) is type(Final): # pragma: no branch
595 initargs = (Final, obj.__type__)
596 elif type(obj) is type(ClassVar):
597 initargs = (ClassVar, obj.__type__)
598 elif type(obj) is type(Generic):
599 initargs = (obj.__origin__, obj.__args__)
600 elif type(obj) is type(Union):
601 initargs = (Union, obj.__args__)
602 elif type(obj) is type(Tuple):
603 initargs = (Tuple, obj.__args__)
604 elif type(obj) is type(Callable):
605 (*args, result) = obj.__args__
606 if len(args) == 1 and args[0] is Ellipsis:
607 args = Ellipsis
608 else:
609 args = list(args)
610 initargs = (Callable, (args, result))
611 else: # pragma: no cover
612 raise pickle.PicklingError(
613 f"Cloudpickle Error: Unknown type {type(obj)}"
614 )
615 return initargs
618# Tornado support
620def is_tornado_coroutine(func):
621 """
622 Return whether *func* is a Tornado coroutine function.
623 Running coroutines are not supported.
624 """
625 if 'tornado.gen' not in sys.modules:
626 return False
627 gen = sys.modules['tornado.gen']
628 if not hasattr(gen, "is_coroutine_function"):
629 # Tornado version is too old
630 return False
631 return gen.is_coroutine_function(func)
634def _rebuild_tornado_coroutine(func):
635 from tornado import gen
636 return gen.coroutine(func)
639# including pickles unloading functions in this namespace
640load = pickle.load
641loads = pickle.loads
644def subimport(name):
645 # We cannot do simply: `return __import__(name)`: Indeed, if ``name`` is
646 # the name of a submodule, __import__ will return the top-level root module
647 # of this submodule. For instance, __import__('os.path') returns the `os`
648 # module.
649 __import__(name)
650 return sys.modules[name]
653def dynamic_subimport(name, vars):
654 mod = types.ModuleType(name)
655 mod.__dict__.update(vars)
656 mod.__dict__['__builtins__'] = builtins.__dict__
657 return mod
660def _gen_ellipsis():
661 return Ellipsis
664def _gen_not_implemented():
665 return NotImplemented
668def _get_cell_contents(cell):
669 try:
670 return cell.cell_contents
671 except ValueError:
672 # sentinel used by ``_fill_function`` which will leave the cell empty
673 return _empty_cell_value
676def instance(cls):
677 """Create a new instance of a class.
679 Parameters
680 ----------
681 cls : type
682 The class to create an instance of.
684 Returns
685 -------
686 instance : cls
687 A new instance of ``cls``.
688 """
689 return cls()
692@instance
693class _empty_cell_value:
694 """sentinel for empty closures
695 """
696 @classmethod
697 def __reduce__(cls):
698 return cls.__name__
701def _fill_function(*args):
702 """Fills in the rest of function data into the skeleton function object
704 The skeleton itself is create by _make_skel_func().
705 """
706 if len(args) == 2:
707 func = args[0]
708 state = args[1]
709 elif len(args) == 5:
710 # Backwards compat for cloudpickle v0.4.0, after which the `module`
711 # argument was introduced
712 func = args[0]
713 keys = ['globals', 'defaults', 'dict', 'closure_values']
714 state = dict(zip(keys, args[1:]))
715 elif len(args) == 6:
716 # Backwards compat for cloudpickle v0.4.1, after which the function
717 # state was passed as a dict to the _fill_function it-self.
718 func = args[0]
719 keys = ['globals', 'defaults', 'dict', 'module', 'closure_values']
720 state = dict(zip(keys, args[1:]))
721 else:
722 raise ValueError(f'Unexpected _fill_value arguments: {args!r}')
724 # - At pickling time, any dynamic global variable used by func is
725 # serialized by value (in state['globals']).
726 # - At unpickling time, func's __globals__ attribute is initialized by
727 # first retrieving an empty isolated namespace that will be shared
728 # with other functions pickled from the same original module
729 # by the same CloudPickler instance and then updated with the
730 # content of state['globals'] to populate the shared isolated
731 # namespace with all the global variables that are specifically
732 # referenced for this function.
733 func.__globals__.update(state['globals'])
735 func.__defaults__ = state['defaults']
736 func.__dict__ = state['dict']
737 if 'annotations' in state:
738 func.__annotations__ = state['annotations']
739 if 'doc' in state:
740 func.__doc__ = state['doc']
741 if 'name' in state:
742 func.__name__ = state['name']
743 if 'module' in state:
744 func.__module__ = state['module']
745 if 'qualname' in state:
746 func.__qualname__ = state['qualname']
747 if 'kwdefaults' in state:
748 func.__kwdefaults__ = state['kwdefaults']
749 # _cloudpickle_subimports is a set of submodules that must be loaded for
750 # the pickled function to work correctly at unpickling time. Now that these
751 # submodules are depickled (hence imported), they can be removed from the
752 # object's state (the object state only served as a reference holder to
753 # these submodules)
754 if '_cloudpickle_submodules' in state:
755 state.pop('_cloudpickle_submodules')
757 cells = func.__closure__
758 if cells is not None:
759 for cell, value in zip(cells, state['closure_values']):
760 if value is not _empty_cell_value:
761 cell_set(cell, value)
763 return func
766def _make_function(code, globals, name, argdefs, closure):
767 # Setting __builtins__ in globals is needed for nogil CPython.
768 globals["__builtins__"] = __builtins__
769 return types.FunctionType(code, globals, name, argdefs, closure)
772def _make_empty_cell():
773 if False:
774 # trick the compiler into creating an empty cell in our lambda
775 cell = None
776 raise AssertionError('this route should not be executed')
778 return (lambda: cell).__closure__[0]
781def _make_cell(value=_empty_cell_value):
782 cell = _make_empty_cell()
783 if value is not _empty_cell_value:
784 cell_set(cell, value)
785 return cell
788def _make_skel_func(code, cell_count, base_globals=None):
789 """ Creates a skeleton function object that contains just the provided
790 code and the correct number of cells in func_closure. All other
791 func attributes (e.g. func_globals) are empty.
792 """
793 # This function is deprecated and should be removed in cloudpickle 1.7
794 warnings.warn(
795 "A pickle file created using an old (<=1.4.1) version of cloudpickle "
796 "is currently being loaded. This is not supported by cloudpickle and "
797 "will break in cloudpickle 1.7", category=UserWarning
798 )
799 # This is backward-compatibility code: for cloudpickle versions between
800 # 0.5.4 and 0.7, base_globals could be a string or None. base_globals
801 # should now always be a dictionary.
802 if base_globals is None or isinstance(base_globals, str):
803 base_globals = {}
805 base_globals['__builtins__'] = __builtins__
807 closure = (
808 tuple(_make_empty_cell() for _ in range(cell_count))
809 if cell_count >= 0 else
810 None
811 )
812 return types.FunctionType(code, base_globals, None, None, closure)
815def _make_skeleton_class(type_constructor, name, bases, type_kwargs,
816 class_tracker_id, extra):
817 """Build dynamic class with an empty __dict__ to be filled once memoized
819 If class_tracker_id is not None, try to lookup an existing class definition
820 matching that id. If none is found, track a newly reconstructed class
821 definition under that id so that other instances stemming from the same
822 class id will also reuse this class definition.
824 The "extra" variable is meant to be a dict (or None) that can be used for
825 forward compatibility shall the need arise.
826 """
827 skeleton_class = types.new_class(
828 name, bases, {'metaclass': type_constructor},
829 lambda ns: ns.update(type_kwargs)
830 )
831 return _lookup_class_or_track(class_tracker_id, skeleton_class)
834def _rehydrate_skeleton_class(skeleton_class, class_dict):
835 """Put attributes from `class_dict` back on `skeleton_class`.
837 See CloudPickler.save_dynamic_class for more info.
838 """
839 registry = None
840 for attrname, attr in class_dict.items():
841 if attrname == "_abc_impl":
842 registry = attr
843 else:
844 setattr(skeleton_class, attrname, attr)
845 if registry is not None:
846 for subclass in registry:
847 skeleton_class.register(subclass)
849 return skeleton_class
852def _make_skeleton_enum(bases, name, qualname, members, module,
853 class_tracker_id, extra):
854 """Build dynamic enum with an empty __dict__ to be filled once memoized
856 The creation of the enum class is inspired by the code of
857 EnumMeta._create_.
859 If class_tracker_id is not None, try to lookup an existing enum definition
860 matching that id. If none is found, track a newly reconstructed enum
861 definition under that id so that other instances stemming from the same
862 class id will also reuse this enum definition.
864 The "extra" variable is meant to be a dict (or None) that can be used for
865 forward compatibility shall the need arise.
866 """
867 # enums always inherit from their base Enum class at the last position in
868 # the list of base classes:
869 enum_base = bases[-1]
870 metacls = enum_base.__class__
871 classdict = metacls.__prepare__(name, bases)
873 for member_name, member_value in members.items():
874 classdict[member_name] = member_value
875 enum_class = metacls.__new__(metacls, name, bases, classdict)
876 enum_class.__module__ = module
877 enum_class.__qualname__ = qualname
879 return _lookup_class_or_track(class_tracker_id, enum_class)
882def _make_typevar(name, bound, constraints, covariant, contravariant,
883 class_tracker_id):
884 tv = typing.TypeVar(
885 name, *constraints, bound=bound,
886 covariant=covariant, contravariant=contravariant
887 )
888 if class_tracker_id is not None:
889 return _lookup_class_or_track(class_tracker_id, tv)
890 else: # pragma: nocover
891 # Only for Python 3.5.3 compat.
892 return tv
895def _decompose_typevar(obj):
896 return (
897 obj.__name__, obj.__bound__, obj.__constraints__,
898 obj.__covariant__, obj.__contravariant__,
899 _get_or_create_tracker_id(obj),
900 )
903def _typevar_reduce(obj):
904 # TypeVar instances require the module information hence why we
905 # are not using the _should_pickle_by_reference directly
906 module_and_name = _lookup_module_and_qualname(obj, name=obj.__name__)
908 if module_and_name is None:
909 return (_make_typevar, _decompose_typevar(obj))
910 elif _is_registered_pickle_by_value(module_and_name[0]):
911 return (_make_typevar, _decompose_typevar(obj))
913 return (getattr, module_and_name)
916def _get_bases(typ):
917 if '__orig_bases__' in getattr(typ, '__dict__', {}):
918 # For generic types (see PEP 560)
919 # Note that simply checking `hasattr(typ, '__orig_bases__')` is not
920 # correct. Subclasses of a fully-parameterized generic class does not
921 # have `__orig_bases__` defined, but `hasattr(typ, '__orig_bases__')`
922 # will return True because it's defined in the base class.
923 bases_attr = '__orig_bases__'
924 else:
925 # For regular class objects
926 bases_attr = '__bases__'
927 return getattr(typ, bases_attr)
930def _make_dict_keys(obj, is_ordered=False):
931 if is_ordered:
932 return OrderedDict.fromkeys(obj).keys()
933 else:
934 return dict.fromkeys(obj).keys()
937def _make_dict_values(obj, is_ordered=False):
938 if is_ordered:
939 return OrderedDict((i, _) for i, _ in enumerate(obj)).values()
940 else:
941 return {i: _ for i, _ in enumerate(obj)}.values()
944def _make_dict_items(obj, is_ordered=False):
945 if is_ordered:
946 return OrderedDict(obj).items()
947 else:
948 return obj.items()