Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.10/site-packages/dill/_dill.py: 40%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# -*- coding: utf-8 -*-
2#
3# Author: Mike McKerns (mmckerns @caltech and @uqfoundation)
4# Copyright (c) 2008-2015 California Institute of Technology.
5# Copyright (c) 2016-2026 The Uncertainty Quantification Foundation.
6# License: 3-clause BSD. The full license text is available at:
7# - https://github.com/uqfoundation/dill/blob/master/LICENSE
8"""
9dill: a utility for serialization of python objects
11The primary functions in `dill` are :func:`dump` and
12:func:`dumps` for serialization ("pickling") to a
13file or to a string, respectively, and :func:`load`
14and :func:`loads` for deserialization ("unpickling"),
15similarly, from a file or from a string. Other notable
16functions are :func:`~dill.dump_module` and
17:func:`~dill.load_module`, which are used to save and
18restore module objects, including an interpreter session.
20Based on code written by Oren Tirosh and Armin Ronacher.
21Extended to a (near) full set of the builtin types (in types module),
22and coded to the pickle interface, by <mmckerns@caltech.edu>.
23Initial port to python3 by Jonathan Dobson, continued by mmckerns.
24Tested against "all" python types (Std. Lib. CH 1-15 @ 2.7) by mmckerns.
25Tested against CH16+ Std. Lib. ... TBD.
26"""
28from __future__ import annotations
30__all__ = [
31 'dump','dumps','load','loads','copy',
32 'Pickler','Unpickler','register','pickle','pickles','check',
33 'DEFAULT_PROTOCOL','HIGHEST_PROTOCOL','HANDLE_FMODE','CONTENTS_FMODE','FILE_FMODE',
34 'PickleError','PickleWarning','PicklingError','PicklingWarning','UnpicklingError',
35 'UnpicklingWarning',
36]
38__module__ = 'dill'
40import warnings
41from .logger import adapter as logger
42from .logger import trace as _trace
43log = logger # backward compatibility (see issue #582)
45import os
46import sys
47diff = None
48_use_diff = False
49OLD38 = (sys.hexversion < 0x3080000)
50OLD39 = (sys.hexversion < 0x3090000)
51OLD310 = (sys.hexversion < 0x30a0000)
52OLD312a7 = (sys.hexversion < 0x30c00a7)
53#XXX: get types from .objtypes ?
54import builtins as __builtin__
55from pickle import _Pickler as StockPickler, Unpickler as StockUnpickler
56from pickle import GLOBAL, POP
57from _contextvars import Context as ContextType
58from _thread import LockType
59from _thread import RLock as RLockType
60try:
61 from _thread import _ExceptHookArgs as ExceptHookArgsType
62except ImportError:
63 ExceptHookArgsType = None
64try:
65 from _thread import _ThreadHandle as ThreadHandleType
66except ImportError:
67 ThreadHandleType = None
68#from io import IOBase
69from types import CodeType, FunctionType, MethodType, GeneratorType, \
70 TracebackType, FrameType, ModuleType, BuiltinMethodType
71BufferType = memoryview #XXX: unregistered
72ClassType = type # no 'old-style' classes
73EllipsisType = type(Ellipsis)
74#FileType = IOBase
75NotImplementedType = type(NotImplemented)
76SliceType = slice
77TypeType = type # 'new-style' classes #XXX: unregistered
78XRangeType = range
79from types import MappingProxyType as DictProxyType, new_class
80from pickle import DEFAULT_PROTOCOL, HIGHEST_PROTOCOL, PickleError, PicklingError, UnpicklingError
81import __main__ as _main_module
82import marshal
83import gc
84# import zlib
85import abc
86import dataclasses
87from weakref import ReferenceType, ProxyType, CallableProxyType, KeyedRef
88from collections import OrderedDict
89import _collections
90from enum import Enum, EnumMeta
91from functools import partial
92from operator import itemgetter, attrgetter
93GENERATOR_FAIL = False
94import importlib.machinery
95EXTENSION_SUFFIXES = tuple(importlib.machinery.EXTENSION_SUFFIXES)
96try:
97 import ctypes
98 HAS_CTYPES = True
99 # if using `pypy`, pythonapi is not found
100 IS_PYPY = not hasattr(ctypes, 'pythonapi')
101except ImportError:
102 HAS_CTYPES = False
103 IS_PYPY = False
104OLDPYPY7323 = IS_PYPY and (sys.pypy_version_info[:3] < (7,3,23))
105OLDPYPY8001 = IS_PYPY and (sys.pypy_version_info[:3] < (8,0,1))
106NumpyUfuncType = None
107NumpyDType = None
108NumpyArrayType = None
109try:
110 if not importlib.machinery.PathFinder().find_spec('numpy'):
111 raise ImportError("No module named 'numpy'")
112 NumpyUfuncType = True
113 NumpyDType = True
114 NumpyArrayType = True
115except ImportError:
116 pass
117def __hook__():
118 global NumpyArrayType, NumpyDType, NumpyUfuncType
119 from numpy import ufunc as NumpyUfuncType
120 from numpy import ndarray as NumpyArrayType
121 from numpy import dtype as NumpyDType
122 return True
123if NumpyArrayType: # then has numpy
124 def ndarraysubclassinstance(obj_type):
125 if all((c.__module__, c.__name__) != ('numpy', 'ndarray') for c in obj_type.__mro__):
126 return False
127 # anything below here is a numpy array (or subclass) instance
128 __hook__() # import numpy (so the following works!!!)
129 # verify that __reduce__ has not been overridden
130 if obj_type.__reduce_ex__ is not NumpyArrayType.__reduce_ex__ \
131 or obj_type.__reduce__ is not NumpyArrayType.__reduce__:
132 return False
133 return True
134 def numpyufunc(obj_type):
135 return any((c.__module__, c.__name__) == ('numpy', 'ufunc') for c in obj_type.__mro__)
136 def numpydtype(obj_type):
137 if all((c.__module__, c.__name__) != ('numpy', 'dtype') for c in obj_type.__mro__):
138 return False
139 # anything below here is a numpy dtype
140 __hook__() # import numpy (so the following works!!!)
141 return obj_type is type(NumpyDType) # handles subclasses
142else:
143 def ndarraysubclassinstance(obj): return False
144 def numpyufunc(obj): return False
145 def numpydtype(obj): return False
147from types import GetSetDescriptorType, ClassMethodDescriptorType, \
148 WrapperDescriptorType, MethodDescriptorType, MemberDescriptorType, \
149 MethodWrapperType #XXX: unused
151# make sure to add these 'hand-built' types to _typemap
152CellType = type((lambda x: lambda y: x)(0).__closure__[0])
153PartialType = type(partial(int, base=2))
154SuperType = type(super(Exception, TypeError()))
155ItemGetterType = type(itemgetter(0))
156AttrGetterType = type(attrgetter('__repr__'))
158try:
159 from functools import _lru_cache_wrapper as LRUCacheType
160except ImportError:
161 LRUCacheType = None
163if not isinstance(LRUCacheType, type):
164 LRUCacheType = None
166def get_file_type(*args, **kwargs):
167 open = kwargs.pop("open", __builtin__.open)
168 f = open(os.devnull, *args, **kwargs)
169 t = type(f)
170 f.close()
171 return t
173IS_PYODIDE = sys.platform == 'emscripten'
175FileType = get_file_type('rb', buffering=0)
176TextWrapperType = get_file_type('r', buffering=-1)
177BufferedRandomType = None if IS_PYODIDE else get_file_type('r+b', buffering=-1)
178BufferedReaderType = get_file_type('rb', buffering=-1)
179BufferedWriterType = get_file_type('wb', buffering=-1)
180try:
181 from _pyio import open as _open
182 PyTextWrapperType = get_file_type('r', buffering=-1, open=_open)
183 PyBufferedRandomType = None if IS_PYODIDE else get_file_type('r+b', buffering=-1, open=_open)
184 PyBufferedReaderType = get_file_type('rb', buffering=-1, open=_open)
185 PyBufferedWriterType = get_file_type('wb', buffering=-1, open=_open)
186except ImportError:
187 PyTextWrapperType = PyBufferedRandomType = PyBufferedReaderType = PyBufferedWriterType = None
188from io import BytesIO as StringIO
189InputType = OutputType = None
190from socket import socket as SocketType
191#FIXME: additionally calls ForkingPickler.register several times
192from multiprocessing.reduction import _reduce_socket as reduce_socket
193try: #pragma: no cover
194 IS_IPYTHON = __IPYTHON__ # is True
195 ExitType = None # IPython.core.autocall.ExitAutocall
196 IPYTHON_SINGLETONS = ('exit', 'quit', 'get_ipython')
197except NameError:
198 IS_IPYTHON = False
199 try: ExitType = type(exit) # apparently 'exit' can be removed
200 except NameError: ExitType = None
201 IPYTHON_SINGLETONS = ()
203import inspect
204import typing
207### Shims for different versions of Python and dill
208class Sentinel(object):
209 """
210 Create a unique sentinel object that is pickled as a constant.
211 """
212 def __init__(self, name, module_name=None):
213 self.name = name
214 if module_name is None:
215 # Use the calling frame's module
216 self.__module__ = inspect.currentframe().f_back.f_globals['__name__']
217 else:
218 self.__module__ = module_name # pragma: no cover
219 def __repr__(self):
220 return self.__module__ + '.' + self.name # pragma: no cover
221 def __copy__(self):
222 return self # pragma: no cover
223 def __deepcopy__(self, memo):
224 return self # pragma: no cover
225 def __reduce__(self):
226 return self.name
227 def __reduce_ex__(self, protocol):
228 return self.name
230from . import _shims
231from ._shims import Reduce, Getattr
233### File modes
234#: Pickles the file handle, preserving mode. The position of the unpickled
235#: object is as for a new file handle.
236HANDLE_FMODE = 0
237#: Pickles the file contents, creating a new file if on load the file does
238#: not exist. The position = min(pickled position, EOF) and mode is chosen
239#: as such that "best" preserves behavior of the original file.
240CONTENTS_FMODE = 1
241#: Pickles the entire file (handle and contents), preserving mode and position.
242FILE_FMODE = 2
244### Shorthands (modified from python2.5/lib/pickle.py)
245def copy(obj, *args, **kwds):
246 """
247 Use pickling to 'copy' an object (i.e. `loads(dumps(obj))`).
249 See :func:`dumps` and :func:`loads` for keyword arguments.
250 """
251 ignore = kwds.pop('ignore', Unpickler.settings['ignore'])
252 return loads(dumps(obj, *args, **kwds), ignore=ignore)
254def dump(obj, file, protocol=None, byref=None, fmode=None, recurse=None, **kwds):#, strictio=None):
255 """
256 Pickle an object to a file.
258 See :func:`dumps` for keyword arguments.
259 """
260 from .settings import settings
261 protocol = settings['protocol'] if protocol is None else int(protocol)
262 _kwds = kwds.copy()
263 _kwds.update(dict(byref=byref, fmode=fmode, recurse=recurse))
264 Pickler(file, protocol, **_kwds).dump(obj)
265 return
267def dumps(obj, protocol=None, byref=None, fmode=None, recurse=None, **kwds):#, strictio=None):
268 """
269 Pickle an object to a string.
271 *protocol* is the pickler protocol, as defined for Python *pickle*.
273 If *byref=True*, then dill behaves a lot more like pickle as certain
274 objects (like modules) are pickled by reference as opposed to attempting
275 to pickle the object itself.
277 If *recurse=True*, then objects referred to in the global dictionary
278 are recursively traced and pickled, instead of the default behavior
279 of attempting to store the entire global dictionary. This is needed for
280 functions defined via *exec()*.
282 *fmode* (:const:`HANDLE_FMODE`, :const:`CONTENTS_FMODE`,
283 or :const:`FILE_FMODE`) indicates how file handles will be pickled.
284 For example, when pickling a data file handle for transfer to a remote
285 compute service, *FILE_FMODE* will include the file contents in the
286 pickle and cursor position so that a remote method can operate
287 transparently on an object with an open file handle.
289 Default values for keyword arguments can be set in :mod:`dill.settings`.
290 """
291 file = StringIO()
292 dump(obj, file, protocol, byref, fmode, recurse, **kwds)#, strictio)
293 return file.getvalue()
295def load(file, ignore=None, **kwds):
296 """
297 Unpickle an object from a file.
299 See :func:`loads` for keyword arguments.
300 """
301 return Unpickler(file, ignore=ignore, **kwds).load()
303def loads(str, ignore=None, **kwds):
304 """
305 Unpickle an object from a string.
307 If *ignore=False* then objects whose class is defined in the module
308 *__main__* are updated to reference the existing class in *__main__*,
309 otherwise they are left to refer to the reconstructed type, which may
310 be different.
312 Default values for keyword arguments can be set in :mod:`dill.settings`.
313 """
314 file = StringIO(str)
315 return load(file, ignore, **kwds)
317# def dumpzs(obj, protocol=None):
318# """pickle an object to a compressed string"""
319# return zlib.compress(dumps(obj, protocol))
321# def loadzs(str):
322# """unpickle an object from a compressed string"""
323# return loads(zlib.decompress(str))
325### End: Shorthands ###
327class MetaCatchingDict(dict):
328 def get(self, key, default=None):
329 try:
330 return self[key]
331 except KeyError:
332 return default
334 def __missing__(self, key):
335 if issubclass(key, type):
336 return save_type
337 else:
338 raise KeyError()
340class PickleWarning(Warning, PickleError):
341 pass
343class PicklingWarning(PickleWarning, PicklingError):
344 pass
346class UnpicklingWarning(PickleWarning, UnpicklingError):
347 pass
349### Extend the Picklers
350class Pickler(StockPickler):
351 """python's Pickler extended to interpreter sessions"""
352 dispatch: typing.Dict[type, typing.Callable[[Pickler, typing.Any], None]] \
353 = MetaCatchingDict(StockPickler.dispatch.copy())
354 """The dispatch table, a dictionary of serializing functions used
355 by Pickler to save objects of specific types. Use :func:`pickle`
356 or :func:`register` to associate types to custom functions.
358 :meta hide-value:
359 """
360 _session = False
361 from .settings import settings
363 def __init__(self, file, *args, **kwds):
364 settings = Pickler.settings
365 _byref = kwds.pop('byref', None)
366 #_strictio = kwds.pop('strictio', None)
367 _fmode = kwds.pop('fmode', None)
368 _recurse = kwds.pop('recurse', None)
369 StockPickler.__init__(self, file, *args, **kwds)
370 self._main = _main_module
371 self._diff_cache = {}
372 self._byref = settings['byref'] if _byref is None else _byref
373 self._strictio = False #_strictio
374 self._fmode = settings['fmode'] if _fmode is None else _fmode
375 self._recurse = settings['recurse'] if _recurse is None else _recurse
376 self._postproc = OrderedDict()
377 self._file = file
379 def save(self, obj, save_persistent_id=True):
380 # numpy hack
381 obj_type = type(obj)
382 if NumpyArrayType and not (obj_type is type or obj_type in Pickler.dispatch):
383 # register if the object is a numpy ufunc
384 # thanks to Paul Kienzle for pointing out ufuncs didn't pickle
385 if numpyufunc(obj_type):
386 @register(obj_type)
387 def save_numpy_ufunc(pickler, obj):
388 logger.trace(pickler, "Nu: %s", obj)
389 name = getattr(obj, '__qualname__', getattr(obj, '__name__', None))
390 StockPickler.save_global(pickler, obj, name=name)
391 logger.trace(pickler, "# Nu")
392 return
393 # NOTE: the above 'save' performs like:
394 # import copy_reg
395 # def udump(f): return f.__name__
396 # def uload(name): return getattr(numpy, name)
397 # copy_reg.pickle(NumpyUfuncType, udump, uload)
398 # register if the object is a numpy dtype
399 if numpydtype(obj_type):
400 @register(obj_type)
401 def save_numpy_dtype(pickler, obj):
402 logger.trace(pickler, "Dt: %s", obj)
403 pickler.save_reduce(_create_dtypemeta, (obj.type,), obj=obj)
404 logger.trace(pickler, "# Dt")
405 return
406 # NOTE: the above 'save' performs like:
407 # import copy_reg
408 # def uload(name): return type(NumpyDType(name))
409 # def udump(f): return uload, (f.type,)
410 # copy_reg.pickle(NumpyDTypeType, udump, uload)
411 # register if the object is a subclassed numpy array instance
412 if ndarraysubclassinstance(obj_type):
413 @register(obj_type)
414 def save_numpy_array(pickler, obj):
415 logger.trace(pickler, "Nu: (%s, %s)", obj.shape, obj.dtype)
416 npdict = getattr(obj, '__dict__', None)
417 f, args, state = obj.__reduce__()
418 pickler.save_reduce(_create_array, (f,args,state,npdict), obj=obj)
419 logger.trace(pickler, "# Nu")
420 return
421 # end numpy hack
423 if GENERATOR_FAIL and obj_type is GeneratorType:
424 msg = "Can't pickle %s: attribute lookup builtins.generator failed" % GeneratorType
425 raise PicklingError(msg)
426 StockPickler.save(self, obj, save_persistent_id)
428 save.__doc__ = StockPickler.save.__doc__
430 def dump(self, obj): #NOTE: if settings change, need to update attributes
431 logger.trace_setup(self)
432 StockPickler.dump(self, obj)
433 dump.__doc__ = StockPickler.dump.__doc__
435class Unpickler(StockUnpickler):
436 """python's Unpickler extended to interpreter sessions and more types"""
437 from .settings import settings
438 _session = False
440 def find_class(self, module, name):
441 if (module, name) == ('__builtin__', '__main__'):
442 return self._main.__dict__ #XXX: above set w/save_module_dict
443 elif (module, name) == ('__builtin__', 'NoneType'):
444 return type(None) #XXX: special case: NoneType missing
445 if module == 'dill.dill': module = 'dill._dill'
446 return StockUnpickler.find_class(self, module, name)
448 def __init__(self, *args, **kwds):
449 settings = Pickler.settings
450 _ignore = kwds.pop('ignore', None)
451 StockUnpickler.__init__(self, *args, **kwds)
452 self._main = _main_module
453 self._ignore = settings['ignore'] if _ignore is None else _ignore
455 def load(self): #NOTE: if settings change, need to update attributes
456 obj = StockUnpickler.load(self)
457 if type(obj).__module__ == getattr(_main_module, '__name__', '__main__'):
458 if not self._ignore:
459 # point obj class to main
460 try: obj.__class__ = getattr(self._main, type(obj).__name__)
461 except (AttributeError,TypeError): pass # defined in a file
462 #_main_module.__dict__.update(obj.__dict__) #XXX: should update globals ?
463 return obj
464 load.__doc__ = StockUnpickler.load.__doc__
465 pass
467'''
468def dispatch_table():
469 """get the dispatch table of registered types"""
470 return Pickler.dispatch
471'''
473pickle_dispatch_copy = StockPickler.dispatch.copy()
475def pickle(t, func):
476 """expose :attr:`~Pickler.dispatch` table for user-created extensions"""
477 Pickler.dispatch[t] = func
478 return
480def register(t):
481 """decorator to register types to Pickler's :attr:`~Pickler.dispatch` table"""
482 def proxy(func):
483 Pickler.dispatch[t] = func
484 return func
485 return proxy
487def _revert_extension():
488 """drop dill-registered types from pickle's dispatch table"""
489 for type, func in list(StockPickler.dispatch.items()):
490 if func.__module__ == __name__:
491 del StockPickler.dispatch[type]
492 if type in pickle_dispatch_copy:
493 StockPickler.dispatch[type] = pickle_dispatch_copy[type]
495def use_diff(on=True):
496 """
497 Reduces size of pickles by only including object which have changed.
499 Decreases pickle size but increases CPU time needed.
500 Also helps avoid some unpickleable objects.
501 MUST be called at start of script, otherwise changes will not be recorded.
502 """
503 global _use_diff, diff
504 _use_diff = on
505 if _use_diff and diff is None:
506 try:
507 from . import diff as d
508 except ImportError:
509 import diff as d
510 diff = d
512def _create_typemap():
513 import types
514 d = dict(list(__builtin__.__dict__.items()) + \
515 list(types.__dict__.items())).items()
516 for key, value in d:
517 if getattr(value, '__module__', None) == 'builtins' \
518 and type(value) is type:
519 yield key, value
520 return
521_reverse_typemap = dict(_create_typemap())
522_reverse_typemap.update({
523 'PartialType': PartialType,
524 'SuperType': SuperType,
525 'ItemGetterType': ItemGetterType,
526 'AttrGetterType': AttrGetterType,
527})
528if sys.hexversion < 0x30800a2:
529 _reverse_typemap.update({
530 'CellType': CellType,
531 })
533# "Incidental" implementation specific types. Unpickling these types in another
534# implementation of Python (PyPy -> CPython) is not guaranteed to work
536# This dictionary should contain all types that appear in Python implementations
537# but are not defined in https://docs.python.org/3/library/types.html#standard-interpreter-types
538x=OrderedDict()
539_incedental_reverse_typemap = {
540 'FileType': FileType,
541 'BufferedRandomType': BufferedRandomType,
542 'BufferedReaderType': BufferedReaderType,
543 'BufferedWriterType': BufferedWriterType,
544 'TextWrapperType': TextWrapperType,
545 'PyBufferedRandomType': PyBufferedRandomType,
546 'PyBufferedReaderType': PyBufferedReaderType,
547 'PyBufferedWriterType': PyBufferedWriterType,
548 'PyTextWrapperType': PyTextWrapperType,
549}
551_incedental_reverse_typemap.update({
552 "DictKeysType": type({}.keys()),
553 "DictValuesType": type({}.values()),
554 "DictItemsType": type({}.items()),
556 "OdictKeysType": type(x.keys()),
557 "OdictValuesType": type(x.values()),
558 "OdictItemsType": type(x.items()),
559})
561if ExitType:
562 _incedental_reverse_typemap['ExitType'] = ExitType
563if InputType:
564 _incedental_reverse_typemap['InputType'] = InputType
565 _incedental_reverse_typemap['OutputType'] = OutputType
567'''
568try:
569 import symtable
570 _incedental_reverse_typemap["SymtableEntryType"] = type(symtable.symtable("", "string", "exec")._table)
571except: #FIXME: fails to pickle
572 pass
574if sys.hexversion >= 0x30a00a0:
575 _incedental_reverse_typemap['LineIteratorType'] = type(compile('3', '', 'eval').co_lines())
576'''
578if sys.hexversion >= 0x30b00b0 and not IS_PYPY:
579 from types import GenericAlias
580 _incedental_reverse_typemap["GenericAliasIteratorType"] = type(iter(GenericAlias(list, (int,))))
581 '''
582 _incedental_reverse_typemap['PositionsIteratorType'] = type(compile('3', '', 'eval').co_positions())
583 '''
585try:
586 import winreg
587 _incedental_reverse_typemap["HKEYType"] = winreg.HKEYType
588except ImportError:
589 pass
591_reverse_typemap.update(_incedental_reverse_typemap)
592_incedental_types = set(_incedental_reverse_typemap.values())
594del x
596_typemap = dict((v, k) for k, v in _reverse_typemap.items())
598def _unmarshal(string):
599 return marshal.loads(string)
601def _load_type(name):
602 return _reverse_typemap[name]
604def _create_type(typeobj, *args):
605 return typeobj(*args)
607def _create_function(fcode, fglobals, fname=None, fdefaults=None,
608 fclosure=None, fdict=None, fkwdefaults=None):
609 # same as FunctionType, but enable passing __dict__ to new function,
610 # __dict__ is the storehouse for attributes added after function creation
611 func = FunctionType(fcode, fglobals or dict(), fname, fdefaults, fclosure)
612 if fdict is not None:
613 func.__dict__.update(fdict) #XXX: better copy? option to copy?
614 if fkwdefaults is not None:
615 func.__kwdefaults__ = fkwdefaults
616 # 'recurse' only stores referenced modules/objects in fglobals,
617 # thus we need to make sure that we have __builtins__ as well
618 if "__builtins__" not in func.__globals__:
619 func.__globals__["__builtins__"] = globals()["__builtins__"]
620 # assert id(fglobals) == id(func.__globals__)
621 return func
623class match:
624 """
625 Make available a limited structural pattern matching-like syntax for Python < 3.10
627 Patterns can be only tuples (without types) currently.
628 Inspired by the package pattern-matching-PEP634.
630 Usage:
631 >>> with match(args) as m:
632 >>> if m.case(('x', 'y')):
633 >>> # use m.x and m.y
634 >>> elif m.case(('x', 'y', 'z')):
635 >>> # use m.x, m.y and m.z
637 Equivalent native code for Python >= 3.10:
638 >>> match args:
639 >>> case (x, y):
640 >>> # use x and y
641 >>> case (x, y, z):
642 >>> # use x, y and z
643 """
644 def __init__(self, value):
645 self.value = value
646 self._fields = None
647 def __enter__(self):
648 return self
649 def __exit__(self, *exc_info):
650 return False
651 def case(self, args): # *args, **kwargs):
652 """just handles tuple patterns"""
653 if len(self.value) != len(args): # + len(kwargs):
654 return False
655 #if not all(isinstance(arg, pat) for arg, pat in zip(self.value[len(args):], kwargs.values())):
656 # return False
657 self.args = args # (*args, *kwargs)
658 return True
659 @property
660 def fields(self):
661 # Only bind names to values if necessary.
662 if self._fields is None:
663 self._fields = dict(zip(self.args, self.value))
664 return self._fields
665 def __getattr__(self, item):
666 return self.fields[item]
668ALL_CODE_PARAMS = [
669 # Version New attribute CodeType parameters
670 ((3,11,'a'), 'co_endlinetable', 'argcount posonlyargcount kwonlyargcount nlocals stacksize flags code consts names varnames filename name qualname firstlineno linetable endlinetable columntable exceptiontable freevars cellvars'),
671 ((3,11), 'co_exceptiontable', 'argcount posonlyargcount kwonlyargcount nlocals stacksize flags code consts names varnames filename name qualname firstlineno linetable exceptiontable freevars cellvars'),
672 ((3,11,'p'), 'co_qualname', 'argcount posonlyargcount kwonlyargcount nlocals stacksize flags code consts names varnames filename name qualname firstlineno linetable freevars cellvars'),
673 ((3,10), 'co_linetable', 'argcount posonlyargcount kwonlyargcount nlocals stacksize flags code consts names varnames filename name firstlineno linetable freevars cellvars'),
674 ((3,8), 'co_posonlyargcount', 'argcount posonlyargcount kwonlyargcount nlocals stacksize flags code consts names varnames filename name firstlineno lnotab freevars cellvars'),
675 ((3,7), 'co_kwonlyargcount', 'argcount kwonlyargcount nlocals stacksize flags code consts names varnames filename name firstlineno lnotab freevars cellvars'),
676 ]
677for version, new_attr, params in ALL_CODE_PARAMS:
678 if hasattr(CodeType, new_attr):
679 CODE_VERSION = version
680 CODE_PARAMS = params.split()
681 break
682ENCODE_PARAMS = set(CODE_PARAMS).intersection(
683 ['code', 'lnotab', 'linetable', 'endlinetable', 'columntable', 'exceptiontable'])
685def _create_code(*args):
686 if not isinstance(args[0], int): # co_lnotab stored from >= 3.10
687 LNOTAB, *args = args
688 else: # from < 3.10 (or pre-LNOTAB storage)
689 LNOTAB = b''
691 with match(args) as m:
692 # Python 3.11/3.12a PyPy 3.12 (18 members)
693 if m.case((
694 'argcount', 'posonlyargcount', 'kwonlyargcount', 'nlocals', 'stacksize', 'flags', # args[0:6]
695 'code', 'consts', 'names', 'varnames', 'filename', 'name', 'qualname', 'firstlineno', # args[6:14]
696 'linetable', 'exceptiontable', 'freevars', 'cellvars' # args[14:]
697 )):
698 if CODE_VERSION == (3,11):
699 if IS_PYPY and OLDPYPY8001 and not OLDPYPY7323:
700 args = tuple(args[:15]) + tuple(args[16:18]) + (args[15],)
701 return CodeType(
702 *args[:6],
703 args[6].encode() if hasattr(args[6], 'encode') else args[6], # code
704 *args[7:14],
705 args[14].encode() if hasattr(args[14], 'encode') else args[14], # linetable
706 args[15].encode() if hasattr(args[15], 'encode') else args[15], # exceptiontable
707 args[16],
708 args[17].encode() if hasattr(args[17], 'encode') else args[17], # pypy-3.11.16
709 )
710 fields = m.fields
711 # PyPy 3.11 7.3.19-7.3.21* (17 members)
712 elif m.case((
713 'argcount', 'posonlyargcount', 'kwonlyargcount', 'nlocals', 'stacksize', 'flags', # args[0:6]
714 'code', 'consts', 'names', 'varnames', 'filename', 'name', 'qualname', # args[6:13]
715 'firstlineno', 'linetable', 'freevars', 'cellvars' # args[13:]
716 )):
717 if CODE_VERSION == (3,11,'p'):
718 return CodeType(
719 *args[:6],
720 args[6].encode() if hasattr(args[6], 'encode') else args[6], # code
721 *args[7:14],
722 args[14].encode() if hasattr(args[14], 'encode') else args[14], # linetable
723 args[15],
724 args[16],
725 )
726 fields = m.fields
727 # Python 3.10 or 3.8/3.9 (16 members)
728 elif m.case((
729 'argcount', 'posonlyargcount', 'kwonlyargcount', 'nlocals', 'stacksize', 'flags', # args[0:6]
730 'code', 'consts', 'names', 'varnames', 'filename', 'name', 'firstlineno', # args[6:13]
731 'LNOTAB_OR_LINETABLE', 'freevars', 'cellvars' # args[13:]
732 )):
733 if CODE_VERSION == (3,10) or CODE_VERSION == (3,8):
734 return CodeType(
735 *args[:6],
736 args[6].encode() if hasattr(args[6], 'encode') else args[6], # code
737 *args[7:13],
738 args[13].encode() if hasattr(args[13], 'encode') else args[13], # lnotab/linetable
739 args[14],
740 args[15],
741 )
742 fields = m.fields
743 if CODE_VERSION >= (3,10):
744 fields['linetable'] = m.LNOTAB_OR_LINETABLE
745 else:
746 fields['lnotab'] = LNOTAB if LNOTAB else m.LNOTAB_OR_LINETABLE
747 # Python 3.7 (15 args)
748 elif m.case((
749 'argcount', 'kwonlyargcount', 'nlocals', 'stacksize', 'flags', # args[0:5]
750 'code', 'consts', 'names', 'varnames', 'filename', 'name', 'firstlineno', # args[5:12]
751 'lnotab', 'freevars', 'cellvars' # args[12:]
752 )):
753 if CODE_VERSION == (3,7):
754 return CodeType(
755 *args[:5],
756 args[5].encode() if hasattr(args[5], 'encode') else args[5], # code
757 *args[6:12],
758 args[12].encode() if hasattr(args[12], 'encode') else args[12], # lnotab
759 args[13],
760 args[14],
761 )
762 fields = m.fields
763 # Python 3.11a (20 members)
764 elif m.case((
765 'argcount', 'posonlyargcount', 'kwonlyargcount', 'nlocals', 'stacksize', 'flags', # args[0:6]
766 'code', 'consts', 'names', 'varnames', 'filename', 'name', 'qualname', 'firstlineno', # args[6:14]
767 'linetable', 'endlinetable', 'columntable', 'exceptiontable', 'freevars', 'cellvars' # args[14:]
768 )):
769 if CODE_VERSION == (3,11,'a'):
770 if IS_PYPY and OLDPYPY8001 and not OLDPYPY7323:
771 args = tuple(args[:17]) + tuple(args[18:20]) + (args[17],)
772 return CodeType(
773 *args[:6],
774 args[6].encode() if hasattr(args[6], 'encode') else args[6], # code
775 *args[7:14],
776 *(a.encode() if hasattr(a, 'encode') else a for a in args[14:18]), # linetable-exceptiontable
777 args[18],
778 args[19].encode() if hasattr(args[19], 'encode') else args[19], # pypy-3.11.6
779 )
780 fields = m.fields
781 else:
782 raise UnpicklingError("pattern match for code object failed")
784 # The args format doesn't match this version.
785 fields.setdefault('posonlyargcount', 0) # from python <= 3.7
786 fields.setdefault('lnotab', LNOTAB) # from python >= 3.10
787 fields.setdefault('linetable', b'') # from python <= 3.9
788 fields.setdefault('qualname', fields['name']) # from python <= 3.10
789 fields.setdefault('exceptiontable', b'') # from python <= 3.10
790 fields.setdefault('endlinetable', None) # from python != 3.11a
791 fields.setdefault('columntable', None) # from python != 3.11a
793 args = tuple(fields[k].encode() if k in ENCODE_PARAMS and hasattr(fields[k], 'encode') else fields[k]
794 for k in CODE_PARAMS)
795 if IS_PYPY and OLDPYPY8001 and not OLDPYPY7323:
796 if len(args) > 17:
797 args = tuple(args[:-3]) + tuple(args[-2:]) + (args[-3],)
798 return CodeType(*args)
800def _create_ftype(ftypeobj, func, args, kwds):
801 if kwds is None:
802 kwds = {}
803 if args is None:
804 args = ()
805 return ftypeobj(func, *args, **kwds)
807def _create_typing_tuple(argz, *args): #NOTE: workaround python/cpython#94245
808 if not argz:
809 return typing.Tuple[()].copy_with(())
810 if argz == ((),):
811 return typing.Tuple[()]
812 return typing.Tuple[argz]
814if ThreadHandleType:
815 def _create_thread_handle(ident, done, *args): #XXX: ignores 'blocking'
816 from threading import _make_thread_handle
817 handle = _make_thread_handle(ident)
818 if done:
819 handle._set_done()
820 return handle
822def _create_lock(locked, *args): #XXX: ignores 'blocking'
823 from threading import Lock
824 lock = Lock()
825 if locked:
826 if not lock.acquire(False):
827 raise UnpicklingError("Cannot acquire lock")
828 return lock
830def _create_rlock(count, owner, *args): #XXX: ignores 'blocking'
831 lock = RLockType()
832 if owner is not None:
833 lock._acquire_restore((count, owner))
834 if owner and not lock._is_owned():
835 raise UnpicklingError("Cannot acquire lock")
836 return lock
838# thanks to matsjoyce for adding all the different file modes
839def _create_filehandle(name, mode, position, closed, open, strictio, fmode, fdata): # buffering=0
840 # only pickles the handle, not the file contents... good? or StringIO(data)?
841 # (for file contents see: http://effbot.org/librarybook/copy-reg.htm)
842 # NOTE: handle special cases first (are there more special cases?)
843 names = {'<stdin>':sys.__stdin__, '<stdout>':sys.__stdout__,
844 '<stderr>':sys.__stderr__} #XXX: better fileno=(0,1,2) ?
845 if name in list(names.keys()):
846 f = names[name] #XXX: safer "f=sys.stdin"
847 elif name == '<tmpfile>':
848 f = os.tmpfile()
849 elif name == '<fdopen>':
850 import tempfile
851 f = tempfile.TemporaryFile(mode)
852 else:
853 try:
854 exists = os.path.exists(name)
855 except Exception:
856 exists = False
857 if not exists:
858 if strictio:
859 raise FileNotFoundError("[Errno 2] No such file or directory: '%s'" % name)
860 elif "r" in mode and fmode != FILE_FMODE:
861 name = '<fdopen>' # or os.devnull?
862 current_size = 0 # or maintain position?
863 else:
864 current_size = os.path.getsize(name)
866 if position > current_size:
867 if strictio:
868 raise ValueError("invalid buffer size")
869 elif fmode == CONTENTS_FMODE:
870 position = current_size
871 # try to open the file by name
872 # NOTE: has different fileno
873 try:
874 #FIXME: missing: *buffering*, encoding, softspace
875 if fmode == FILE_FMODE:
876 f = open(name, mode if "w" in mode else
877 "wb" if isinstance(fdata, bytes) else "w")
878 f.write(fdata)
879 if "w" not in mode:
880 f.close()
881 f = open(name, mode)
882 elif name == '<fdopen>': # file did not exist
883 import tempfile
884 f = tempfile.TemporaryFile(mode)
885 # treat x mode as w mode
886 elif fmode == CONTENTS_FMODE \
887 and ("w" in mode or "x" in mode):
888 # stop truncation when opening
889 flags = os.O_CREAT
890 if "+" in mode:
891 flags |= os.O_RDWR
892 else:
893 flags |= os.O_WRONLY
894 f = os.fdopen(os.open(name, flags, 0o600), mode)
895 # set name to the correct value
896 r = getattr(f, "buffer", f)
897 r = getattr(r, "raw", r)
898 r.name = name
899 assert f.name == name
900 else:
901 f = open(name, mode)
902 except (IOError, FileNotFoundError):
903 err = sys.exc_info()[1]
904 raise UnpicklingError(err)
905 if closed:
906 f.close()
907 elif position >= 0 and fmode != HANDLE_FMODE:
908 f.seek(position)
909 return f
911def _create_stringi(value, position, closed):
912 f = StringIO(value)
913 if closed: f.close()
914 else: f.seek(position)
915 return f
917def _create_stringo(value, position, closed):
918 f = StringIO()
919 if closed: f.close()
920 else:
921 f.write(value)
922 f.seek(position)
923 return f
925class _itemgetter_helper(object):
926 def __init__(self):
927 self.items = []
928 def __getitem__(self, item):
929 self.items.append(item)
930 return
932class _attrgetter_helper(object):
933 def __init__(self, attrs, index=None):
934 self.attrs = attrs
935 self.index = index
936 def __getattribute__(self, attr):
937 attrs = object.__getattribute__(self, "attrs")
938 index = object.__getattribute__(self, "index")
939 if index is None:
940 index = len(attrs)
941 attrs.append(attr)
942 else:
943 attrs[index] = ".".join([attrs[index], attr])
944 return type(self)(attrs, index)
946class _dictproxy_helper(dict):
947 def __ror__(self, a):
948 return a
950_dictproxy_helper_instance = _dictproxy_helper()
952__d = {}
953try:
954 # In CPython 3.9 and later, this trick can be used to exploit the
955 # implementation of the __or__ function of MappingProxyType to get the true
956 # mapping referenced by the proxy. It may work for other implementations,
957 # but is not guaranteed.
958 MAPPING_PROXY_TRICK = __d is (DictProxyType(__d) | _dictproxy_helper_instance)
959except Exception:
960 MAPPING_PROXY_TRICK = False
961del __d
963# _CELL_REF and _CELL_EMPTY are used to stay compatible with versions of dill
964# whose _create_cell functions do not have a default value.
965# _CELL_REF can be safely removed entirely (replaced by empty tuples for calls
966# to _create_cell) once breaking changes are allowed.
967_CELL_REF = None
968_CELL_EMPTY = Sentinel('_CELL_EMPTY')
970def _create_cell(contents=None):
971 if contents is not _CELL_EMPTY:
972 value = contents
973 return (lambda: value).__closure__[0]
975def _create_weakref(obj, *args):
976 from weakref import ref
977 if obj is None: # it's dead
978 from collections import UserDict
979 return ref(UserDict(), *args)
980 return ref(obj, *args)
982def _create_keyedref(obj, callback, key):
983 if obj is None: # it's dead
984 from collections import UserDict
985 obj = UserDict()
986 return KeyedRef(obj, callback, key)
988def _create_weakproxy(obj, callable=False, *args):
989 from weakref import proxy
990 if obj is None: # it's dead
991 if callable: return proxy(lambda x:x, *args)
992 from collections import UserDict
993 return proxy(UserDict(), *args)
994 return proxy(obj, *args)
996def _eval_repr(repr_str):
997 return eval(repr_str)
999def _create_array(f, args, state, npdict=None):
1000 #array = numpy.core.multiarray._reconstruct(*args)
1001 array = f(*args)
1002 array.__setstate__(state)
1003 if npdict is not None: # we also have saved state in __dict__
1004 array.__dict__.update(npdict)
1005 return array
1007def _create_dtypemeta(scalar_type):
1008 if NumpyDType is True: __hook__() # a bit hacky I think
1009 if scalar_type is None:
1010 return NumpyDType
1011 return type(NumpyDType(scalar_type))
1013def _create_namedtuple(name, fieldnames, modulename, defaults=None):
1014 class_ = _import_module(modulename + '.' + name, safe=True)
1015 if class_ is not None:
1016 return class_
1017 import collections
1018 t = collections.namedtuple(name, fieldnames, defaults=defaults, module=modulename)
1019 return t
1021def _create_capsule(pointer, name, context, destructor):
1022 attr_found = False
1023 try:
1024 # based on https://github.com/python/cpython/blob/f4095e53ab708d95e019c909d5928502775ba68f/Objects/capsule.c#L209-L231
1025 uname = name.decode('utf8')
1026 for i in range(1, uname.count('.')+1):
1027 names = uname.rsplit('.', i)
1028 try:
1029 obj = __import__(names[0])
1030 for attr in names[1:]:
1031 obj = getattr(obj, attr)
1032 except (ImportError, AttributeError):
1033 continue # this split didn't resolve, try a shorter module path
1034 capsule = obj
1035 attr_found = True
1036 break
1037 except Exception:
1038 pass
1040 if attr_found:
1041 if _PyCapsule_IsValid(capsule, name):
1042 return capsule
1043 raise UnpicklingError("%s object exists at %s but a PyCapsule object was expected." % (type(capsule), name))
1044 else:
1045 #warnings.warn('Creating a new PyCapsule %s for a C data structure that may not be present in memory. Segmentation faults or other memory errors are possible.' % (name,), UnpicklingWarning)
1046 capsule = _PyCapsule_New(pointer, name, destructor)
1047 _PyCapsule_SetContext(capsule, context)
1048 return capsule
1050def _getattr(objclass, name, repr_str):
1051 attr = None
1052 if IS_PYPY:
1053 try: # hack to grab the reference directly
1054 attr = repr_str.split("'")[3]
1055 attr = eval(attr+'.__dict__["'+name+'"]')
1056 except Exception: pass
1057 if attr is None:
1058 # grab the descriptor off its owning class, repr_str is not evaluated
1059 try:
1060 attr = objclass.__dict__
1061 if type(attr) is DictProxyType:
1062 if sys.hexversion > 0x30f00a0 and name in ('__weakref__','__dict__'):
1063 attr = _dictproxy_helper.__dict__[name]
1064 else:
1065 attr = attr[name]
1066 else:
1067 attr = getattr(objclass,name)
1068 except (AttributeError, KeyError):
1069 attr = getattr(objclass,name)
1070 return attr
1072def _get_attr(self, name):
1073 # stop recursive pickling
1074 return getattr(self, name, None) or getattr(__builtin__, name)
1076def _import_module(import_name, safe=False):
1077 try:
1078 if import_name.startswith('__runtime__.'):
1079 return sys.modules[import_name]
1080 elif '.' in import_name:
1081 items = import_name.split('.')
1082 module = '.'.join(items[:-1])
1083 obj = items[-1]
1084 submodule = getattr(__import__(module, None, None, [obj]), obj)
1085 if isinstance(submodule, (ModuleType, type)):
1086 return submodule
1087 return __import__(import_name, None, None, [obj])
1088 else:
1089 return __import__(import_name)
1090 except (ImportError, AttributeError, KeyError):
1091 if safe:
1092 return None
1093 raise
1095# https://github.com/python/cpython/blob/a8912a0f8d9eba6d502c37d522221f9933e976db/Lib/pickle.py#L322-L333
1096def _getattribute(obj, name):
1097 for subpath in name.split('.'):
1098 if subpath == '<locals>':
1099 raise AttributeError("Can't get local attribute {!r} on {!r}"
1100 .format(name, obj))
1101 try:
1102 parent = obj
1103 obj = getattr(obj, subpath)
1104 except AttributeError:
1105 raise AttributeError("Can't get attribute {!r} on {!r}"
1106 .format(name, obj))
1107 return obj, parent
1109def _locate_function(obj, pickler=None):
1110 module_name = getattr(obj, '__module__', None)
1111 if module_name in ['__main__', None] or \
1112 pickler and is_dill(pickler, child=False) and pickler._session and module_name == pickler._main.__name__:
1113 return False
1114 if hasattr(obj, '__qualname__'):
1115 module = _import_module(module_name, safe=True)
1116 try:
1117 found, _ = _getattribute(module, obj.__qualname__)
1118 return found is obj
1119 except AttributeError:
1120 return False
1121 else:
1122 found = _import_module(module_name + '.' + obj.__name__, safe=True)
1123 return found is obj
1126def _setitems(dest, source):
1127 for k, v in source.items():
1128 dest[k] = v
1131def _save_with_postproc(pickler, reduction, is_pickler_dill=None, obj=Getattr.NO_DEFAULT, postproc_list=None):
1132 if obj is Getattr.NO_DEFAULT:
1133 obj = Reduce(reduction) # pragma: no cover
1135 if is_pickler_dill is None:
1136 is_pickler_dill = is_dill(pickler, child=True)
1137 if is_pickler_dill:
1138 # assert id(obj) not in pickler._postproc, str(obj) + ' already pushed on stack!'
1139 # if not hasattr(pickler, 'x'): pickler.x = 0
1140 # print(pickler.x*' ', 'push', obj, id(obj), pickler._recurse)
1141 # pickler.x += 1
1142 if postproc_list is None:
1143 postproc_list = []
1145 # Recursive object not supported. Default to a global instead.
1146 if id(obj) in pickler._postproc:
1147 name = '%s.%s ' % (obj.__module__, getattr(obj, '__qualname__', obj.__name__)) if hasattr(obj, '__module__') else ''
1148 warnings.warn('Cannot pickle %r: %shas recursive self-references that trigger a RecursionError.' % (obj, name), PicklingWarning)
1149 pickler.save_global(obj)
1150 return
1151 pickler._postproc[id(obj)] = postproc_list
1153 # TODO: Use state_setter in Python 3.8 to allow for faster cPickle implementations
1154 pickler.save_reduce(*reduction, obj=obj)
1156 if is_pickler_dill:
1157 # pickler.x -= 1
1158 # print(pickler.x*' ', 'pop', obj, id(obj))
1159 postproc = pickler._postproc.pop(id(obj))
1160 # assert postproc_list == postproc, 'Stack tampered!'
1161 for reduction in reversed(postproc):
1162 if reduction[0] is _setitems:
1163 # use the internal machinery of pickle.py to speedup when
1164 # updating a dictionary in postproc
1165 dest, source = reduction[1]
1166 if source:
1167 pickler.write(pickler.get(pickler.memo[id(dest)][0]))
1168 if sys.hexversion < 0x30e00a1:
1169 pickler._batch_setitems(iter(source.items()))
1170 else:
1171 pickler._batch_setitems(iter(source.items()), obj=obj)
1172 else:
1173 # Updating with an empty dictionary. Same as doing nothing.
1174 continue
1175 else:
1176 pickler.save_reduce(*reduction)
1177 # pop None created by calling preprocessing step off stack
1178 pickler.write(POP)
1180@register(float)
1181def save_float(pickler, obj):
1182 numpy = sys.modules.get('numpy')
1183 special = False
1184 if numpy is not None:
1185 if obj is numpy.nan or obj is numpy.inf: special = True
1186 if special is False: #XXX: no logger
1187 StockPickler.save_float(pickler, obj)
1188 elif obj is numpy.nan:
1189 logger.trace(pickler, "Nan: %s", obj)
1190 pickler.save_reduce(getattr, (numpy, 'nan'), obj=obj)
1191 logger.trace(pickler, "# Nan")
1192 elif obj is numpy.inf:
1193 logger.trace(pickler, "Inf: %s", obj)
1194 pickler.save_reduce(getattr, (numpy, 'inf'), obj=obj)
1195 logger.trace(pickler, "# Inf")
1196 return
1198#@register(CodeType)
1199#def save_code(pickler, obj):
1200# logger.trace(pickler, "Co: %s", obj)
1201# pickler.save_reduce(_unmarshal, (marshal.dumps(obj),), obj=obj)
1202# logger.trace(pickler, "# Co")
1203# return
1205# The following function is based on 'save_codeobject' from 'cloudpickle'
1206# Copyright (c) 2012, Regents of the University of California.
1207# Copyright (c) 2009 `PiCloud, Inc. <http://www.picloud.com>`_.
1208# License: https://github.com/cloudpipe/cloudpickle/blob/master/LICENSE
1209@register(CodeType)
1210def save_code(pickler, obj):
1211 logger.trace(pickler, "Co: %s", obj)
1212 if hasattr(obj, "co_endlinetable"): # python 3.11a (20 args)
1213 args = (
1214 obj.co_lnotab, # for < python 3.10 [not counted in args]
1215 obj.co_argcount, obj.co_posonlyargcount,
1216 obj.co_kwonlyargcount, obj.co_nlocals, obj.co_stacksize,
1217 obj.co_flags, obj.co_code, obj.co_consts, obj.co_names,
1218 obj.co_varnames, obj.co_filename, obj.co_name, obj.co_qualname,
1219 obj.co_firstlineno, obj.co_linetable, obj.co_endlinetable,
1220 obj.co_columntable, obj.co_exceptiontable, obj.co_freevars,
1221 obj.co_cellvars
1222 )
1223 elif hasattr(obj, "co_exceptiontable"): # python 3.11 (18 args)
1224 with warnings.catch_warnings():
1225 if not OLD312a7: # issue 597
1226 warnings.filterwarnings('ignore', category=DeprecationWarning)
1227 args = ( # [not counted in args]
1228 getattr(obj, 'co_lnotab', b''), # for < python 3.10; 3.15.0b1+
1229 obj.co_argcount, obj.co_posonlyargcount,
1230 obj.co_kwonlyargcount, obj.co_nlocals, obj.co_stacksize,
1231 obj.co_flags, obj.co_code, obj.co_consts, obj.co_names,
1232 obj.co_varnames, obj.co_filename, obj.co_name, obj.co_qualname,
1233 obj.co_firstlineno, obj.co_linetable, obj.co_exceptiontable,
1234 obj.co_freevars, obj.co_cellvars
1235 )
1236 elif hasattr(obj, "co_qualname"): # pypy 3.11 7.3.19+ (17 args)
1237 args = (
1238 obj.co_lnotab, obj.co_argcount, obj.co_posonlyargcount,
1239 obj.co_kwonlyargcount, obj.co_nlocals, obj.co_stacksize,
1240 obj.co_flags, obj.co_code, obj.co_consts, obj.co_names,
1241 obj.co_varnames, obj.co_filename, obj.co_name, obj.co_qualname,
1242 obj.co_firstlineno, obj.co_linetable, obj.co_freevars,
1243 obj.co_cellvars
1244 )
1245 elif hasattr(obj, "co_linetable"): # python 3.10 (16 args)
1246 args = (
1247 obj.co_lnotab, # for < python 3.10 [not counted in args]
1248 obj.co_argcount, obj.co_posonlyargcount,
1249 obj.co_kwonlyargcount, obj.co_nlocals, obj.co_stacksize,
1250 obj.co_flags, obj.co_code, obj.co_consts, obj.co_names,
1251 obj.co_varnames, obj.co_filename, obj.co_name,
1252 obj.co_firstlineno, obj.co_linetable, obj.co_freevars,
1253 obj.co_cellvars
1254 )
1255 elif hasattr(obj, "co_posonlyargcount"): # python 3.8 (16 args)
1256 args = (
1257 obj.co_argcount, obj.co_posonlyargcount,
1258 obj.co_kwonlyargcount, obj.co_nlocals, obj.co_stacksize,
1259 obj.co_flags, obj.co_code, obj.co_consts, obj.co_names,
1260 obj.co_varnames, obj.co_filename, obj.co_name,
1261 obj.co_firstlineno, obj.co_lnotab, obj.co_freevars,
1262 obj.co_cellvars
1263 )
1264 else: # python 3.7 (15 args)
1265 args = (
1266 obj.co_argcount, obj.co_kwonlyargcount, obj.co_nlocals,
1267 obj.co_stacksize, obj.co_flags, obj.co_code, obj.co_consts,
1268 obj.co_names, obj.co_varnames, obj.co_filename,
1269 obj.co_name, obj.co_firstlineno, obj.co_lnotab,
1270 obj.co_freevars, obj.co_cellvars
1271 )
1273 pickler.save_reduce(_create_code, args, obj=obj)
1274 logger.trace(pickler, "# Co")
1275 return
1277def _repr_dict(obj):
1278 """Make a short string representation of a dictionary."""
1279 return "<%s object at %#012x>" % (type(obj).__name__, id(obj))
1281@register(dict)
1282def save_module_dict(pickler, obj):
1283 if is_dill(pickler, child=False) and obj == pickler._main.__dict__ and \
1284 not (pickler._session and pickler._first_pass):
1285 logger.trace(pickler, "D1: %s", _repr_dict(obj)) # obj
1286 pickler.write(bytes('c__builtin__\n__main__\n', 'UTF-8'))
1287 logger.trace(pickler, "# D1")
1288 elif (not is_dill(pickler, child=False)) and (obj == _main_module.__dict__):
1289 logger.trace(pickler, "D3: %s", _repr_dict(obj)) # obj
1290 pickler.write(bytes('c__main__\n__dict__\n', 'UTF-8')) #XXX: works in general?
1291 logger.trace(pickler, "# D3")
1292 elif '__name__' in obj and obj != _main_module.__dict__ \
1293 and type(obj['__name__']) is str and obj['__name__'] \
1294 and obj is getattr(_import_module(obj['__name__'],True), '__dict__', None):
1295 logger.trace(pickler, "D4: %s", _repr_dict(obj)) # obj
1296 pickler.write(bytes('c%s\n__dict__\n' % obj['__name__'], 'UTF-8'))
1297 logger.trace(pickler, "# D4")
1298 else:
1299 logger.trace(pickler, "D2: %s", _repr_dict(obj)) # obj
1300 if is_dill(pickler, child=False) and pickler._session:
1301 # we only care about session the first pass thru
1302 pickler._first_pass = False
1303 StockPickler.save_dict(pickler, obj)
1304 logger.trace(pickler, "# D2")
1305 return
1308if not OLD310 and MAPPING_PROXY_TRICK:
1309 def save_dict_view(dicttype):
1310 def save_dict_view_for_function(func):
1311 def _save_dict_view(pickler, obj):
1312 logger.trace(pickler, "Dkvi: <%s>", obj)
1313 mapping = obj.mapping | _dictproxy_helper_instance
1314 pickler.save_reduce(func, (mapping,), obj=obj)
1315 logger.trace(pickler, "# Dkvi")
1316 return _save_dict_view
1317 return [
1318 (funcname, save_dict_view_for_function(getattr(dicttype, funcname)))
1319 for funcname in ('keys', 'values', 'items')
1320 ]
1321else:
1322 # The following functions are based on 'cloudpickle'
1323 # https://github.com/cloudpipe/cloudpickle/blob/5d89947288a18029672596a4d719093cc6d5a412/cloudpickle/cloudpickle.py#L922-L940
1324 # Copyright (c) 2012, Regents of the University of California.
1325 # Copyright (c) 2009 `PiCloud, Inc. <http://www.picloud.com>`_.
1326 # License: https://github.com/cloudpipe/cloudpickle/blob/master/LICENSE
1327 def save_dict_view(dicttype):
1328 def save_dict_keys(pickler, obj):
1329 logger.trace(pickler, "Dk: <%s>", obj)
1330 dict_constructor = _shims.Reduce(dicttype.fromkeys, (list(obj),))
1331 pickler.save_reduce(dicttype.keys, (dict_constructor,), obj=obj)
1332 logger.trace(pickler, "# Dk")
1334 def save_dict_values(pickler, obj):
1335 logger.trace(pickler, "Dv: <%s>", obj)
1336 dict_constructor = _shims.Reduce(dicttype, (enumerate(obj),))
1337 pickler.save_reduce(dicttype.values, (dict_constructor,), obj=obj)
1338 logger.trace(pickler, "# Dv")
1340 def save_dict_items(pickler, obj):
1341 logger.trace(pickler, "Di: <%s>", obj)
1342 pickler.save_reduce(dicttype.items, (dicttype(obj),), obj=obj)
1343 logger.trace(pickler, "# Di")
1345 return (
1346 ('keys', save_dict_keys),
1347 ('values', save_dict_values),
1348 ('items', save_dict_items)
1349 )
1351for __dicttype in (
1352 dict,
1353 OrderedDict
1354):
1355 # Pure Python views already pickle their state without a custom reducer.
1356 if __dicttype is OrderedDict and OrderedDict is not getattr(_collections, 'OrderedDict', None):
1357 continue
1358 __obj = __dicttype()
1359 for __funcname, __savefunc in save_dict_view(__dicttype):
1360 __tview = type(getattr(__obj, __funcname)())
1361 if __tview not in Pickler.dispatch:
1362 Pickler.dispatch[__tview] = __savefunc
1363del __dicttype, __obj, __funcname, __tview, __savefunc
1366@register(ClassType)
1367def save_classobj(pickler, obj): #FIXME: enable pickler._byref
1368 if not _locate_function(obj, pickler):
1369 logger.trace(pickler, "C1: %s", obj)
1370 pickler.save_reduce(ClassType, (obj.__name__, obj.__bases__,
1371 obj.__dict__), obj=obj)
1372 #XXX: or obj.__dict__.copy()), obj=obj) ?
1373 logger.trace(pickler, "# C1")
1374 else:
1375 logger.trace(pickler, "C2: %s", obj)
1376 name = getattr(obj, '__qualname__', getattr(obj, '__name__', None))
1377 StockPickler.save_global(pickler, obj, name=name)
1378 logger.trace(pickler, "# C2")
1379 return
1381@register(typing._GenericAlias)
1382def save_generic_alias(pickler, obj):
1383 args = obj.__args__
1384 if type(obj.__reduce__()) is str:
1385 logger.trace(pickler, "Ga0: %s", obj)
1386 StockPickler.save_global(pickler, obj, name=obj.__reduce__())
1387 logger.trace(pickler, "# Ga0")
1388 elif obj.__origin__ is tuple and (not args or args == ((),)):
1389 logger.trace(pickler, "Ga1: %s", obj)
1390 pickler.save_reduce(_create_typing_tuple, (args,), obj=obj)
1391 logger.trace(pickler, "# Ga1")
1392 else:
1393 logger.trace(pickler, "Ga2: %s", obj)
1394 StockPickler.save_reduce(pickler, *obj.__reduce__(), obj=obj)
1395 logger.trace(pickler, "# Ga2")
1396 return
1398def _create_typevar(args, kwargs, typevar=typing.TypeVar):
1399 return typevar(*args, **kwargs)
1401def _locate_typevar(obj, pickler=None):
1402 # TypeVars (and ParamSpecs, TypeVarTuples) have no __qualname__,
1403 # so look them up by name in their module
1404 module_name = getattr(obj, '__module__', None)
1405 if module_name in ['__main__', None] or \
1406 pickler and is_dill(pickler, child=False) and pickler._session and module_name == pickler._main.__name__:
1407 return False
1408 module = _import_module(module_name, safe=True)
1409 return getattr(module, obj.__name__, None) is obj
1411def _get_typevar_kwargs(obj):
1412 # only pass the options that are set, as some (e.g. infer_variance and
1413 # default) are not accepted by older versions of python
1414 kwargs = {}
1415 bound = getattr(obj, '__bound__', None)
1416 if bound is not None:
1417 kwargs['bound'] = bound
1418 for attr in ('covariant', 'contravariant', 'infer_variance'):
1419 if getattr(obj, '__%s__' % attr, False):
1420 kwargs[attr] = True
1421 if getattr(obj, 'has_default', lambda: False)(): # python >= 3.13
1422 kwargs['default'] = obj.__default__
1423 return kwargs
1425@register(typing.ParamSpec)
1426@register(typing.TypeVar)
1427def save_typevar(pickler, obj):
1428 if _locate_typevar(obj, pickler):
1429 logger.trace(pickler, "Tv2: %s", obj)
1430 StockPickler.save_global(pickler, obj, name=obj.__name__)
1431 logger.trace(pickler, "# Tv2")
1432 return
1433 logger.trace(pickler, "Tv1: %s", obj)
1434 args = (obj.__name__,) + getattr(obj, '__constraints__', ())
1435 kwargs = _get_typevar_kwargs(obj)
1436 pickler.save_reduce(_create_typevar, (args, kwargs, type(obj)), obj=obj)
1437 logger.trace(pickler, "# Tv1")
1439if hasattr(typing, 'TypeVarTuple'): # python >= 3.11
1440 register(typing.TypeVarTuple)(save_typevar)
1442if ThreadHandleType:
1443 @register(ThreadHandleType)
1444 def save_thread_handle(pickler, obj):
1445 logger.trace(pickler, "Th: %s", obj)
1446 pickler.save_reduce(_create_thread_handle, (obj.ident, obj.is_done()), obj=obj)
1447 logger.trace(pickler, "# Th")
1448 return
1450@register(LockType) #XXX: copied Thread will have new Event (due to new Lock)
1451def save_lock(pickler, obj):
1452 logger.trace(pickler, "Lo: %s", obj)
1453 pickler.save_reduce(_create_lock, (obj.locked(),), obj=obj)
1454 logger.trace(pickler, "# Lo")
1455 return
1457@register(RLockType)
1458def save_rlock(pickler, obj):
1459 logger.trace(pickler, "RL: %s", obj)
1460 r = obj.__repr__() # don't use _release_save as it unlocks the lock
1461 count = int(r.split('count=')[1].split()[0].rstrip('>'))
1462 owner = int(r.split('owner=')[1].split()[0])
1463 pickler.save_reduce(_create_rlock, (count,owner,), obj=obj)
1464 logger.trace(pickler, "# RL")
1465 return
1467#@register(SocketType) #FIXME: causes multiprocess test_pickling FAIL
1468def save_socket(pickler, obj):
1469 logger.trace(pickler, "So: %s", obj)
1470 pickler.save_reduce(*reduce_socket(obj))
1471 logger.trace(pickler, "# So")
1472 return
1474def _save_file(pickler, obj, open_):
1475 if obj.closed:
1476 position = 0
1477 else:
1478 obj.flush()
1479 if obj in (sys.__stdout__, sys.__stderr__, sys.__stdin__):
1480 position = -1
1481 else:
1482 position = obj.tell()
1483 if is_dill(pickler, child=True) and pickler._fmode == FILE_FMODE:
1484 with open_(obj.name, "rb" if "b" in obj.mode else "r") as f:
1485 fdata = f.read()
1486 else:
1487 fdata = ""
1488 if is_dill(pickler, child=True):
1489 strictio = pickler._strictio
1490 fmode = pickler._fmode
1491 else:
1492 strictio = False
1493 fmode = 0 # HANDLE_FMODE
1494 pickler.save_reduce(_create_filehandle, (obj.name, obj.mode, position,
1495 obj.closed, open_, strictio,
1496 fmode, fdata), obj=obj)
1497 return
1500@register(FileType) #XXX: in 3.x has buffer=0, needs different _create?
1501@register(BufferedReaderType)
1502@register(BufferedWriterType)
1503@register(TextWrapperType)
1504def save_file(pickler, obj):
1505 logger.trace(pickler, "Fi: %s", obj)
1506 f = _save_file(pickler, obj, open)
1507 logger.trace(pickler, "# Fi")
1508 return f
1510if BufferedRandomType:
1511 @register(BufferedRandomType)
1512 def save_file(pickler, obj):
1513 logger.trace(pickler, "Fi: %s", obj)
1514 f = _save_file(pickler, obj, open)
1515 logger.trace(pickler, "# Fi")
1516 return f
1518if PyTextWrapperType:
1519 @register(PyBufferedReaderType)
1520 @register(PyBufferedWriterType)
1521 @register(PyTextWrapperType)
1522 def save_file(pickler, obj):
1523 logger.trace(pickler, "Fi: %s", obj)
1524 f = _save_file(pickler, obj, _open)
1525 logger.trace(pickler, "# Fi")
1526 return f
1528 if PyBufferedRandomType:
1529 @register(PyBufferedRandomType)
1530 def save_file(pickler, obj):
1531 logger.trace(pickler, "Fi: %s", obj)
1532 f = _save_file(pickler, obj, _open)
1533 logger.trace(pickler, "# Fi")
1534 return f
1537# The following two functions are based on 'saveCStringIoInput'
1538# and 'saveCStringIoOutput' from spickle
1539# Copyright (c) 2011 by science+computing ag
1540# License: http://www.apache.org/licenses/LICENSE-2.0
1541if InputType:
1542 @register(InputType)
1543 def save_stringi(pickler, obj):
1544 logger.trace(pickler, "Io: %s", obj)
1545 if obj.closed:
1546 value = ''; position = 0
1547 else:
1548 value = obj.getvalue(); position = obj.tell()
1549 pickler.save_reduce(_create_stringi, (value, position, \
1550 obj.closed), obj=obj)
1551 logger.trace(pickler, "# Io")
1552 return
1554 @register(OutputType)
1555 def save_stringo(pickler, obj):
1556 logger.trace(pickler, "Io: %s", obj)
1557 if obj.closed:
1558 value = ''; position = 0
1559 else:
1560 value = obj.getvalue(); position = obj.tell()
1561 pickler.save_reduce(_create_stringo, (value, position, \
1562 obj.closed), obj=obj)
1563 logger.trace(pickler, "# Io")
1564 return
1566if LRUCacheType is not None:
1567 from functools import lru_cache
1568 @register(LRUCacheType)
1569 def save_lru_cache(pickler, obj):
1570 logger.trace(pickler, "LRU: %s", obj)
1571 if OLD39:
1572 kwargs = obj.cache_info()
1573 args = (kwargs.maxsize,)
1574 else:
1575 kwargs = obj.cache_parameters()
1576 args = (kwargs['maxsize'], kwargs['typed'])
1577 if args != lru_cache.__defaults__:
1578 wrapper = Reduce(lru_cache, args, is_callable=True)
1579 else:
1580 wrapper = lru_cache
1581 pickler.save_reduce(wrapper, (obj.__wrapped__,), state=obj.__dict__, obj=obj)
1582 logger.trace(pickler, "# LRU")
1583 return
1585@register(SuperType)
1586def save_super(pickler, obj):
1587 logger.trace(pickler, "Su: %s", obj)
1588 pickler.save_reduce(super, (obj.__thisclass__, obj.__self__), obj=obj)
1589 logger.trace(pickler, "# Su")
1590 return
1592if IS_PYPY:
1593 @register(MethodType)
1594 def save_instancemethod0(pickler, obj):
1595 code = getattr(obj.__func__, '__code__', None)
1596 if code is not None and type(code) is not CodeType \
1597 and getattr(obj.__self__, obj.__name__) == obj:
1598 # Some PyPy builtin functions have no module name
1599 logger.trace(pickler, "Me2: %s", obj)
1600 # TODO: verify that this works for all PyPy builtin methods
1601 pickler.save_reduce(getattr, (obj.__self__, obj.__name__), obj=obj)
1602 logger.trace(pickler, "# Me2")
1603 return
1605 logger.trace(pickler, "Me1: %s", obj)
1606 pickler.save_reduce(MethodType, (obj.__func__, obj.__self__), obj=obj)
1607 logger.trace(pickler, "# Me1")
1608 return
1609else:
1610 @register(MethodType)
1611 def save_instancemethod0(pickler, obj):
1612 logger.trace(pickler, "Me1: %s", obj)
1613 pickler.save_reduce(MethodType, (obj.__func__, obj.__self__), obj=obj)
1614 logger.trace(pickler, "# Me1")
1615 return
1617if not IS_PYPY:
1618 @register(MemberDescriptorType)
1619 @register(GetSetDescriptorType)
1620 @register(MethodDescriptorType)
1621 @register(WrapperDescriptorType)
1622 @register(ClassMethodDescriptorType)
1623 def save_wrapper_descriptor(pickler, obj):
1624 logger.trace(pickler, "Wr: %s", obj)
1625 pickler.save_reduce(_getattr, (obj.__objclass__, obj.__name__,
1626 obj.__repr__()), obj=obj)
1627 logger.trace(pickler, "# Wr")
1628 return
1629else:
1630 @register(MemberDescriptorType)
1631 @register(GetSetDescriptorType)
1632 def save_wrapper_descriptor(pickler, obj):
1633 logger.trace(pickler, "Wr: %s", obj)
1634 pickler.save_reduce(_getattr, (obj.__objclass__, obj.__name__,
1635 obj.__repr__()), obj=obj)
1636 logger.trace(pickler, "# Wr")
1637 return
1639@register(CellType)
1640def save_cell(pickler, obj):
1641 try:
1642 f = obj.cell_contents
1643 except ValueError: # cell is empty
1644 logger.trace(pickler, "Ce3: %s", obj)
1645 # _shims._CELL_EMPTY is defined in _shims.py to support PyPy 2.7.
1646 # It unpickles to a sentinel object _dill._CELL_EMPTY, also created in
1647 # _shims.py. This object is not present in Python 3 because the cell's
1648 # contents can be deleted in newer versions of Python. The reduce object
1649 # will instead unpickle to None if unpickled in Python 3.
1651 # When breaking changes are made to dill, (_shims._CELL_EMPTY,) can
1652 # be replaced by () OR the delattr function can be removed repending on
1653 # whichever is more convenient.
1654 pickler.save_reduce(_create_cell, (_shims._CELL_EMPTY,), obj=obj)
1655 # Call the function _delattr on the cell's cell_contents attribute
1656 # The result of this function call will be None
1657 pickler.save_reduce(_shims._delattr, (obj, 'cell_contents'))
1658 # pop None created by calling _delattr off stack
1659 pickler.write(POP)
1660 logger.trace(pickler, "# Ce3")
1661 return
1662 if is_dill(pickler, child=True):
1663 if id(f) in pickler._postproc:
1664 # Already seen. Add to its postprocessing.
1665 postproc = pickler._postproc[id(f)]
1666 else:
1667 # Haven't seen it. Add to the highest possible object and set its
1668 # value as late as possible to prevent cycle.
1669 postproc = next(iter(pickler._postproc.values()), None)
1670 if postproc is not None:
1671 logger.trace(pickler, "Ce2: %s", obj)
1672 # _CELL_REF is defined in _shims.py to support older versions of
1673 # dill. When breaking changes are made to dill, (_CELL_REF,) can
1674 # be replaced by ()
1675 pickler.save_reduce(_create_cell, (_CELL_REF,), obj=obj)
1676 postproc.append((_shims._setattr, (obj, 'cell_contents', f)))
1677 logger.trace(pickler, "# Ce2")
1678 return
1679 logger.trace(pickler, "Ce1: %s", obj)
1680 pickler.save_reduce(_create_cell, (f,), obj=obj)
1681 logger.trace(pickler, "# Ce1")
1682 return
1684if MAPPING_PROXY_TRICK:
1685 @register(DictProxyType)
1686 def save_dictproxy(pickler, obj):
1687 logger.trace(pickler, "Mp: %s", _repr_dict(obj)) # obj
1688 mapping = obj | _dictproxy_helper_instance
1689 pickler.save_reduce(DictProxyType, (mapping,), obj=obj)
1690 logger.trace(pickler, "# Mp")
1691 return
1692else:
1693 @register(DictProxyType)
1694 def save_dictproxy(pickler, obj):
1695 logger.trace(pickler, "Mp: %s", _repr_dict(obj)) # obj
1696 pickler.save_reduce(DictProxyType, (obj.copy(),), obj=obj)
1697 logger.trace(pickler, "# Mp")
1698 return
1700@register(SliceType)
1701def save_slice(pickler, obj):
1702 logger.trace(pickler, "Sl: %s", obj)
1703 pickler.save_reduce(slice, (obj.start, obj.stop, obj.step), obj=obj)
1704 logger.trace(pickler, "# Sl")
1705 return
1707@register(XRangeType)
1708@register(EllipsisType)
1709@register(NotImplementedType)
1710def save_singleton(pickler, obj):
1711 logger.trace(pickler, "Si: %s", obj)
1712 pickler.save_reduce(_eval_repr, (obj.__repr__(),), obj=obj)
1713 logger.trace(pickler, "# Si")
1714 return
1716def _proxy_helper(obj): # a dead proxy returns a reference to None
1717 """get memory address of proxy's reference object"""
1718 _repr = repr(obj)
1719 try: _str = str(obj)
1720 except ReferenceError: # it's a dead proxy
1721 return id(None)
1722 if _str == _repr: return id(obj) # it's a repr
1723 try: # either way, it's a proxy from here
1724 address = int(_str.rstrip('>').split(' at ')[-1], base=16)
1725 except ValueError: # special case: proxy of a 'type'
1726 if not IS_PYPY:
1727 address = int(_repr.rstrip('>').split(' at ')[-1], base=16)
1728 else:
1729 objects = iter(gc.get_objects())
1730 for _obj in objects:
1731 if repr(_obj) == _str: return id(_obj)
1732 # all bad below... nothing found so throw ReferenceError
1733 msg = "Cannot reference object for proxy at '%s'" % id(obj)
1734 raise ReferenceError(msg)
1735 return address
1737def _locate_object(address, module=None):
1738 """get object located at the given memory address (inverse of id(obj))"""
1739 special = [None, True, False] #XXX: more...?
1740 for obj in special:
1741 if address == id(obj): return obj
1742 if module:
1743 objects = iter(module.__dict__.values())
1744 else: objects = iter(gc.get_objects())
1745 for obj in objects:
1746 if address == id(obj): return obj
1747 # all bad below... nothing found so throw ReferenceError or TypeError
1748 try: address = hex(address)
1749 except TypeError:
1750 raise TypeError("'%s' is not a valid memory address" % str(address))
1751 raise ReferenceError("Cannot reference object at '%s'" % address)
1753@register(ReferenceType)
1754def save_weakref(pickler, obj):
1755 refobj = obj()
1756 logger.trace(pickler, "R1: %s", obj)
1757 #refobj = ctypes.pythonapi.PyWeakref_GetObject(obj) # dead returns "None"
1758 pickler.save_reduce(_create_weakref, (refobj, obj.__callback__), obj=obj)
1759 logger.trace(pickler, "# R1")
1760 return
1762@register(KeyedRef)
1763def save_keyedref(pickler, obj):
1764 logger.trace(pickler, "R3: %s", obj)
1765 pickler.save_reduce(_create_keyedref, (obj(), obj.__callback__, None),
1766 state=(None, {'key': obj.key}), obj=obj)
1767 logger.trace(pickler, "# R3")
1768 return
1770@register(ProxyType)
1771@register(CallableProxyType)
1772def save_weakproxy(pickler, obj):
1773 # Must do string substitution here and use %r to avoid ReferenceError.
1774 logger.trace(pickler, "R2: %r" % obj)
1775 refobj = _locate_object(_proxy_helper(obj))
1776 pickler.save_reduce(_create_weakproxy, (refobj, callable(obj)), obj=obj)
1777 logger.trace(pickler, "# R2")
1778 return
1780def _is_builtin_module(module):
1781 if not hasattr(module, "__file__"): return True
1782 if module.__file__ is None: return False
1783 # If a module file name starts with prefix, it should be a builtin
1784 # module, so should always be pickled as a reference.
1785 names = ["base_prefix", "base_exec_prefix", "exec_prefix", "prefix", "real_prefix"]
1786 rp = os.path.realpath
1787 # See https://github.com/uqfoundation/dill/issues/566
1788 return (
1789 any(
1790 module.__file__.startswith(getattr(sys, name))
1791 or rp(module.__file__).startswith(rp(getattr(sys, name)))
1792 for name in names
1793 if hasattr(sys, name)
1794 )
1795 or module.__file__.endswith(EXTENSION_SUFFIXES)
1796 or 'site-packages' in module.__file__
1797 )
1799def _is_imported_module(module):
1800 return getattr(module, '__loader__', None) is not None or module in sys.modules.values()
1802@register(ModuleType)
1803def save_module(pickler, obj):
1804 if False: #_use_diff:
1805 if obj.__name__.split('.', 1)[0] != "dill":
1806 try:
1807 changed = diff.whats_changed(obj, seen=pickler._diff_cache)[0]
1808 except RuntimeError: # not memorised module, probably part of dill
1809 pass
1810 else:
1811 logger.trace(pickler, "M2: %s with diff", obj)
1812 logger.info("Diff: %s", changed.keys())
1813 pickler.save_reduce(_import_module, (obj.__name__,), obj=obj,
1814 state=changed)
1815 logger.trace(pickler, "# M2")
1816 return
1818 logger.trace(pickler, "M1: %s", obj)
1819 pickler.save_reduce(_import_module, (obj.__name__,), obj=obj)
1820 logger.trace(pickler, "# M1")
1821 else:
1822 builtin_mod = _is_builtin_module(obj)
1823 is_session_main = is_dill(pickler, child=True) and obj is pickler._main
1824 if (obj.__name__ not in ("builtins", "dill", "dill._dill") and not builtin_mod
1825 or is_session_main):
1826 logger.trace(pickler, "M1: %s", obj)
1827 # Hack for handling module-type objects in load_module().
1828 mod_name = obj.__name__ if _is_imported_module(obj) else '__runtime__.%s' % obj.__name__
1829 # Second references are saved as __builtin__.__main__ in save_module_dict().
1830 main_dict = obj.__dict__.copy()
1831 for item in ('__builtins__', '__loader__'):
1832 main_dict.pop(item, None)
1833 for item in IPYTHON_SINGLETONS: #pragma: no cover
1834 if getattr(main_dict.get(item), '__module__', '').startswith('IPython'):
1835 del main_dict[item]
1836 pickler.save_reduce(_import_module, (mod_name,), obj=obj, state=main_dict)
1837 logger.trace(pickler, "# M1")
1838 elif obj.__name__ == "dill._dill":
1839 logger.trace(pickler, "M2: %s", obj)
1840 pickler.save_global(obj, name="_dill")
1841 logger.trace(pickler, "# M2")
1842 else:
1843 logger.trace(pickler, "M2: %s", obj)
1844 pickler.save_reduce(_import_module, (obj.__name__,), obj=obj)
1845 logger.trace(pickler, "# M2")
1846 return
1848# The following function is based on '_extract_class_dict' from 'cloudpickle'
1849# Copyright (c) 2012, Regents of the University of California.
1850# Copyright (c) 2009 `PiCloud, Inc. <http://www.picloud.com>`_.
1851# License: https://github.com/cloudpipe/cloudpickle/blob/master/LICENSE
1852def _get_typedict_type(cls, clsdict, attrs, postproc_list):
1853 """Retrieve a copy of the dict of a class without the inherited methods"""
1854 if len(cls.__bases__) == 1:
1855 inherited_dict = cls.__bases__[0].__dict__
1856 else:
1857 inherited_dict = {}
1858 for base in reversed(cls.__bases__):
1859 inherited_dict.update(base.__dict__)
1860 to_remove = []
1861 for name, value in dict.items(clsdict):
1862 try:
1863 base_value = inherited_dict[name]
1864 if value is base_value and hasattr(value, '__qualname__'):
1865 to_remove.append(name)
1866 except KeyError:
1867 pass
1868 for name in to_remove:
1869 dict.pop(clsdict, name)
1871 if issubclass(type(cls), type):
1872 clsdict.pop('__dict__', None)
1873 clsdict.pop('__weakref__', None)
1874 # clsdict.pop('__prepare__', None)
1875 return clsdict, attrs
1877def _get_typedict_abc(obj, _dict, attrs, postproc_list):
1878 if hasattr(abc, '_get_dump'):
1879 (registry, _, _, _) = abc._get_dump(obj)
1880 register = obj.register
1881 postproc_list.extend((register, (reg(),)) for reg in registry)
1882 elif hasattr(obj, '_abc_registry'):
1883 registry = obj._abc_registry
1884 register = obj.register
1885 postproc_list.extend((register, (reg,)) for reg in registry)
1886 else:
1887 raise PicklingError("Cannot find registry of ABC %s", obj)
1889 if '_abc_registry' in _dict:
1890 _dict.pop('_abc_registry', None)
1891 _dict.pop('_abc_cache', None)
1892 _dict.pop('_abc_negative_cache', None)
1893 # _dict.pop('_abc_negative_cache_version', None)
1894 else:
1895 _dict.pop('_abc_impl', None)
1896 return _dict, attrs
1898@register(TypeType)
1899def save_type(pickler, obj, postproc_list=None):
1900 if obj in _typemap:
1901 logger.trace(pickler, "T1: %s", obj)
1902 # if obj in _incedental_types:
1903 # warnings.warn('Type %r may only exist on this implementation of Python and cannot be unpickled in other implementations.' % (obj,), PicklingWarning)
1904 pickler.save_reduce(_load_type, (_typemap[obj],), obj=obj)
1905 logger.trace(pickler, "# T1")
1906 elif obj.__bases__ == (tuple,) and all([hasattr(obj, attr) for attr in ('_fields','_asdict','_make','_replace')]):
1907 # special case: namedtuples
1908 logger.trace(pickler, "T6: %s", obj)
1910 obj_name = getattr(obj, '__qualname__', getattr(obj, '__name__', None))
1911 if obj.__name__ != obj_name:
1912 if postproc_list is None:
1913 postproc_list = []
1914 postproc_list.append((setattr, (obj, '__qualname__', obj_name)))
1916 if not obj._field_defaults:
1917 _save_with_postproc(pickler, (_create_namedtuple, (obj.__name__, obj._fields, obj.__module__)), obj=obj, postproc_list=postproc_list)
1918 else:
1919 defaults = [obj._field_defaults[field] for field in obj._fields if field in obj._field_defaults]
1920 _save_with_postproc(pickler, (_create_namedtuple, (obj.__name__, obj._fields, obj.__module__, defaults)), obj=obj, postproc_list=postproc_list)
1921 logger.trace(pickler, "# T6")
1922 return
1924 # special caes: NoneType, NotImplementedType, EllipsisType, EnumMeta, etc
1925 elif obj is type(None):
1926 logger.trace(pickler, "T7: %s", obj)
1927 #XXX: pickler.save_reduce(type, (None,), obj=obj)
1928 pickler.write(GLOBAL + b'__builtin__\nNoneType\n')
1929 logger.trace(pickler, "# T7")
1930 elif obj is NotImplementedType:
1931 logger.trace(pickler, "T7: %s", obj)
1932 pickler.save_reduce(type, (NotImplemented,), obj=obj)
1933 logger.trace(pickler, "# T7")
1934 elif obj is EllipsisType:
1935 logger.trace(pickler, "T7: %s", obj)
1936 pickler.save_reduce(type, (Ellipsis,), obj=obj)
1937 logger.trace(pickler, "# T7")
1938 elif obj is EnumMeta:
1939 logger.trace(pickler, "T7: %s", obj)
1940 pickler.write(GLOBAL + b'enum\nEnumMeta\n')
1941 logger.trace(pickler, "# T7")
1942 elif obj is ExceptHookArgsType: #NOTE: must be after NoneType for pypy
1943 logger.trace(pickler, "T7: %s", obj)
1944 pickler.write(GLOBAL + b'threading\nExceptHookArgs\n')
1945 logger.trace(pickler, "# T7")
1947 else:
1948 _byref = getattr(pickler, '_byref', None)
1949 obj_recursive = id(obj) in getattr(pickler, '_postproc', ())
1950 incorrectly_named = not _locate_function(obj, pickler)
1951 if not _byref and not obj_recursive and incorrectly_named: # not a function, but the name was held over
1952 if postproc_list is None:
1953 postproc_list = []
1955 # thanks to Tom Stepleton pointing out pickler._session unneeded
1956 logger.trace(pickler, "T2: %s", obj)
1957 _dict, attrs = _get_typedict_type(obj, obj.__dict__.copy(), None, postproc_list) # copy dict proxy to a dict
1959 #print (_dict)
1960 #print ("%s\n%s" % (type(obj), obj.__name__))
1961 #print ("%s\n%s" % (obj.__bases__, obj.__dict__))
1962 slots = _dict.get('__slots__', ())
1963 if type(slots) == str:
1964 # __slots__ accepts a single string
1965 slots = (slots,)
1967 for name in slots:
1968 _dict.pop(name, None)
1970 if isinstance(obj, abc.ABCMeta):
1971 logger.trace(pickler, "ABC: %s", obj)
1972 _dict, attrs = _get_typedict_abc(obj, _dict, attrs, postproc_list)
1973 logger.trace(pickler, "# ABC")
1975 qualname = getattr(obj, '__qualname__', None)
1976 if attrs is not None:
1977 for k, v in attrs.items():
1978 postproc_list.append((setattr, (obj, k, v)))
1979 # TODO: Consider using the state argument to save_reduce?
1980 if qualname is not None:
1981 postproc_list.append((setattr, (obj, '__qualname__', qualname)))
1983 if not hasattr(obj, '__orig_bases__'):
1984 _save_with_postproc(pickler, (_create_type, (
1985 type(obj), obj.__name__, obj.__bases__, _dict
1986 )), obj=obj, postproc_list=postproc_list)
1987 else:
1988 # This case will always work, but might be overkill.
1989 _metadict = {
1990 'metaclass': type(obj)
1991 }
1993 if _dict:
1994 _dict_update = PartialType(_setitems, source=_dict)
1995 else:
1996 _dict_update = None
1998 _save_with_postproc(pickler, (new_class, (
1999 obj.__name__, obj.__orig_bases__, _metadict, _dict_update
2000 )), obj=obj, postproc_list=postproc_list)
2001 logger.trace(pickler, "# T2")
2002 else:
2003 obj_name = getattr(obj, '__qualname__', getattr(obj, '__name__', None))
2004 logger.trace(pickler, "T4: %s", obj)
2005 if incorrectly_named:
2006 warnings.warn(
2007 "Cannot locate reference to %r." % (obj,),
2008 PicklingWarning,
2009 stacklevel=3,
2010 )
2011 if obj_recursive:
2012 warnings.warn(
2013 "Cannot pickle %r: %s.%s has recursive self-references that "
2014 "trigger a RecursionError." % (obj, obj.__module__, obj_name),
2015 PicklingWarning,
2016 stacklevel=3,
2017 )
2018 #print (obj.__dict__)
2019 #print ("%s\n%s" % (type(obj), obj.__name__))
2020 #print ("%s\n%s" % (obj.__bases__, obj.__dict__))
2021 StockPickler.save_global(pickler, obj, name=obj_name)
2022 logger.trace(pickler, "# T4")
2023 return
2025@register(property)
2026@register(abc.abstractproperty)
2027def save_property(pickler, obj):
2028 logger.trace(pickler, "Pr: %s", obj)
2029 pickler.save_reduce(type(obj), (obj.fget, obj.fset, obj.fdel, obj.__doc__),
2030 obj=obj)
2031 logger.trace(pickler, "# Pr")
2033@register(staticmethod)
2034@register(classmethod)
2035@register(abc.abstractstaticmethod)
2036@register(abc.abstractclassmethod)
2037def save_classmethod(pickler, obj):
2038 logger.trace(pickler, "Cm: %s", obj)
2039 orig_func = obj.__func__
2041 # if type(obj.__dict__) is dict:
2042 # if obj.__dict__:
2043 # state = obj.__dict__
2044 # else:
2045 # state = None
2046 # else:
2047 # state = (None, {'__dict__', obj.__dict__})
2049 pickler.save_reduce(type(obj), (orig_func,), obj=obj)
2050 logger.trace(pickler, "# Cm")
2052@register(FunctionType)
2053def save_function(pickler, obj):
2054 if not _locate_function(obj, pickler):
2055 if type(obj.__code__) is not CodeType:
2056 # Some PyPy builtin functions have no module name, and thus are not
2057 # able to be located
2058 module_name = getattr(obj, '__module__', None)
2059 if module_name is None:
2060 module_name = __builtin__.__name__
2061 module = _import_module(module_name, safe=True)
2062 _pypy_builtin = False
2063 try:
2064 found, _ = _getattribute(module, obj.__qualname__)
2065 if getattr(found, '__func__', None) is obj:
2066 _pypy_builtin = True
2067 except AttributeError:
2068 pass
2070 if _pypy_builtin:
2071 logger.trace(pickler, "F3: %s", obj)
2072 pickler.save_reduce(getattr, (found, '__func__'), obj=obj)
2073 logger.trace(pickler, "# F3")
2074 return
2076 logger.trace(pickler, "F1: %s", obj)
2077 _recurse = getattr(pickler, '_recurse', None)
2078 _postproc = getattr(pickler, '_postproc', None)
2079 _main_modified = getattr(pickler, '_main_modified', None)
2080 _original_main = getattr(pickler, '_original_main', __builtin__)#'None'
2081 postproc_list = []
2082 if _recurse:
2083 # recurse to get all globals referred to by obj
2084 from .detect import globalvars
2085 globs_copy = globalvars(obj, recurse=True, builtin=True)
2087 # Add the name of the module to the globs dictionary to prevent
2088 # the duplication of the dictionary. Pickle the unpopulated
2089 # globals dictionary and set the remaining items after the function
2090 # is created to correctly handle recursion.
2091 globs = {'__name__': obj.__module__}
2092 else:
2093 globs_copy = obj.__globals__
2095 # If the globals is the __dict__ from the module being saved as a
2096 # session, substitute it by the dictionary being actually saved.
2097 if _main_modified and globs_copy is _original_main.__dict__:
2098 globs_copy = getattr(pickler, '_main', _original_main).__dict__
2099 globs = globs_copy
2100 # If the globals is a module __dict__, do not save it in the pickle.
2101 elif globs_copy is not None and obj.__module__ is not None and \
2102 getattr(_import_module(obj.__module__, True), '__dict__', None) is globs_copy:
2103 globs = globs_copy
2104 else:
2105 globs = {'__name__': obj.__module__}
2107 if globs_copy is not None and globs is not globs_copy:
2108 # In the case that the globals are copied, we need to ensure that
2109 # the globals dictionary is updated when all objects in the
2110 # dictionary are already created.
2111 glob_ids = {id(g) for g in globs_copy.values()}
2112 for stack_element in _postproc:
2113 if stack_element in glob_ids:
2114 _postproc[stack_element].append((_setitems, (globs, globs_copy)))
2115 break
2116 else:
2117 postproc_list.append((_setitems, (globs, globs_copy)))
2119 closure = obj.__closure__
2120 state_dict = {}
2121 for fattrname in ('__doc__', '__kwdefaults__', '__annotations__'):
2122 fattr = getattr(obj, fattrname, None)
2123 if fattr is not None:
2124 state_dict[fattrname] = fattr
2125 if obj.__qualname__ != obj.__name__:
2126 state_dict['__qualname__'] = obj.__qualname__
2127 if '__name__' not in globs or obj.__module__ != globs['__name__']:
2128 state_dict['__module__'] = obj.__module__
2130 state = obj.__dict__
2131 if type(state) is not dict:
2132 state_dict['__dict__'] = state
2133 state = None
2134 if state_dict:
2135 state = state, state_dict
2137 _save_with_postproc(pickler, (_create_function, (
2138 obj.__code__, globs, obj.__name__, obj.__defaults__,
2139 closure
2140 ), state), obj=obj, postproc_list=postproc_list)
2142 # Lift closure cell update to earliest function (#458)
2143 if _postproc:
2144 topmost_postproc = next(iter(_postproc.values()), None)
2145 if closure and topmost_postproc:
2146 for cell in closure:
2147 possible_postproc = (setattr, (cell, 'cell_contents', obj))
2148 try:
2149 topmost_postproc.remove(possible_postproc)
2150 except ValueError:
2151 continue
2153 # Change the value of the cell
2154 pickler.save_reduce(*possible_postproc)
2155 # pop None created by calling preprocessing step off stack
2156 pickler.write(POP)
2158 logger.trace(pickler, "# F1")
2159 else:
2160 logger.trace(pickler, "F2: %s", obj)
2161 name = getattr(obj, '__qualname__', getattr(obj, '__name__', None))
2162 StockPickler.save_global(pickler, obj, name=name)
2163 logger.trace(pickler, "# F2")
2164 return
2166if HAS_CTYPES and hasattr(ctypes, 'pythonapi'):
2167 _PyCapsule_New = ctypes.pythonapi.PyCapsule_New
2168 _PyCapsule_New.argtypes = (ctypes.c_void_p, ctypes.c_char_p, ctypes.c_void_p)
2169 _PyCapsule_New.restype = ctypes.py_object
2170 _PyCapsule_GetPointer = ctypes.pythonapi.PyCapsule_GetPointer
2171 _PyCapsule_GetPointer.argtypes = (ctypes.py_object, ctypes.c_char_p)
2172 _PyCapsule_GetPointer.restype = ctypes.c_void_p
2173 _PyCapsule_GetDestructor = ctypes.pythonapi.PyCapsule_GetDestructor
2174 _PyCapsule_GetDestructor.argtypes = (ctypes.py_object,)
2175 _PyCapsule_GetDestructor.restype = ctypes.c_void_p
2176 _PyCapsule_GetContext = ctypes.pythonapi.PyCapsule_GetContext
2177 _PyCapsule_GetContext.argtypes = (ctypes.py_object,)
2178 _PyCapsule_GetContext.restype = ctypes.c_void_p
2179 _PyCapsule_GetName = ctypes.pythonapi.PyCapsule_GetName
2180 _PyCapsule_GetName.argtypes = (ctypes.py_object,)
2181 _PyCapsule_GetName.restype = ctypes.c_char_p
2182 _PyCapsule_IsValid = ctypes.pythonapi.PyCapsule_IsValid
2183 _PyCapsule_IsValid.argtypes = (ctypes.py_object, ctypes.c_char_p)
2184 _PyCapsule_IsValid.restype = ctypes.c_bool
2185 _PyCapsule_SetContext = ctypes.pythonapi.PyCapsule_SetContext
2186 _PyCapsule_SetContext.argtypes = (ctypes.py_object, ctypes.c_void_p)
2187 _PyCapsule_SetDestructor = ctypes.pythonapi.PyCapsule_SetDestructor
2188 _PyCapsule_SetDestructor.argtypes = (ctypes.py_object, ctypes.c_void_p)
2189 _PyCapsule_SetName = ctypes.pythonapi.PyCapsule_SetName
2190 _PyCapsule_SetName.argtypes = (ctypes.py_object, ctypes.c_char_p)
2191 _PyCapsule_SetPointer = ctypes.pythonapi.PyCapsule_SetPointer
2192 _PyCapsule_SetPointer.argtypes = (ctypes.py_object, ctypes.c_void_p)
2193 #from _socket import CAPI as _testcapsule
2194 _testcapsule_name = b'dill._dill._testcapsule'
2195 _testcapsule = _PyCapsule_New(
2196 ctypes.cast(_PyCapsule_New, ctypes.c_void_p),
2197 ctypes.c_char_p(_testcapsule_name),
2198 None
2199 )
2200 PyCapsuleType = type(_testcapsule)
2201 @register(PyCapsuleType)
2202 def save_capsule(pickler, obj):
2203 logger.trace(pickler, "Cap: %s", obj)
2204 name = _PyCapsule_GetName(obj)
2205 #warnings.warn('Pickling a PyCapsule (%s) does not pickle any C data structures and could cause segmentation faults or other memory errors when unpickling.' % (name,), PicklingWarning)
2206 pointer = _PyCapsule_GetPointer(obj, name)
2207 context = _PyCapsule_GetContext(obj)
2208 destructor = _PyCapsule_GetDestructor(obj)
2209 pickler.save_reduce(_create_capsule, (pointer, name, context, destructor), obj=obj)
2210 logger.trace(pickler, "# Cap")
2211 _incedental_reverse_typemap['PyCapsuleType'] = PyCapsuleType
2212 _reverse_typemap['PyCapsuleType'] = PyCapsuleType
2213 _incedental_types.add(PyCapsuleType)
2214else:
2215 _testcapsule = None
2217@register(ContextType)
2218def save_context(pickler, obj):
2219 logger.trace(pickler, "Cx: %s", obj)
2220 pickler.save_reduce(ContextType, tuple(obj.items()), obj=obj)
2221 logger.trace(pickler, "# Cx")
2224#############################
2225# A quick fix for issue #500
2226# This should be removed when a better solution is found.
2228if hasattr(dataclasses, "_HAS_DEFAULT_FACTORY_CLASS"):
2229 @register(dataclasses._HAS_DEFAULT_FACTORY_CLASS)
2230 def save_dataclasses_HAS_DEFAULT_FACTORY_CLASS(pickler, obj):
2231 logger.trace(pickler, "DcHDF: %s", obj)
2232 pickler.write(GLOBAL + b"dataclasses\n_HAS_DEFAULT_FACTORY\n")
2233 logger.trace(pickler, "# DcHDF")
2235if hasattr(dataclasses, "MISSING"):
2236 @register(type(dataclasses.MISSING))
2237 def save_dataclasses_MISSING_TYPE(pickler, obj):
2238 logger.trace(pickler, "DcM: %s", obj)
2239 pickler.write(GLOBAL + b"dataclasses\nMISSING\n")
2240 logger.trace(pickler, "# DcM")
2242if hasattr(dataclasses, "KW_ONLY"):
2243 @register(type(dataclasses.KW_ONLY))
2244 def save_dataclasses_KW_ONLY_TYPE(pickler, obj):
2245 logger.trace(pickler, "DcKWO: %s", obj)
2246 pickler.write(GLOBAL + b"dataclasses\nKW_ONLY\n")
2247 logger.trace(pickler, "# DcKWO")
2249if hasattr(dataclasses, "_FIELD_BASE"):
2250 @register(dataclasses._FIELD_BASE)
2251 def save_dataclasses_FIELD_BASE(pickler, obj):
2252 logger.trace(pickler, "DcFB: %s", obj)
2253 pickler.write(GLOBAL + b"dataclasses\n" + obj.name.encode() + b"\n")
2254 logger.trace(pickler, "# DcFB")
2256#############################
2258# quick sanity checking
2259def pickles(obj,exact=False,safe=False,**kwds):
2260 """
2261 Quick check if object pickles with dill.
2263 If *exact=True* then an equality test is done to check if the reconstructed
2264 object matches the original object.
2266 If *safe=True* then any exception will raised in copy signal that the
2267 object is not picklable, otherwise only pickling errors will be trapped.
2269 Additional keyword arguments are as :func:`dumps` and :func:`loads`.
2270 """
2271 if safe: exceptions = (Exception,) # RuntimeError, ValueError
2272 else:
2273 exceptions = (TypeError, AssertionError, NotImplementedError, PicklingError, UnpicklingError)
2274 try:
2275 pik = copy(obj, **kwds)
2276 #FIXME: should check types match first, then check content if "exact"
2277 try:
2278 #FIXME: should be "(pik == obj).all()" for numpy comparison, though that'll fail if shapes differ
2279 result = bool(pik.all() == obj.all())
2280 except (AttributeError, TypeError):
2281 with warnings.catch_warnings():
2282 warnings.simplefilter('ignore') #FIXME: be specific
2283 result = pik == obj
2284 if hasattr(result, 'toarray'): # for unusual types like sparse matrix
2285 result = result.toarray().all()
2286 if result: return True
2287 if not exact:
2288 result = type(pik) == type(obj)
2289 if result: return result
2290 # class instances might have been dumped with byref=False
2291 return repr(type(pik)) == repr(type(obj)) #XXX: InstanceType?
2292 return False
2293 except exceptions:
2294 return False
2296def check(obj, *args, **kwds):
2297 """
2298 Check pickling of an object across another process.
2300 *python* is the path to the python interpreter (defaults to sys.executable)
2302 Set *verbose=True* to print the unpickled object in the other process.
2304 Additional keyword arguments are as :func:`dumps` and :func:`loads`.
2305 """
2306 # == undocumented ==
2307 # python -- the string path or executable name of the selected python
2308 # verbose -- if True, be verbose about printing warning messages
2309 # all other args and kwds are passed to dill.dumps #FIXME: ignore on load
2310 verbose = kwds.pop('verbose', False)
2311 python = kwds.pop('python', None)
2312 if python is None:
2313 import sys
2314 python = sys.executable
2315 # type check
2316 isinstance(python, str)
2317 import subprocess
2318 fail = True
2319 try:
2320 _obj = dumps(obj, *args, **kwds)
2321 fail = False
2322 finally:
2323 if fail and verbose:
2324 print("DUMP FAILED")
2325 #FIXME: fails if python interpreter path contains spaces
2326 # Use the following instead (which also processes the 'ignore' keyword):
2327 # ignore = kwds.pop('ignore', None)
2328 # unpickle = "dill.loads(%s, ignore=%s)"%(repr(_obj), repr(ignore))
2329 # cmd = [python, "-c", "import dill; print(%s)"%unpickle]
2330 # msg = "SUCCESS" if not subprocess.call(cmd) else "LOAD FAILED"
2331 msg = "%s -c import dill; print(dill.loads(%s))" % (python, repr(_obj))
2332 msg = "SUCCESS" if not subprocess.call(msg.split(None,2)) else "LOAD FAILED"
2333 if verbose:
2334 print(msg)
2335 return
2337# use to protect against missing attributes
2338def is_dill(pickler, child=None):
2339 "check the dill-ness of your pickler"
2340 if child is False or not hasattr(pickler.__class__, 'mro'):
2341 return 'dill' in pickler.__module__
2342 return Pickler in pickler.__class__.mro()
2344def _extend():
2345 """extend pickle with all of dill's registered types"""
2346 # need to have pickle not choke on _main_module? use is_dill(pickler)
2347 for t,func in Pickler.dispatch.items():
2348 try:
2349 StockPickler.dispatch[t] = func
2350 except Exception: #TypeError, PicklingError, UnpicklingError
2351 logger.trace(pickler, "skip: %s", t)
2352 return
2354del diff, _use_diff, use_diff
2356# EOF