Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.10/site-packages/dill/_dill.py: 40%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# -*- coding: utf-8 -*-
2#
3# Author: Mike McKerns (mmckerns @caltech and @uqfoundation)
4# Copyright (c) 2008-2015 California Institute of Technology.
5# Copyright (c) 2016-2026 The Uncertainty Quantification Foundation.
6# License: 3-clause BSD. The full license text is available at:
7# - https://github.com/uqfoundation/dill/blob/master/LICENSE
8"""
9dill: a utility for serialization of python objects
11The primary functions in `dill` are :func:`dump` and
12:func:`dumps` for serialization ("pickling") to a
13file or to a string, respectively, and :func:`load`
14and :func:`loads` for deserialization ("unpickling"),
15similarly, from a file or from a string. Other notable
16functions are :func:`~dill.dump_module` and
17:func:`~dill.load_module`, which are used to save and
18restore module objects, including an interpreter session.
20Based on code written by Oren Tirosh and Armin Ronacher.
21Extended to a (near) full set of the builtin types (in types module),
22and coded to the pickle interface, by <mmckerns@caltech.edu>.
23Initial port to python3 by Jonathan Dobson, continued by mmckerns.
24Tested against "all" python types (Std. Lib. CH 1-15 @ 2.7) by mmckerns.
25Tested against CH16+ Std. Lib. ... TBD.
26"""
28from __future__ import annotations
30__all__ = [
31 'dump','dumps','load','loads','copy',
32 'Pickler','Unpickler','register','pickle','pickles','check',
33 'DEFAULT_PROTOCOL','HIGHEST_PROTOCOL','HANDLE_FMODE','CONTENTS_FMODE','FILE_FMODE',
34 'PickleError','PickleWarning','PicklingError','PicklingWarning','UnpicklingError',
35 'UnpicklingWarning',
36]
38__module__ = 'dill'
40import warnings
41from .logger import adapter as logger
42from .logger import trace as _trace
43log = logger # backward compatibility (see issue #582)
45import os
46import sys
47diff = None
48_use_diff = False
49OLD38 = (sys.hexversion < 0x3080000)
50OLD39 = (sys.hexversion < 0x3090000)
51OLD310 = (sys.hexversion < 0x30a0000)
52OLD312a7 = (sys.hexversion < 0x30c00a7)
53#XXX: get types from .objtypes ?
54import builtins as __builtin__
55from pickle import _Pickler as StockPickler, Unpickler as StockUnpickler
56from pickle import GLOBAL, POP
57from _contextvars import Context as ContextType
58from _thread import LockType
59from _thread import RLock as RLockType
60try:
61 from _thread import _ExceptHookArgs as ExceptHookArgsType
62except ImportError:
63 ExceptHookArgsType = None
64try:
65 from _thread import _ThreadHandle as ThreadHandleType
66except ImportError:
67 ThreadHandleType = None
68#from io import IOBase
69from types import CodeType, FunctionType, MethodType, GeneratorType, \
70 TracebackType, FrameType, ModuleType, BuiltinMethodType
71BufferType = memoryview #XXX: unregistered
72ClassType = type # no 'old-style' classes
73EllipsisType = type(Ellipsis)
74#FileType = IOBase
75NotImplementedType = type(NotImplemented)
76SliceType = slice
77TypeType = type # 'new-style' classes #XXX: unregistered
78XRangeType = range
79from types import MappingProxyType as DictProxyType, new_class
80from pickle import DEFAULT_PROTOCOL, HIGHEST_PROTOCOL, PickleError, PicklingError, UnpicklingError
81import __main__ as _main_module
82import marshal
83import gc
84# import zlib
85import abc
86import dataclasses
87from weakref import ReferenceType, ProxyType, CallableProxyType
88from collections import OrderedDict
89from enum import Enum, EnumMeta
90from functools import partial
91from operator import itemgetter, attrgetter
92GENERATOR_FAIL = False
93import importlib.machinery
94EXTENSION_SUFFIXES = tuple(importlib.machinery.EXTENSION_SUFFIXES)
95try:
96 import ctypes
97 HAS_CTYPES = True
98 # if using `pypy`, pythonapi is not found
99 IS_PYPY = not hasattr(ctypes, 'pythonapi')
100except ImportError:
101 HAS_CTYPES = False
102 IS_PYPY = False
103NumpyUfuncType = None
104NumpyDType = None
105NumpyArrayType = None
106try:
107 if not importlib.machinery.PathFinder().find_spec('numpy'):
108 raise ImportError("No module named 'numpy'")
109 NumpyUfuncType = True
110 NumpyDType = True
111 NumpyArrayType = True
112except ImportError:
113 pass
114def __hook__():
115 global NumpyArrayType, NumpyDType, NumpyUfuncType
116 from numpy import ufunc as NumpyUfuncType
117 from numpy import ndarray as NumpyArrayType
118 from numpy import dtype as NumpyDType
119 return True
120if NumpyArrayType: # then has numpy
121 def ndarraysubclassinstance(obj_type):
122 if all((c.__module__, c.__name__) != ('numpy', 'ndarray') for c in obj_type.__mro__):
123 return False
124 # anything below here is a numpy array (or subclass) instance
125 __hook__() # import numpy (so the following works!!!)
126 # verify that __reduce__ has not been overridden
127 if obj_type.__reduce_ex__ is not NumpyArrayType.__reduce_ex__ \
128 or obj_type.__reduce__ is not NumpyArrayType.__reduce__:
129 return False
130 return True
131 def numpyufunc(obj_type):
132 return any((c.__module__, c.__name__) == ('numpy', 'ufunc') for c in obj_type.__mro__)
133 def numpydtype(obj_type):
134 if all((c.__module__, c.__name__) != ('numpy', 'dtype') for c in obj_type.__mro__):
135 return False
136 # anything below here is a numpy dtype
137 __hook__() # import numpy (so the following works!!!)
138 return obj_type is type(NumpyDType) # handles subclasses
139else:
140 def ndarraysubclassinstance(obj): return False
141 def numpyufunc(obj): return False
142 def numpydtype(obj): return False
144from types import GetSetDescriptorType, ClassMethodDescriptorType, \
145 WrapperDescriptorType, MethodDescriptorType, MemberDescriptorType, \
146 MethodWrapperType #XXX: unused
148# make sure to add these 'hand-built' types to _typemap
149CellType = type((lambda x: lambda y: x)(0).__closure__[0])
150PartialType = type(partial(int, base=2))
151SuperType = type(super(Exception, TypeError()))
152ItemGetterType = type(itemgetter(0))
153AttrGetterType = type(attrgetter('__repr__'))
155try:
156 from functools import _lru_cache_wrapper as LRUCacheType
157except ImportError:
158 LRUCacheType = None
160if not isinstance(LRUCacheType, type):
161 LRUCacheType = None
163def get_file_type(*args, **kwargs):
164 open = kwargs.pop("open", __builtin__.open)
165 f = open(os.devnull, *args, **kwargs)
166 t = type(f)
167 f.close()
168 return t
170IS_PYODIDE = sys.platform == 'emscripten'
172FileType = get_file_type('rb', buffering=0)
173TextWrapperType = get_file_type('r', buffering=-1)
174BufferedRandomType = None if IS_PYODIDE else get_file_type('r+b', buffering=-1)
175BufferedReaderType = get_file_type('rb', buffering=-1)
176BufferedWriterType = get_file_type('wb', buffering=-1)
177try:
178 from _pyio import open as _open
179 PyTextWrapperType = get_file_type('r', buffering=-1, open=_open)
180 PyBufferedRandomType = None if IS_PYODIDE else get_file_type('r+b', buffering=-1, open=_open)
181 PyBufferedReaderType = get_file_type('rb', buffering=-1, open=_open)
182 PyBufferedWriterType = get_file_type('wb', buffering=-1, open=_open)
183except ImportError:
184 PyTextWrapperType = PyBufferedRandomType = PyBufferedReaderType = PyBufferedWriterType = None
185from io import BytesIO as StringIO
186InputType = OutputType = None
187from socket import socket as SocketType
188#FIXME: additionally calls ForkingPickler.register several times
189from multiprocessing.reduction import _reduce_socket as reduce_socket
190try: #pragma: no cover
191 IS_IPYTHON = __IPYTHON__ # is True
192 ExitType = None # IPython.core.autocall.ExitAutocall
193 IPYTHON_SINGLETONS = ('exit', 'quit', 'get_ipython')
194except NameError:
195 IS_IPYTHON = False
196 try: ExitType = type(exit) # apparently 'exit' can be removed
197 except NameError: ExitType = None
198 IPYTHON_SINGLETONS = ()
200import inspect
201import typing
204### Shims for different versions of Python and dill
205class Sentinel(object):
206 """
207 Create a unique sentinel object that is pickled as a constant.
208 """
209 def __init__(self, name, module_name=None):
210 self.name = name
211 if module_name is None:
212 # Use the calling frame's module
213 self.__module__ = inspect.currentframe().f_back.f_globals['__name__']
214 else:
215 self.__module__ = module_name # pragma: no cover
216 def __repr__(self):
217 return self.__module__ + '.' + self.name # pragma: no cover
218 def __copy__(self):
219 return self # pragma: no cover
220 def __deepcopy__(self, memo):
221 return self # pragma: no cover
222 def __reduce__(self):
223 return self.name
224 def __reduce_ex__(self, protocol):
225 return self.name
227from . import _shims
228from ._shims import Reduce, Getattr
230### File modes
231#: Pickles the file handle, preserving mode. The position of the unpickled
232#: object is as for a new file handle.
233HANDLE_FMODE = 0
234#: Pickles the file contents, creating a new file if on load the file does
235#: not exist. The position = min(pickled position, EOF) and mode is chosen
236#: as such that "best" preserves behavior of the original file.
237CONTENTS_FMODE = 1
238#: Pickles the entire file (handle and contents), preserving mode and position.
239FILE_FMODE = 2
241### Shorthands (modified from python2.5/lib/pickle.py)
242def copy(obj, *args, **kwds):
243 """
244 Use pickling to 'copy' an object (i.e. `loads(dumps(obj))`).
246 See :func:`dumps` and :func:`loads` for keyword arguments.
247 """
248 ignore = kwds.pop('ignore', Unpickler.settings['ignore'])
249 return loads(dumps(obj, *args, **kwds), ignore=ignore)
251def dump(obj, file, protocol=None, byref=None, fmode=None, recurse=None, **kwds):#, strictio=None):
252 """
253 Pickle an object to a file.
255 See :func:`dumps` for keyword arguments.
256 """
257 from .settings import settings
258 protocol = settings['protocol'] if protocol is None else int(protocol)
259 _kwds = kwds.copy()
260 _kwds.update(dict(byref=byref, fmode=fmode, recurse=recurse))
261 Pickler(file, protocol, **_kwds).dump(obj)
262 return
264def dumps(obj, protocol=None, byref=None, fmode=None, recurse=None, **kwds):#, strictio=None):
265 """
266 Pickle an object to a string.
268 *protocol* is the pickler protocol, as defined for Python *pickle*.
270 If *byref=True*, then dill behaves a lot more like pickle as certain
271 objects (like modules) are pickled by reference as opposed to attempting
272 to pickle the object itself.
274 If *recurse=True*, then objects referred to in the global dictionary
275 are recursively traced and pickled, instead of the default behavior
276 of attempting to store the entire global dictionary. This is needed for
277 functions defined via *exec()*.
279 *fmode* (:const:`HANDLE_FMODE`, :const:`CONTENTS_FMODE`,
280 or :const:`FILE_FMODE`) indicates how file handles will be pickled.
281 For example, when pickling a data file handle for transfer to a remote
282 compute service, *FILE_FMODE* will include the file contents in the
283 pickle and cursor position so that a remote method can operate
284 transparently on an object with an open file handle.
286 Default values for keyword arguments can be set in :mod:`dill.settings`.
287 """
288 file = StringIO()
289 dump(obj, file, protocol, byref, fmode, recurse, **kwds)#, strictio)
290 return file.getvalue()
292def load(file, ignore=None, **kwds):
293 """
294 Unpickle an object from a file.
296 See :func:`loads` for keyword arguments.
297 """
298 return Unpickler(file, ignore=ignore, **kwds).load()
300def loads(str, ignore=None, **kwds):
301 """
302 Unpickle an object from a string.
304 If *ignore=False* then objects whose class is defined in the module
305 *__main__* are updated to reference the existing class in *__main__*,
306 otherwise they are left to refer to the reconstructed type, which may
307 be different.
309 Default values for keyword arguments can be set in :mod:`dill.settings`.
310 """
311 file = StringIO(str)
312 return load(file, ignore, **kwds)
314# def dumpzs(obj, protocol=None):
315# """pickle an object to a compressed string"""
316# return zlib.compress(dumps(obj, protocol))
318# def loadzs(str):
319# """unpickle an object from a compressed string"""
320# return loads(zlib.decompress(str))
322### End: Shorthands ###
324class MetaCatchingDict(dict):
325 def get(self, key, default=None):
326 try:
327 return self[key]
328 except KeyError:
329 return default
331 def __missing__(self, key):
332 if issubclass(key, type):
333 return save_type
334 else:
335 raise KeyError()
337class PickleWarning(Warning, PickleError):
338 pass
340class PicklingWarning(PickleWarning, PicklingError):
341 pass
343class UnpicklingWarning(PickleWarning, UnpicklingError):
344 pass
346### Extend the Picklers
347class Pickler(StockPickler):
348 """python's Pickler extended to interpreter sessions"""
349 dispatch: typing.Dict[type, typing.Callable[[Pickler, typing.Any], None]] \
350 = MetaCatchingDict(StockPickler.dispatch.copy())
351 """The dispatch table, a dictionary of serializing functions used
352 by Pickler to save objects of specific types. Use :func:`pickle`
353 or :func:`register` to associate types to custom functions.
355 :meta hide-value:
356 """
357 _session = False
358 from .settings import settings
360 def __init__(self, file, *args, **kwds):
361 settings = Pickler.settings
362 _byref = kwds.pop('byref', None)
363 #_strictio = kwds.pop('strictio', None)
364 _fmode = kwds.pop('fmode', None)
365 _recurse = kwds.pop('recurse', None)
366 StockPickler.__init__(self, file, *args, **kwds)
367 self._main = _main_module
368 self._diff_cache = {}
369 self._byref = settings['byref'] if _byref is None else _byref
370 self._strictio = False #_strictio
371 self._fmode = settings['fmode'] if _fmode is None else _fmode
372 self._recurse = settings['recurse'] if _recurse is None else _recurse
373 self._postproc = OrderedDict()
374 self._file = file
376 def save(self, obj, save_persistent_id=True):
377 # numpy hack
378 obj_type = type(obj)
379 if NumpyArrayType and not (obj_type is type or obj_type in Pickler.dispatch):
380 # register if the object is a numpy ufunc
381 # thanks to Paul Kienzle for pointing out ufuncs didn't pickle
382 if numpyufunc(obj_type):
383 @register(obj_type)
384 def save_numpy_ufunc(pickler, obj):
385 logger.trace(pickler, "Nu: %s", obj)
386 name = getattr(obj, '__qualname__', getattr(obj, '__name__', None))
387 StockPickler.save_global(pickler, obj, name=name)
388 logger.trace(pickler, "# Nu")
389 return
390 # NOTE: the above 'save' performs like:
391 # import copy_reg
392 # def udump(f): return f.__name__
393 # def uload(name): return getattr(numpy, name)
394 # copy_reg.pickle(NumpyUfuncType, udump, uload)
395 # register if the object is a numpy dtype
396 if numpydtype(obj_type):
397 @register(obj_type)
398 def save_numpy_dtype(pickler, obj):
399 logger.trace(pickler, "Dt: %s", obj)
400 pickler.save_reduce(_create_dtypemeta, (obj.type,), obj=obj)
401 logger.trace(pickler, "# Dt")
402 return
403 # NOTE: the above 'save' performs like:
404 # import copy_reg
405 # def uload(name): return type(NumpyDType(name))
406 # def udump(f): return uload, (f.type,)
407 # copy_reg.pickle(NumpyDTypeType, udump, uload)
408 # register if the object is a subclassed numpy array instance
409 if ndarraysubclassinstance(obj_type):
410 @register(obj_type)
411 def save_numpy_array(pickler, obj):
412 logger.trace(pickler, "Nu: (%s, %s)", obj.shape, obj.dtype)
413 npdict = getattr(obj, '__dict__', None)
414 f, args, state = obj.__reduce__()
415 pickler.save_reduce(_create_array, (f,args,state,npdict), obj=obj)
416 logger.trace(pickler, "# Nu")
417 return
418 # end numpy hack
420 if GENERATOR_FAIL and obj_type is GeneratorType:
421 msg = "Can't pickle %s: attribute lookup builtins.generator failed" % GeneratorType
422 raise PicklingError(msg)
423 StockPickler.save(self, obj, save_persistent_id)
425 save.__doc__ = StockPickler.save.__doc__
427 def dump(self, obj): #NOTE: if settings change, need to update attributes
428 logger.trace_setup(self)
429 StockPickler.dump(self, obj)
430 dump.__doc__ = StockPickler.dump.__doc__
432class Unpickler(StockUnpickler):
433 """python's Unpickler extended to interpreter sessions and more types"""
434 from .settings import settings
435 _session = False
437 def find_class(self, module, name):
438 if (module, name) == ('__builtin__', '__main__'):
439 return self._main.__dict__ #XXX: above set w/save_module_dict
440 elif (module, name) == ('__builtin__', 'NoneType'):
441 return type(None) #XXX: special case: NoneType missing
442 if module == 'dill.dill': module = 'dill._dill'
443 return StockUnpickler.find_class(self, module, name)
445 def __init__(self, *args, **kwds):
446 settings = Pickler.settings
447 _ignore = kwds.pop('ignore', None)
448 StockUnpickler.__init__(self, *args, **kwds)
449 self._main = _main_module
450 self._ignore = settings['ignore'] if _ignore is None else _ignore
452 def load(self): #NOTE: if settings change, need to update attributes
453 obj = StockUnpickler.load(self)
454 if type(obj).__module__ == getattr(_main_module, '__name__', '__main__'):
455 if not self._ignore:
456 # point obj class to main
457 try: obj.__class__ = getattr(self._main, type(obj).__name__)
458 except (AttributeError,TypeError): pass # defined in a file
459 #_main_module.__dict__.update(obj.__dict__) #XXX: should update globals ?
460 return obj
461 load.__doc__ = StockUnpickler.load.__doc__
462 pass
464'''
465def dispatch_table():
466 """get the dispatch table of registered types"""
467 return Pickler.dispatch
468'''
470pickle_dispatch_copy = StockPickler.dispatch.copy()
472def pickle(t, func):
473 """expose :attr:`~Pickler.dispatch` table for user-created extensions"""
474 Pickler.dispatch[t] = func
475 return
477def register(t):
478 """decorator to register types to Pickler's :attr:`~Pickler.dispatch` table"""
479 def proxy(func):
480 Pickler.dispatch[t] = func
481 return func
482 return proxy
484def _revert_extension():
485 """drop dill-registered types from pickle's dispatch table"""
486 for type, func in list(StockPickler.dispatch.items()):
487 if func.__module__ == __name__:
488 del StockPickler.dispatch[type]
489 if type in pickle_dispatch_copy:
490 StockPickler.dispatch[type] = pickle_dispatch_copy[type]
492def use_diff(on=True):
493 """
494 Reduces size of pickles by only including object which have changed.
496 Decreases pickle size but increases CPU time needed.
497 Also helps avoid some unpickleable objects.
498 MUST be called at start of script, otherwise changes will not be recorded.
499 """
500 global _use_diff, diff
501 _use_diff = on
502 if _use_diff and diff is None:
503 try:
504 from . import diff as d
505 except ImportError:
506 import diff as d
507 diff = d
509def _create_typemap():
510 import types
511 d = dict(list(__builtin__.__dict__.items()) + \
512 list(types.__dict__.items())).items()
513 for key, value in d:
514 if getattr(value, '__module__', None) == 'builtins' \
515 and type(value) is type:
516 yield key, value
517 return
518_reverse_typemap = dict(_create_typemap())
519_reverse_typemap.update({
520 'PartialType': PartialType,
521 'SuperType': SuperType,
522 'ItemGetterType': ItemGetterType,
523 'AttrGetterType': AttrGetterType,
524})
525if sys.hexversion < 0x30800a2:
526 _reverse_typemap.update({
527 'CellType': CellType,
528 })
530# "Incidental" implementation specific types. Unpickling these types in another
531# implementation of Python (PyPy -> CPython) is not guaranteed to work
533# This dictionary should contain all types that appear in Python implementations
534# but are not defined in https://docs.python.org/3/library/types.html#standard-interpreter-types
535x=OrderedDict()
536_incedental_reverse_typemap = {
537 'FileType': FileType,
538 'BufferedRandomType': BufferedRandomType,
539 'BufferedReaderType': BufferedReaderType,
540 'BufferedWriterType': BufferedWriterType,
541 'TextWrapperType': TextWrapperType,
542 'PyBufferedRandomType': PyBufferedRandomType,
543 'PyBufferedReaderType': PyBufferedReaderType,
544 'PyBufferedWriterType': PyBufferedWriterType,
545 'PyTextWrapperType': PyTextWrapperType,
546}
548_incedental_reverse_typemap.update({
549 "DictKeysType": type({}.keys()),
550 "DictValuesType": type({}.values()),
551 "DictItemsType": type({}.items()),
553 "OdictKeysType": type(x.keys()),
554 "OdictValuesType": type(x.values()),
555 "OdictItemsType": type(x.items()),
556})
558if ExitType:
559 _incedental_reverse_typemap['ExitType'] = ExitType
560if InputType:
561 _incedental_reverse_typemap['InputType'] = InputType
562 _incedental_reverse_typemap['OutputType'] = OutputType
564'''
565try:
566 import symtable
567 _incedental_reverse_typemap["SymtableEntryType"] = type(symtable.symtable("", "string", "exec")._table)
568except: #FIXME: fails to pickle
569 pass
571if sys.hexversion >= 0x30a00a0:
572 _incedental_reverse_typemap['LineIteratorType'] = type(compile('3', '', 'eval').co_lines())
573'''
575if sys.hexversion >= 0x30b00b0 and not IS_PYPY:
576 from types import GenericAlias
577 _incedental_reverse_typemap["GenericAliasIteratorType"] = type(iter(GenericAlias(list, (int,))))
578 '''
579 _incedental_reverse_typemap['PositionsIteratorType'] = type(compile('3', '', 'eval').co_positions())
580 '''
582try:
583 import winreg
584 _incedental_reverse_typemap["HKEYType"] = winreg.HKEYType
585except ImportError:
586 pass
588_reverse_typemap.update(_incedental_reverse_typemap)
589_incedental_types = set(_incedental_reverse_typemap.values())
591del x
593_typemap = dict((v, k) for k, v in _reverse_typemap.items())
595def _unmarshal(string):
596 return marshal.loads(string)
598def _load_type(name):
599 return _reverse_typemap[name]
601def _create_type(typeobj, *args):
602 return typeobj(*args)
604def _create_function(fcode, fglobals, fname=None, fdefaults=None,
605 fclosure=None, fdict=None, fkwdefaults=None):
606 # same as FunctionType, but enable passing __dict__ to new function,
607 # __dict__ is the storehouse for attributes added after function creation
608 func = FunctionType(fcode, fglobals or dict(), fname, fdefaults, fclosure)
609 if fdict is not None:
610 func.__dict__.update(fdict) #XXX: better copy? option to copy?
611 if fkwdefaults is not None:
612 func.__kwdefaults__ = fkwdefaults
613 # 'recurse' only stores referenced modules/objects in fglobals,
614 # thus we need to make sure that we have __builtins__ as well
615 if "__builtins__" not in func.__globals__:
616 func.__globals__["__builtins__"] = globals()["__builtins__"]
617 # assert id(fglobals) == id(func.__globals__)
618 return func
620class match:
621 """
622 Make available a limited structural pattern matching-like syntax for Python < 3.10
624 Patterns can be only tuples (without types) currently.
625 Inspired by the package pattern-matching-PEP634.
627 Usage:
628 >>> with match(args) as m:
629 >>> if m.case(('x', 'y')):
630 >>> # use m.x and m.y
631 >>> elif m.case(('x', 'y', 'z')):
632 >>> # use m.x, m.y and m.z
634 Equivalent native code for Python >= 3.10:
635 >>> match args:
636 >>> case (x, y):
637 >>> # use x and y
638 >>> case (x, y, z):
639 >>> # use x, y and z
640 """
641 def __init__(self, value):
642 self.value = value
643 self._fields = None
644 def __enter__(self):
645 return self
646 def __exit__(self, *exc_info):
647 return False
648 def case(self, args): # *args, **kwargs):
649 """just handles tuple patterns"""
650 if len(self.value) != len(args): # + len(kwargs):
651 return False
652 #if not all(isinstance(arg, pat) for arg, pat in zip(self.value[len(args):], kwargs.values())):
653 # return False
654 self.args = args # (*args, *kwargs)
655 return True
656 @property
657 def fields(self):
658 # Only bind names to values if necessary.
659 if self._fields is None:
660 self._fields = dict(zip(self.args, self.value))
661 return self._fields
662 def __getattr__(self, item):
663 return self.fields[item]
665ALL_CODE_PARAMS = [
666 # Version New attribute CodeType parameters
667 ((3,11,'a'), 'co_endlinetable', 'argcount posonlyargcount kwonlyargcount nlocals stacksize flags code consts names varnames filename name qualname firstlineno linetable endlinetable columntable exceptiontable freevars cellvars'),
668 ((3,11), 'co_exceptiontable', 'argcount posonlyargcount kwonlyargcount nlocals stacksize flags code consts names varnames filename name qualname firstlineno linetable exceptiontable freevars cellvars'),
669 ((3,11,'p'), 'co_qualname', 'argcount posonlyargcount kwonlyargcount nlocals stacksize flags code consts names varnames filename name qualname firstlineno linetable freevars cellvars'),
670 ((3,10), 'co_linetable', 'argcount posonlyargcount kwonlyargcount nlocals stacksize flags code consts names varnames filename name firstlineno linetable freevars cellvars'),
671 ((3,8), 'co_posonlyargcount', 'argcount posonlyargcount kwonlyargcount nlocals stacksize flags code consts names varnames filename name firstlineno lnotab freevars cellvars'),
672 ((3,7), 'co_kwonlyargcount', 'argcount kwonlyargcount nlocals stacksize flags code consts names varnames filename name firstlineno lnotab freevars cellvars'),
673 ]
674for version, new_attr, params in ALL_CODE_PARAMS:
675 if hasattr(CodeType, new_attr):
676 CODE_VERSION = version
677 CODE_PARAMS = params.split()
678 break
679ENCODE_PARAMS = set(CODE_PARAMS).intersection(
680 ['code', 'lnotab', 'linetable', 'endlinetable', 'columntable', 'exceptiontable'])
682def _create_code(*args):
683 if not isinstance(args[0], int): # co_lnotab stored from >= 3.10
684 LNOTAB, *args = args
685 else: # from < 3.10 (or pre-LNOTAB storage)
686 LNOTAB = b''
688 with match(args) as m:
689 # Python 3.11/3.12a (18 members)
690 if m.case((
691 'argcount', 'posonlyargcount', 'kwonlyargcount', 'nlocals', 'stacksize', 'flags', # args[0:6]
692 'code', 'consts', 'names', 'varnames', 'filename', 'name', 'qualname', 'firstlineno', # args[6:14]
693 'linetable', 'exceptiontable', 'freevars', 'cellvars' # args[14:]
694 )):
695 if CODE_VERSION == (3,11):
696 return CodeType(
697 *args[:6],
698 args[6].encode() if hasattr(args[6], 'encode') else args[6], # code
699 *args[7:14],
700 args[14].encode() if hasattr(args[14], 'encode') else args[14], # linetable
701 args[15].encode() if hasattr(args[15], 'encode') else args[15], # exceptiontable
702 args[16],
703 args[17],
704 )
705 fields = m.fields
706 # PyPy 3.11 7.3.19+ (17 members)
707 elif m.case((
708 'argcount', 'posonlyargcount', 'kwonlyargcount', 'nlocals', 'stacksize', 'flags', # args[0:6]
709 'code', 'consts', 'names', 'varnames', 'filename', 'name', 'qualname', # args[6:13]
710 'firstlineno', 'linetable', 'freevars', 'cellvars' # args[13:]
711 )):
712 if CODE_VERSION == (3,11,'p'):
713 return CodeType(
714 *args[:6],
715 args[6].encode() if hasattr(args[6], 'encode') else args[6], # code
716 *args[7:14],
717 args[14].encode() if hasattr(args[14], 'encode') else args[14], # linetable
718 args[15],
719 args[16],
720 )
721 fields = m.fields
722 # Python 3.10 or 3.8/3.9 (16 members)
723 elif m.case((
724 'argcount', 'posonlyargcount', 'kwonlyargcount', 'nlocals', 'stacksize', 'flags', # args[0:6]
725 'code', 'consts', 'names', 'varnames', 'filename', 'name', 'firstlineno', # args[6:13]
726 'LNOTAB_OR_LINETABLE', 'freevars', 'cellvars' # args[13:]
727 )):
728 if CODE_VERSION == (3,10) or CODE_VERSION == (3,8):
729 return CodeType(
730 *args[:6],
731 args[6].encode() if hasattr(args[6], 'encode') else args[6], # code
732 *args[7:13],
733 args[13].encode() if hasattr(args[13], 'encode') else args[13], # lnotab/linetable
734 args[14],
735 args[15],
736 )
737 fields = m.fields
738 if CODE_VERSION >= (3,10):
739 fields['linetable'] = m.LNOTAB_OR_LINETABLE
740 else:
741 fields['lnotab'] = LNOTAB if LNOTAB else m.LNOTAB_OR_LINETABLE
742 # Python 3.7 (15 args)
743 elif m.case((
744 'argcount', 'kwonlyargcount', 'nlocals', 'stacksize', 'flags', # args[0:5]
745 'code', 'consts', 'names', 'varnames', 'filename', 'name', 'firstlineno', # args[5:12]
746 'lnotab', 'freevars', 'cellvars' # args[12:]
747 )):
748 if CODE_VERSION == (3,7):
749 return CodeType(
750 *args[:5],
751 args[5].encode() if hasattr(args[5], 'encode') else args[5], # code
752 *args[6:12],
753 args[12].encode() if hasattr(args[12], 'encode') else args[12], # lnotab
754 args[13],
755 args[14],
756 )
757 fields = m.fields
758 # Python 3.11a (20 members)
759 elif m.case((
760 'argcount', 'posonlyargcount', 'kwonlyargcount', 'nlocals', 'stacksize', 'flags', # args[0:6]
761 'code', 'consts', 'names', 'varnames', 'filename', 'name', 'qualname', 'firstlineno', # args[6:14]
762 'linetable', 'endlinetable', 'columntable', 'exceptiontable', 'freevars', 'cellvars' # args[14:]
763 )):
764 if CODE_VERSION == (3,11,'a'):
765 return CodeType(
766 *args[:6],
767 args[6].encode() if hasattr(args[6], 'encode') else args[6], # code
768 *args[7:14],
769 *(a.encode() if hasattr(a, 'encode') else a for a in args[14:18]), # linetable-exceptiontable
770 args[18],
771 args[19],
772 )
773 fields = m.fields
774 else:
775 raise UnpicklingError("pattern match for code object failed")
777 # The args format doesn't match this version.
778 fields.setdefault('posonlyargcount', 0) # from python <= 3.7
779 fields.setdefault('lnotab', LNOTAB) # from python >= 3.10
780 fields.setdefault('linetable', b'') # from python <= 3.9
781 fields.setdefault('qualname', fields['name']) # from python <= 3.10
782 fields.setdefault('exceptiontable', b'') # from python <= 3.10
783 fields.setdefault('endlinetable', None) # from python != 3.11a
784 fields.setdefault('columntable', None) # from python != 3.11a
786 args = (fields[k].encode() if k in ENCODE_PARAMS and hasattr(fields[k], 'encode') else fields[k]
787 for k in CODE_PARAMS)
788 return CodeType(*args)
790def _create_ftype(ftypeobj, func, args, kwds):
791 if kwds is None:
792 kwds = {}
793 if args is None:
794 args = ()
795 return ftypeobj(func, *args, **kwds)
797def _create_typing_tuple(argz, *args): #NOTE: workaround python/cpython#94245
798 if not argz:
799 return typing.Tuple[()].copy_with(())
800 if argz == ((),):
801 return typing.Tuple[()]
802 return typing.Tuple[argz]
804if ThreadHandleType:
805 def _create_thread_handle(ident, done, *args): #XXX: ignores 'blocking'
806 from threading import _make_thread_handle
807 handle = _make_thread_handle(ident)
808 if done:
809 handle._set_done()
810 return handle
812def _create_lock(locked, *args): #XXX: ignores 'blocking'
813 from threading import Lock
814 lock = Lock()
815 if locked:
816 if not lock.acquire(False):
817 raise UnpicklingError("Cannot acquire lock")
818 return lock
820def _create_rlock(count, owner, *args): #XXX: ignores 'blocking'
821 lock = RLockType()
822 if owner is not None:
823 lock._acquire_restore((count, owner))
824 if owner and not lock._is_owned():
825 raise UnpicklingError("Cannot acquire lock")
826 return lock
828# thanks to matsjoyce for adding all the different file modes
829def _create_filehandle(name, mode, position, closed, open, strictio, fmode, fdata): # buffering=0
830 # only pickles the handle, not the file contents... good? or StringIO(data)?
831 # (for file contents see: http://effbot.org/librarybook/copy-reg.htm)
832 # NOTE: handle special cases first (are there more special cases?)
833 names = {'<stdin>':sys.__stdin__, '<stdout>':sys.__stdout__,
834 '<stderr>':sys.__stderr__} #XXX: better fileno=(0,1,2) ?
835 if name in list(names.keys()):
836 f = names[name] #XXX: safer "f=sys.stdin"
837 elif name == '<tmpfile>':
838 f = os.tmpfile()
839 elif name == '<fdopen>':
840 import tempfile
841 f = tempfile.TemporaryFile(mode)
842 else:
843 try:
844 exists = os.path.exists(name)
845 except Exception:
846 exists = False
847 if not exists:
848 if strictio:
849 raise FileNotFoundError("[Errno 2] No such file or directory: '%s'" % name)
850 elif "r" in mode and fmode != FILE_FMODE:
851 name = '<fdopen>' # or os.devnull?
852 current_size = 0 # or maintain position?
853 else:
854 current_size = os.path.getsize(name)
856 if position > current_size:
857 if strictio:
858 raise ValueError("invalid buffer size")
859 elif fmode == CONTENTS_FMODE:
860 position = current_size
861 # try to open the file by name
862 # NOTE: has different fileno
863 try:
864 #FIXME: missing: *buffering*, encoding, softspace
865 if fmode == FILE_FMODE:
866 f = open(name, mode if "w" in mode else "w")
867 f.write(fdata)
868 if "w" not in mode:
869 f.close()
870 f = open(name, mode)
871 elif name == '<fdopen>': # file did not exist
872 import tempfile
873 f = tempfile.TemporaryFile(mode)
874 # treat x mode as w mode
875 elif fmode == CONTENTS_FMODE \
876 and ("w" in mode or "x" in mode):
877 # stop truncation when opening
878 flags = os.O_CREAT
879 if "+" in mode:
880 flags |= os.O_RDWR
881 else:
882 flags |= os.O_WRONLY
883 f = os.fdopen(os.open(name, flags, 0o600), mode)
884 # set name to the correct value
885 r = getattr(f, "buffer", f)
886 r = getattr(r, "raw", r)
887 r.name = name
888 assert f.name == name
889 else:
890 f = open(name, mode)
891 except (IOError, FileNotFoundError):
892 err = sys.exc_info()[1]
893 raise UnpicklingError(err)
894 if closed:
895 f.close()
896 elif position >= 0 and fmode != HANDLE_FMODE:
897 f.seek(position)
898 return f
900def _create_stringi(value, position, closed):
901 f = StringIO(value)
902 if closed: f.close()
903 else: f.seek(position)
904 return f
906def _create_stringo(value, position, closed):
907 f = StringIO()
908 if closed: f.close()
909 else:
910 f.write(value)
911 f.seek(position)
912 return f
914class _itemgetter_helper(object):
915 def __init__(self):
916 self.items = []
917 def __getitem__(self, item):
918 self.items.append(item)
919 return
921class _attrgetter_helper(object):
922 def __init__(self, attrs, index=None):
923 self.attrs = attrs
924 self.index = index
925 def __getattribute__(self, attr):
926 attrs = object.__getattribute__(self, "attrs")
927 index = object.__getattribute__(self, "index")
928 if index is None:
929 index = len(attrs)
930 attrs.append(attr)
931 else:
932 attrs[index] = ".".join([attrs[index], attr])
933 return type(self)(attrs, index)
935class _dictproxy_helper(dict):
936 def __ror__(self, a):
937 return a
939_dictproxy_helper_instance = _dictproxy_helper()
941__d = {}
942try:
943 # In CPython 3.9 and later, this trick can be used to exploit the
944 # implementation of the __or__ function of MappingProxyType to get the true
945 # mapping referenced by the proxy. It may work for other implementations,
946 # but is not guaranteed.
947 MAPPING_PROXY_TRICK = __d is (DictProxyType(__d) | _dictproxy_helper_instance)
948except Exception:
949 MAPPING_PROXY_TRICK = False
950del __d
952# _CELL_REF and _CELL_EMPTY are used to stay compatible with versions of dill
953# whose _create_cell functions do not have a default value.
954# _CELL_REF can be safely removed entirely (replaced by empty tuples for calls
955# to _create_cell) once breaking changes are allowed.
956_CELL_REF = None
957_CELL_EMPTY = Sentinel('_CELL_EMPTY')
959def _create_cell(contents=None):
960 if contents is not _CELL_EMPTY:
961 value = contents
962 return (lambda: value).__closure__[0]
964def _create_weakref(obj, *args):
965 from weakref import ref
966 if obj is None: # it's dead
967 from collections import UserDict
968 return ref(UserDict(), *args)
969 return ref(obj, *args)
971def _create_weakproxy(obj, callable=False, *args):
972 from weakref import proxy
973 if obj is None: # it's dead
974 if callable: return proxy(lambda x:x, *args)
975 from collections import UserDict
976 return proxy(UserDict(), *args)
977 return proxy(obj, *args)
979def _eval_repr(repr_str):
980 return eval(repr_str)
982def _create_array(f, args, state, npdict=None):
983 #array = numpy.core.multiarray._reconstruct(*args)
984 array = f(*args)
985 array.__setstate__(state)
986 if npdict is not None: # we also have saved state in __dict__
987 array.__dict__.update(npdict)
988 return array
990def _create_dtypemeta(scalar_type):
991 if NumpyDType is True: __hook__() # a bit hacky I think
992 if scalar_type is None:
993 return NumpyDType
994 return type(NumpyDType(scalar_type))
996def _create_namedtuple(name, fieldnames, modulename, defaults=None):
997 class_ = _import_module(modulename + '.' + name, safe=True)
998 if class_ is not None:
999 return class_
1000 import collections
1001 t = collections.namedtuple(name, fieldnames, defaults=defaults, module=modulename)
1002 return t
1004def _create_capsule(pointer, name, context, destructor):
1005 attr_found = False
1006 try:
1007 # based on https://github.com/python/cpython/blob/f4095e53ab708d95e019c909d5928502775ba68f/Objects/capsule.c#L209-L231
1008 uname = name.decode('utf8')
1009 for i in range(1, uname.count('.')+1):
1010 names = uname.rsplit('.', i)
1011 try:
1012 module = __import__(names[0])
1013 except ImportError:
1014 pass
1015 obj = module
1016 for attr in names[1:]:
1017 obj = getattr(obj, attr)
1018 capsule = obj
1019 attr_found = True
1020 break
1021 except Exception:
1022 pass
1024 if attr_found:
1025 if _PyCapsule_IsValid(capsule, name):
1026 return capsule
1027 raise UnpicklingError("%s object exists at %s but a PyCapsule object was expected." % (type(capsule), name))
1028 else:
1029 #warnings.warn('Creating a new PyCapsule %s for a C data structure that may not be present in memory. Segmentation faults or other memory errors are possible.' % (name,), UnpicklingWarning)
1030 capsule = _PyCapsule_New(pointer, name, destructor)
1031 _PyCapsule_SetContext(capsule, context)
1032 return capsule
1034def _getattr(objclass, name, repr_str):
1035 attr = None
1036 if IS_PYPY:
1037 try: # hack to grab the reference directly
1038 attr = repr_str.split("'")[3]
1039 attr = eval(attr+'.__dict__["'+name+'"]')
1040 except Exception: pass
1041 if attr is None:
1042 # grab the descriptor off its owning class, repr_str is not evaluated
1043 try:
1044 attr = objclass.__dict__
1045 if type(attr) is DictProxyType:
1046 if sys.hexversion > 0x30f00a0 and name in ('__weakref__','__dict__'):
1047 attr = _dictproxy_helper.__dict__[name]
1048 else:
1049 attr = attr[name]
1050 else:
1051 attr = getattr(objclass,name)
1052 except (AttributeError, KeyError):
1053 attr = getattr(objclass,name)
1054 return attr
1056def _get_attr(self, name):
1057 # stop recursive pickling
1058 return getattr(self, name, None) or getattr(__builtin__, name)
1060def _import_module(import_name, safe=False):
1061 try:
1062 if import_name.startswith('__runtime__.'):
1063 return sys.modules[import_name]
1064 elif '.' in import_name:
1065 items = import_name.split('.')
1066 module = '.'.join(items[:-1])
1067 obj = items[-1]
1068 submodule = getattr(__import__(module, None, None, [obj]), obj)
1069 if isinstance(submodule, (ModuleType, type)):
1070 return submodule
1071 return __import__(import_name, None, None, [obj])
1072 else:
1073 return __import__(import_name)
1074 except (ImportError, AttributeError, KeyError):
1075 if safe:
1076 return None
1077 raise
1079# https://github.com/python/cpython/blob/a8912a0f8d9eba6d502c37d522221f9933e976db/Lib/pickle.py#L322-L333
1080def _getattribute(obj, name):
1081 for subpath in name.split('.'):
1082 if subpath == '<locals>':
1083 raise AttributeError("Can't get local attribute {!r} on {!r}"
1084 .format(name, obj))
1085 try:
1086 parent = obj
1087 obj = getattr(obj, subpath)
1088 except AttributeError:
1089 raise AttributeError("Can't get attribute {!r} on {!r}"
1090 .format(name, obj))
1091 return obj, parent
1093def _locate_function(obj, pickler=None):
1094 module_name = getattr(obj, '__module__', None)
1095 if module_name in ['__main__', None] or \
1096 pickler and is_dill(pickler, child=False) and pickler._session and module_name == pickler._main.__name__:
1097 return False
1098 if hasattr(obj, '__qualname__'):
1099 module = _import_module(module_name, safe=True)
1100 try:
1101 found, _ = _getattribute(module, obj.__qualname__)
1102 return found is obj
1103 except AttributeError:
1104 return False
1105 else:
1106 found = _import_module(module_name + '.' + obj.__name__, safe=True)
1107 return found is obj
1110def _setitems(dest, source):
1111 for k, v in source.items():
1112 dest[k] = v
1115def _save_with_postproc(pickler, reduction, is_pickler_dill=None, obj=Getattr.NO_DEFAULT, postproc_list=None):
1116 if obj is Getattr.NO_DEFAULT:
1117 obj = Reduce(reduction) # pragma: no cover
1119 if is_pickler_dill is None:
1120 is_pickler_dill = is_dill(pickler, child=True)
1121 if is_pickler_dill:
1122 # assert id(obj) not in pickler._postproc, str(obj) + ' already pushed on stack!'
1123 # if not hasattr(pickler, 'x'): pickler.x = 0
1124 # print(pickler.x*' ', 'push', obj, id(obj), pickler._recurse)
1125 # pickler.x += 1
1126 if postproc_list is None:
1127 postproc_list = []
1129 # Recursive object not supported. Default to a global instead.
1130 if id(obj) in pickler._postproc:
1131 name = '%s.%s ' % (obj.__module__, getattr(obj, '__qualname__', obj.__name__)) if hasattr(obj, '__module__') else ''
1132 warnings.warn('Cannot pickle %r: %shas recursive self-references that trigger a RecursionError.' % (obj, name), PicklingWarning)
1133 pickler.save_global(obj)
1134 return
1135 pickler._postproc[id(obj)] = postproc_list
1137 # TODO: Use state_setter in Python 3.8 to allow for faster cPickle implementations
1138 pickler.save_reduce(*reduction, obj=obj)
1140 if is_pickler_dill:
1141 # pickler.x -= 1
1142 # print(pickler.x*' ', 'pop', obj, id(obj))
1143 postproc = pickler._postproc.pop(id(obj))
1144 # assert postproc_list == postproc, 'Stack tampered!'
1145 for reduction in reversed(postproc):
1146 if reduction[0] is _setitems:
1147 # use the internal machinery of pickle.py to speedup when
1148 # updating a dictionary in postproc
1149 dest, source = reduction[1]
1150 if source:
1151 pickler.write(pickler.get(pickler.memo[id(dest)][0]))
1152 if sys.hexversion < 0x30e00a1:
1153 pickler._batch_setitems(iter(source.items()))
1154 else:
1155 pickler._batch_setitems(iter(source.items()), obj=obj)
1156 else:
1157 # Updating with an empty dictionary. Same as doing nothing.
1158 continue
1159 else:
1160 pickler.save_reduce(*reduction)
1161 # pop None created by calling preprocessing step off stack
1162 pickler.write(POP)
1164#@register(CodeType)
1165#def save_code(pickler, obj):
1166# logger.trace(pickler, "Co: %s", obj)
1167# pickler.save_reduce(_unmarshal, (marshal.dumps(obj),), obj=obj)
1168# logger.trace(pickler, "# Co")
1169# return
1171# The following function is based on 'save_codeobject' from 'cloudpickle'
1172# Copyright (c) 2012, Regents of the University of California.
1173# Copyright (c) 2009 `PiCloud, Inc. <http://www.picloud.com>`_.
1174# License: https://github.com/cloudpipe/cloudpickle/blob/master/LICENSE
1175@register(CodeType)
1176def save_code(pickler, obj):
1177 logger.trace(pickler, "Co: %s", obj)
1178 if hasattr(obj, "co_endlinetable"): # python 3.11a (20 args)
1179 args = (
1180 obj.co_lnotab, # for < python 3.10 [not counted in args]
1181 obj.co_argcount, obj.co_posonlyargcount,
1182 obj.co_kwonlyargcount, obj.co_nlocals, obj.co_stacksize,
1183 obj.co_flags, obj.co_code, obj.co_consts, obj.co_names,
1184 obj.co_varnames, obj.co_filename, obj.co_name, obj.co_qualname,
1185 obj.co_firstlineno, obj.co_linetable, obj.co_endlinetable,
1186 obj.co_columntable, obj.co_exceptiontable, obj.co_freevars,
1187 obj.co_cellvars
1188 )
1189 elif hasattr(obj, "co_exceptiontable"): # python 3.11 (18 args)
1190 with warnings.catch_warnings():
1191 if not OLD312a7: # issue 597
1192 warnings.filterwarnings('ignore', category=DeprecationWarning)
1193 args = ( # [not counted in args]
1194 getattr(obj, 'co_lnotab', b''), # for < python 3.10; 3.15.0b1+
1195 obj.co_argcount, obj.co_posonlyargcount,
1196 obj.co_kwonlyargcount, obj.co_nlocals, obj.co_stacksize,
1197 obj.co_flags, obj.co_code, obj.co_consts, obj.co_names,
1198 obj.co_varnames, obj.co_filename, obj.co_name, obj.co_qualname,
1199 obj.co_firstlineno, obj.co_linetable, obj.co_exceptiontable,
1200 obj.co_freevars, obj.co_cellvars
1201 )
1202 elif hasattr(obj, "co_qualname"): # pypy 3.11 7.3.19+ (17 args)
1203 args = (
1204 obj.co_lnotab, obj.co_argcount, obj.co_posonlyargcount,
1205 obj.co_kwonlyargcount, obj.co_nlocals, obj.co_stacksize,
1206 obj.co_flags, obj.co_code, obj.co_consts, obj.co_names,
1207 obj.co_varnames, obj.co_filename, obj.co_name, obj.co_qualname,
1208 obj.co_firstlineno, obj.co_linetable, obj.co_freevars,
1209 obj.co_cellvars
1210 )
1211 elif hasattr(obj, "co_linetable"): # python 3.10 (16 args)
1212 args = (
1213 obj.co_lnotab, # for < python 3.10 [not counted in args]
1214 obj.co_argcount, obj.co_posonlyargcount,
1215 obj.co_kwonlyargcount, obj.co_nlocals, obj.co_stacksize,
1216 obj.co_flags, obj.co_code, obj.co_consts, obj.co_names,
1217 obj.co_varnames, obj.co_filename, obj.co_name,
1218 obj.co_firstlineno, obj.co_linetable, obj.co_freevars,
1219 obj.co_cellvars
1220 )
1221 elif hasattr(obj, "co_posonlyargcount"): # python 3.8 (16 args)
1222 args = (
1223 obj.co_argcount, obj.co_posonlyargcount,
1224 obj.co_kwonlyargcount, obj.co_nlocals, obj.co_stacksize,
1225 obj.co_flags, obj.co_code, obj.co_consts, obj.co_names,
1226 obj.co_varnames, obj.co_filename, obj.co_name,
1227 obj.co_firstlineno, obj.co_lnotab, obj.co_freevars,
1228 obj.co_cellvars
1229 )
1230 else: # python 3.7 (15 args)
1231 args = (
1232 obj.co_argcount, obj.co_kwonlyargcount, obj.co_nlocals,
1233 obj.co_stacksize, obj.co_flags, obj.co_code, obj.co_consts,
1234 obj.co_names, obj.co_varnames, obj.co_filename,
1235 obj.co_name, obj.co_firstlineno, obj.co_lnotab,
1236 obj.co_freevars, obj.co_cellvars
1237 )
1239 pickler.save_reduce(_create_code, args, obj=obj)
1240 logger.trace(pickler, "# Co")
1241 return
1243def _repr_dict(obj):
1244 """Make a short string representation of a dictionary."""
1245 return "<%s object at %#012x>" % (type(obj).__name__, id(obj))
1247@register(dict)
1248def save_module_dict(pickler, obj):
1249 if is_dill(pickler, child=False) and obj == pickler._main.__dict__ and \
1250 not (pickler._session and pickler._first_pass):
1251 logger.trace(pickler, "D1: %s", _repr_dict(obj)) # obj
1252 pickler.write(bytes('c__builtin__\n__main__\n', 'UTF-8'))
1253 logger.trace(pickler, "# D1")
1254 elif (not is_dill(pickler, child=False)) and (obj == _main_module.__dict__):
1255 logger.trace(pickler, "D3: %s", _repr_dict(obj)) # obj
1256 pickler.write(bytes('c__main__\n__dict__\n', 'UTF-8')) #XXX: works in general?
1257 logger.trace(pickler, "# D3")
1258 elif '__name__' in obj and obj != _main_module.__dict__ \
1259 and type(obj['__name__']) is str \
1260 and obj is getattr(_import_module(obj['__name__'],True), '__dict__', None):
1261 logger.trace(pickler, "D4: %s", _repr_dict(obj)) # obj
1262 pickler.write(bytes('c%s\n__dict__\n' % obj['__name__'], 'UTF-8'))
1263 logger.trace(pickler, "# D4")
1264 else:
1265 logger.trace(pickler, "D2: %s", _repr_dict(obj)) # obj
1266 if is_dill(pickler, child=False) and pickler._session:
1267 # we only care about session the first pass thru
1268 pickler._first_pass = False
1269 StockPickler.save_dict(pickler, obj)
1270 logger.trace(pickler, "# D2")
1271 return
1274if not OLD310 and MAPPING_PROXY_TRICK:
1275 def save_dict_view(dicttype):
1276 def save_dict_view_for_function(func):
1277 def _save_dict_view(pickler, obj):
1278 logger.trace(pickler, "Dkvi: <%s>", obj)
1279 mapping = obj.mapping | _dictproxy_helper_instance
1280 pickler.save_reduce(func, (mapping,), obj=obj)
1281 logger.trace(pickler, "# Dkvi")
1282 return _save_dict_view
1283 return [
1284 (funcname, save_dict_view_for_function(getattr(dicttype, funcname)))
1285 for funcname in ('keys', 'values', 'items')
1286 ]
1287else:
1288 # The following functions are based on 'cloudpickle'
1289 # https://github.com/cloudpipe/cloudpickle/blob/5d89947288a18029672596a4d719093cc6d5a412/cloudpickle/cloudpickle.py#L922-L940
1290 # Copyright (c) 2012, Regents of the University of California.
1291 # Copyright (c) 2009 `PiCloud, Inc. <http://www.picloud.com>`_.
1292 # License: https://github.com/cloudpipe/cloudpickle/blob/master/LICENSE
1293 def save_dict_view(dicttype):
1294 def save_dict_keys(pickler, obj):
1295 logger.trace(pickler, "Dk: <%s>", obj)
1296 dict_constructor = _shims.Reduce(dicttype.fromkeys, (list(obj),))
1297 pickler.save_reduce(dicttype.keys, (dict_constructor,), obj=obj)
1298 logger.trace(pickler, "# Dk")
1300 def save_dict_values(pickler, obj):
1301 logger.trace(pickler, "Dv: <%s>", obj)
1302 dict_constructor = _shims.Reduce(dicttype, (enumerate(obj),))
1303 pickler.save_reduce(dicttype.values, (dict_constructor,), obj=obj)
1304 logger.trace(pickler, "# Dv")
1306 def save_dict_items(pickler, obj):
1307 logger.trace(pickler, "Di: <%s>", obj)
1308 pickler.save_reduce(dicttype.items, (dicttype(obj),), obj=obj)
1309 logger.trace(pickler, "# Di")
1311 return (
1312 ('keys', save_dict_keys),
1313 ('values', save_dict_values),
1314 ('items', save_dict_items)
1315 )
1317for __dicttype in (
1318 dict,
1319 OrderedDict
1320):
1321 __obj = __dicttype()
1322 for __funcname, __savefunc in save_dict_view(__dicttype):
1323 __tview = type(getattr(__obj, __funcname)())
1324 if __tview not in Pickler.dispatch:
1325 Pickler.dispatch[__tview] = __savefunc
1326del __dicttype, __obj, __funcname, __tview, __savefunc
1329@register(ClassType)
1330def save_classobj(pickler, obj): #FIXME: enable pickler._byref
1331 if not _locate_function(obj, pickler):
1332 logger.trace(pickler, "C1: %s", obj)
1333 pickler.save_reduce(ClassType, (obj.__name__, obj.__bases__,
1334 obj.__dict__), obj=obj)
1335 #XXX: or obj.__dict__.copy()), obj=obj) ?
1336 logger.trace(pickler, "# C1")
1337 else:
1338 logger.trace(pickler, "C2: %s", obj)
1339 name = getattr(obj, '__qualname__', getattr(obj, '__name__', None))
1340 StockPickler.save_global(pickler, obj, name=name)
1341 logger.trace(pickler, "# C2")
1342 return
1344@register(typing._GenericAlias)
1345def save_generic_alias(pickler, obj):
1346 args = obj.__args__
1347 if type(obj.__reduce__()) is str:
1348 logger.trace(pickler, "Ga0: %s", obj)
1349 StockPickler.save_global(pickler, obj, name=obj.__reduce__())
1350 logger.trace(pickler, "# Ga0")
1351 elif obj.__origin__ is tuple and (not args or args == ((),)):
1352 logger.trace(pickler, "Ga1: %s", obj)
1353 pickler.save_reduce(_create_typing_tuple, (args,), obj=obj)
1354 logger.trace(pickler, "# Ga1")
1355 else:
1356 logger.trace(pickler, "Ga2: %s", obj)
1357 StockPickler.save_reduce(pickler, *obj.__reduce__(), obj=obj)
1358 logger.trace(pickler, "# Ga2")
1359 return
1361if ThreadHandleType:
1362 @register(ThreadHandleType)
1363 def save_thread_handle(pickler, obj):
1364 logger.trace(pickler, "Th: %s", obj)
1365 pickler.save_reduce(_create_thread_handle, (obj.ident, obj.is_done()), obj=obj)
1366 logger.trace(pickler, "# Th")
1367 return
1369@register(LockType) #XXX: copied Thread will have new Event (due to new Lock)
1370def save_lock(pickler, obj):
1371 logger.trace(pickler, "Lo: %s", obj)
1372 pickler.save_reduce(_create_lock, (obj.locked(),), obj=obj)
1373 logger.trace(pickler, "# Lo")
1374 return
1376@register(RLockType)
1377def save_rlock(pickler, obj):
1378 logger.trace(pickler, "RL: %s", obj)
1379 r = obj.__repr__() # don't use _release_save as it unlocks the lock
1380 count = int(r.split('count=')[1].split()[0].rstrip('>'))
1381 owner = int(r.split('owner=')[1].split()[0])
1382 pickler.save_reduce(_create_rlock, (count,owner,), obj=obj)
1383 logger.trace(pickler, "# RL")
1384 return
1386#@register(SocketType) #FIXME: causes multiprocess test_pickling FAIL
1387def save_socket(pickler, obj):
1388 logger.trace(pickler, "So: %s", obj)
1389 pickler.save_reduce(*reduce_socket(obj))
1390 logger.trace(pickler, "# So")
1391 return
1393def _save_file(pickler, obj, open_):
1394 if obj.closed:
1395 position = 0
1396 else:
1397 obj.flush()
1398 if obj in (sys.__stdout__, sys.__stderr__, sys.__stdin__):
1399 position = -1
1400 else:
1401 position = obj.tell()
1402 if is_dill(pickler, child=True) and pickler._fmode == FILE_FMODE:
1403 f = open_(obj.name, "r")
1404 fdata = f.read()
1405 f.close()
1406 else:
1407 fdata = ""
1408 if is_dill(pickler, child=True):
1409 strictio = pickler._strictio
1410 fmode = pickler._fmode
1411 else:
1412 strictio = False
1413 fmode = 0 # HANDLE_FMODE
1414 pickler.save_reduce(_create_filehandle, (obj.name, obj.mode, position,
1415 obj.closed, open_, strictio,
1416 fmode, fdata), obj=obj)
1417 return
1420@register(FileType) #XXX: in 3.x has buffer=0, needs different _create?
1421@register(BufferedReaderType)
1422@register(BufferedWriterType)
1423@register(TextWrapperType)
1424def save_file(pickler, obj):
1425 logger.trace(pickler, "Fi: %s", obj)
1426 f = _save_file(pickler, obj, open)
1427 logger.trace(pickler, "# Fi")
1428 return f
1430if BufferedRandomType:
1431 @register(BufferedRandomType)
1432 def save_file(pickler, obj):
1433 logger.trace(pickler, "Fi: %s", obj)
1434 f = _save_file(pickler, obj, open)
1435 logger.trace(pickler, "# Fi")
1436 return f
1438if PyTextWrapperType:
1439 @register(PyBufferedReaderType)
1440 @register(PyBufferedWriterType)
1441 @register(PyTextWrapperType)
1442 def save_file(pickler, obj):
1443 logger.trace(pickler, "Fi: %s", obj)
1444 f = _save_file(pickler, obj, _open)
1445 logger.trace(pickler, "# Fi")
1446 return f
1448 if PyBufferedRandomType:
1449 @register(PyBufferedRandomType)
1450 def save_file(pickler, obj):
1451 logger.trace(pickler, "Fi: %s", obj)
1452 f = _save_file(pickler, obj, _open)
1453 logger.trace(pickler, "# Fi")
1454 return f
1457# The following two functions are based on 'saveCStringIoInput'
1458# and 'saveCStringIoOutput' from spickle
1459# Copyright (c) 2011 by science+computing ag
1460# License: http://www.apache.org/licenses/LICENSE-2.0
1461if InputType:
1462 @register(InputType)
1463 def save_stringi(pickler, obj):
1464 logger.trace(pickler, "Io: %s", obj)
1465 if obj.closed:
1466 value = ''; position = 0
1467 else:
1468 value = obj.getvalue(); position = obj.tell()
1469 pickler.save_reduce(_create_stringi, (value, position, \
1470 obj.closed), obj=obj)
1471 logger.trace(pickler, "# Io")
1472 return
1474 @register(OutputType)
1475 def save_stringo(pickler, obj):
1476 logger.trace(pickler, "Io: %s", obj)
1477 if obj.closed:
1478 value = ''; position = 0
1479 else:
1480 value = obj.getvalue(); position = obj.tell()
1481 pickler.save_reduce(_create_stringo, (value, position, \
1482 obj.closed), obj=obj)
1483 logger.trace(pickler, "# Io")
1484 return
1486if LRUCacheType is not None:
1487 from functools import lru_cache
1488 @register(LRUCacheType)
1489 def save_lru_cache(pickler, obj):
1490 logger.trace(pickler, "LRU: %s", obj)
1491 if OLD39:
1492 kwargs = obj.cache_info()
1493 args = (kwargs.maxsize,)
1494 else:
1495 kwargs = obj.cache_parameters()
1496 args = (kwargs['maxsize'], kwargs['typed'])
1497 if args != lru_cache.__defaults__:
1498 wrapper = Reduce(lru_cache, args, is_callable=True)
1499 else:
1500 wrapper = lru_cache
1501 pickler.save_reduce(wrapper, (obj.__wrapped__,), obj=obj)
1502 logger.trace(pickler, "# LRU")
1503 return
1505@register(SuperType)
1506def save_super(pickler, obj):
1507 logger.trace(pickler, "Su: %s", obj)
1508 pickler.save_reduce(super, (obj.__thisclass__, obj.__self__), obj=obj)
1509 logger.trace(pickler, "# Su")
1510 return
1512if IS_PYPY:
1513 @register(MethodType)
1514 def save_instancemethod0(pickler, obj):
1515 code = getattr(obj.__func__, '__code__', None)
1516 if code is not None and type(code) is not CodeType \
1517 and getattr(obj.__self__, obj.__name__) == obj:
1518 # Some PyPy builtin functions have no module name
1519 logger.trace(pickler, "Me2: %s", obj)
1520 # TODO: verify that this works for all PyPy builtin methods
1521 pickler.save_reduce(getattr, (obj.__self__, obj.__name__), obj=obj)
1522 logger.trace(pickler, "# Me2")
1523 return
1525 logger.trace(pickler, "Me1: %s", obj)
1526 pickler.save_reduce(MethodType, (obj.__func__, obj.__self__), obj=obj)
1527 logger.trace(pickler, "# Me1")
1528 return
1529else:
1530 @register(MethodType)
1531 def save_instancemethod0(pickler, obj):
1532 logger.trace(pickler, "Me1: %s", obj)
1533 pickler.save_reduce(MethodType, (obj.__func__, obj.__self__), obj=obj)
1534 logger.trace(pickler, "# Me1")
1535 return
1537if not IS_PYPY:
1538 @register(MemberDescriptorType)
1539 @register(GetSetDescriptorType)
1540 @register(MethodDescriptorType)
1541 @register(WrapperDescriptorType)
1542 @register(ClassMethodDescriptorType)
1543 def save_wrapper_descriptor(pickler, obj):
1544 logger.trace(pickler, "Wr: %s", obj)
1545 pickler.save_reduce(_getattr, (obj.__objclass__, obj.__name__,
1546 obj.__repr__()), obj=obj)
1547 logger.trace(pickler, "# Wr")
1548 return
1549else:
1550 @register(MemberDescriptorType)
1551 @register(GetSetDescriptorType)
1552 def save_wrapper_descriptor(pickler, obj):
1553 logger.trace(pickler, "Wr: %s", obj)
1554 pickler.save_reduce(_getattr, (obj.__objclass__, obj.__name__,
1555 obj.__repr__()), obj=obj)
1556 logger.trace(pickler, "# Wr")
1557 return
1559@register(CellType)
1560def save_cell(pickler, obj):
1561 try:
1562 f = obj.cell_contents
1563 except ValueError: # cell is empty
1564 logger.trace(pickler, "Ce3: %s", obj)
1565 # _shims._CELL_EMPTY is defined in _shims.py to support PyPy 2.7.
1566 # It unpickles to a sentinel object _dill._CELL_EMPTY, also created in
1567 # _shims.py. This object is not present in Python 3 because the cell's
1568 # contents can be deleted in newer versions of Python. The reduce object
1569 # will instead unpickle to None if unpickled in Python 3.
1571 # When breaking changes are made to dill, (_shims._CELL_EMPTY,) can
1572 # be replaced by () OR the delattr function can be removed repending on
1573 # whichever is more convenient.
1574 pickler.save_reduce(_create_cell, (_shims._CELL_EMPTY,), obj=obj)
1575 # Call the function _delattr on the cell's cell_contents attribute
1576 # The result of this function call will be None
1577 pickler.save_reduce(_shims._delattr, (obj, 'cell_contents'))
1578 # pop None created by calling _delattr off stack
1579 pickler.write(POP)
1580 logger.trace(pickler, "# Ce3")
1581 return
1582 if is_dill(pickler, child=True):
1583 if id(f) in pickler._postproc:
1584 # Already seen. Add to its postprocessing.
1585 postproc = pickler._postproc[id(f)]
1586 else:
1587 # Haven't seen it. Add to the highest possible object and set its
1588 # value as late as possible to prevent cycle.
1589 postproc = next(iter(pickler._postproc.values()), None)
1590 if postproc is not None:
1591 logger.trace(pickler, "Ce2: %s", obj)
1592 # _CELL_REF is defined in _shims.py to support older versions of
1593 # dill. When breaking changes are made to dill, (_CELL_REF,) can
1594 # be replaced by ()
1595 pickler.save_reduce(_create_cell, (_CELL_REF,), obj=obj)
1596 postproc.append((_shims._setattr, (obj, 'cell_contents', f)))
1597 logger.trace(pickler, "# Ce2")
1598 return
1599 logger.trace(pickler, "Ce1: %s", obj)
1600 pickler.save_reduce(_create_cell, (f,), obj=obj)
1601 logger.trace(pickler, "# Ce1")
1602 return
1604if MAPPING_PROXY_TRICK:
1605 @register(DictProxyType)
1606 def save_dictproxy(pickler, obj):
1607 logger.trace(pickler, "Mp: %s", _repr_dict(obj)) # obj
1608 mapping = obj | _dictproxy_helper_instance
1609 pickler.save_reduce(DictProxyType, (mapping,), obj=obj)
1610 logger.trace(pickler, "# Mp")
1611 return
1612else:
1613 @register(DictProxyType)
1614 def save_dictproxy(pickler, obj):
1615 logger.trace(pickler, "Mp: %s", _repr_dict(obj)) # obj
1616 pickler.save_reduce(DictProxyType, (obj.copy(),), obj=obj)
1617 logger.trace(pickler, "# Mp")
1618 return
1620@register(SliceType)
1621def save_slice(pickler, obj):
1622 logger.trace(pickler, "Sl: %s", obj)
1623 pickler.save_reduce(slice, (obj.start, obj.stop, obj.step), obj=obj)
1624 logger.trace(pickler, "# Sl")
1625 return
1627@register(XRangeType)
1628@register(EllipsisType)
1629@register(NotImplementedType)
1630def save_singleton(pickler, obj):
1631 logger.trace(pickler, "Si: %s", obj)
1632 pickler.save_reduce(_eval_repr, (obj.__repr__(),), obj=obj)
1633 logger.trace(pickler, "# Si")
1634 return
1636def _proxy_helper(obj): # a dead proxy returns a reference to None
1637 """get memory address of proxy's reference object"""
1638 _repr = repr(obj)
1639 try: _str = str(obj)
1640 except ReferenceError: # it's a dead proxy
1641 return id(None)
1642 if _str == _repr: return id(obj) # it's a repr
1643 try: # either way, it's a proxy from here
1644 address = int(_str.rstrip('>').split(' at ')[-1], base=16)
1645 except ValueError: # special case: proxy of a 'type'
1646 if not IS_PYPY:
1647 address = int(_repr.rstrip('>').split(' at ')[-1], base=16)
1648 else:
1649 objects = iter(gc.get_objects())
1650 for _obj in objects:
1651 if repr(_obj) == _str: return id(_obj)
1652 # all bad below... nothing found so throw ReferenceError
1653 msg = "Cannot reference object for proxy at '%s'" % id(obj)
1654 raise ReferenceError(msg)
1655 return address
1657def _locate_object(address, module=None):
1658 """get object located at the given memory address (inverse of id(obj))"""
1659 special = [None, True, False] #XXX: more...?
1660 for obj in special:
1661 if address == id(obj): return obj
1662 if module:
1663 objects = iter(module.__dict__.values())
1664 else: objects = iter(gc.get_objects())
1665 for obj in objects:
1666 if address == id(obj): return obj
1667 # all bad below... nothing found so throw ReferenceError or TypeError
1668 try: address = hex(address)
1669 except TypeError:
1670 raise TypeError("'%s' is not a valid memory address" % str(address))
1671 raise ReferenceError("Cannot reference object at '%s'" % address)
1673@register(ReferenceType)
1674def save_weakref(pickler, obj):
1675 refobj = obj()
1676 logger.trace(pickler, "R1: %s", obj)
1677 #refobj = ctypes.pythonapi.PyWeakref_GetObject(obj) # dead returns "None"
1678 pickler.save_reduce(_create_weakref, (refobj,), obj=obj)
1679 logger.trace(pickler, "# R1")
1680 return
1682@register(ProxyType)
1683@register(CallableProxyType)
1684def save_weakproxy(pickler, obj):
1685 # Must do string substitution here and use %r to avoid ReferenceError.
1686 logger.trace(pickler, "R2: %r" % obj)
1687 refobj = _locate_object(_proxy_helper(obj))
1688 pickler.save_reduce(_create_weakproxy, (refobj, callable(obj)), obj=obj)
1689 logger.trace(pickler, "# R2")
1690 return
1692def _is_builtin_module(module):
1693 if not hasattr(module, "__file__"): return True
1694 if module.__file__ is None: return False
1695 # If a module file name starts with prefix, it should be a builtin
1696 # module, so should always be pickled as a reference.
1697 names = ["base_prefix", "base_exec_prefix", "exec_prefix", "prefix", "real_prefix"]
1698 rp = os.path.realpath
1699 # See https://github.com/uqfoundation/dill/issues/566
1700 return (
1701 any(
1702 module.__file__.startswith(getattr(sys, name))
1703 or rp(module.__file__).startswith(rp(getattr(sys, name)))
1704 for name in names
1705 if hasattr(sys, name)
1706 )
1707 or module.__file__.endswith(EXTENSION_SUFFIXES)
1708 or 'site-packages' in module.__file__
1709 )
1711def _is_imported_module(module):
1712 return getattr(module, '__loader__', None) is not None or module in sys.modules.values()
1714@register(ModuleType)
1715def save_module(pickler, obj):
1716 if False: #_use_diff:
1717 if obj.__name__.split('.', 1)[0] != "dill":
1718 try:
1719 changed = diff.whats_changed(obj, seen=pickler._diff_cache)[0]
1720 except RuntimeError: # not memorised module, probably part of dill
1721 pass
1722 else:
1723 logger.trace(pickler, "M2: %s with diff", obj)
1724 logger.info("Diff: %s", changed.keys())
1725 pickler.save_reduce(_import_module, (obj.__name__,), obj=obj,
1726 state=changed)
1727 logger.trace(pickler, "# M2")
1728 return
1730 logger.trace(pickler, "M1: %s", obj)
1731 pickler.save_reduce(_import_module, (obj.__name__,), obj=obj)
1732 logger.trace(pickler, "# M1")
1733 else:
1734 builtin_mod = _is_builtin_module(obj)
1735 is_session_main = is_dill(pickler, child=True) and obj is pickler._main
1736 if (obj.__name__ not in ("builtins", "dill", "dill._dill") and not builtin_mod
1737 or is_session_main):
1738 logger.trace(pickler, "M1: %s", obj)
1739 # Hack for handling module-type objects in load_module().
1740 mod_name = obj.__name__ if _is_imported_module(obj) else '__runtime__.%s' % obj.__name__
1741 # Second references are saved as __builtin__.__main__ in save_module_dict().
1742 main_dict = obj.__dict__.copy()
1743 for item in ('__builtins__', '__loader__'):
1744 main_dict.pop(item, None)
1745 for item in IPYTHON_SINGLETONS: #pragma: no cover
1746 if getattr(main_dict.get(item), '__module__', '').startswith('IPython'):
1747 del main_dict[item]
1748 pickler.save_reduce(_import_module, (mod_name,), obj=obj, state=main_dict)
1749 logger.trace(pickler, "# M1")
1750 elif obj.__name__ == "dill._dill":
1751 logger.trace(pickler, "M2: %s", obj)
1752 pickler.save_global(obj, name="_dill")
1753 logger.trace(pickler, "# M2")
1754 else:
1755 logger.trace(pickler, "M2: %s", obj)
1756 pickler.save_reduce(_import_module, (obj.__name__,), obj=obj)
1757 logger.trace(pickler, "# M2")
1758 return
1760# The following function is based on '_extract_class_dict' from 'cloudpickle'
1761# Copyright (c) 2012, Regents of the University of California.
1762# Copyright (c) 2009 `PiCloud, Inc. <http://www.picloud.com>`_.
1763# License: https://github.com/cloudpipe/cloudpickle/blob/master/LICENSE
1764def _get_typedict_type(cls, clsdict, attrs, postproc_list):
1765 """Retrieve a copy of the dict of a class without the inherited methods"""
1766 if len(cls.__bases__) == 1:
1767 inherited_dict = cls.__bases__[0].__dict__
1768 else:
1769 inherited_dict = {}
1770 for base in reversed(cls.__bases__):
1771 inherited_dict.update(base.__dict__)
1772 to_remove = []
1773 for name, value in dict.items(clsdict):
1774 try:
1775 base_value = inherited_dict[name]
1776 if value is base_value and hasattr(value, '__qualname__'):
1777 to_remove.append(name)
1778 except KeyError:
1779 pass
1780 for name in to_remove:
1781 dict.pop(clsdict, name)
1783 if issubclass(type(cls), type):
1784 clsdict.pop('__dict__', None)
1785 clsdict.pop('__weakref__', None)
1786 # clsdict.pop('__prepare__', None)
1787 return clsdict, attrs
1789def _get_typedict_abc(obj, _dict, attrs, postproc_list):
1790 if hasattr(abc, '_get_dump'):
1791 (registry, _, _, _) = abc._get_dump(obj)
1792 register = obj.register
1793 postproc_list.extend((register, (reg(),)) for reg in registry)
1794 elif hasattr(obj, '_abc_registry'):
1795 registry = obj._abc_registry
1796 register = obj.register
1797 postproc_list.extend((register, (reg,)) for reg in registry)
1798 else:
1799 raise PicklingError("Cannot find registry of ABC %s", obj)
1801 if '_abc_registry' in _dict:
1802 _dict.pop('_abc_registry', None)
1803 _dict.pop('_abc_cache', None)
1804 _dict.pop('_abc_negative_cache', None)
1805 # _dict.pop('_abc_negative_cache_version', None)
1806 else:
1807 _dict.pop('_abc_impl', None)
1808 return _dict, attrs
1810@register(TypeType)
1811def save_type(pickler, obj, postproc_list=None):
1812 if obj in _typemap:
1813 logger.trace(pickler, "T1: %s", obj)
1814 # if obj in _incedental_types:
1815 # warnings.warn('Type %r may only exist on this implementation of Python and cannot be unpickled in other implementations.' % (obj,), PicklingWarning)
1816 pickler.save_reduce(_load_type, (_typemap[obj],), obj=obj)
1817 logger.trace(pickler, "# T1")
1818 elif obj.__bases__ == (tuple,) and all([hasattr(obj, attr) for attr in ('_fields','_asdict','_make','_replace')]):
1819 # special case: namedtuples
1820 logger.trace(pickler, "T6: %s", obj)
1822 obj_name = getattr(obj, '__qualname__', getattr(obj, '__name__', None))
1823 if obj.__name__ != obj_name:
1824 if postproc_list is None:
1825 postproc_list = []
1826 postproc_list.append((setattr, (obj, '__qualname__', obj_name)))
1828 if not obj._field_defaults:
1829 _save_with_postproc(pickler, (_create_namedtuple, (obj.__name__, obj._fields, obj.__module__)), obj=obj, postproc_list=postproc_list)
1830 else:
1831 defaults = [obj._field_defaults[field] for field in obj._fields if field in obj._field_defaults]
1832 _save_with_postproc(pickler, (_create_namedtuple, (obj.__name__, obj._fields, obj.__module__, defaults)), obj=obj, postproc_list=postproc_list)
1833 logger.trace(pickler, "# T6")
1834 return
1836 # special caes: NoneType, NotImplementedType, EllipsisType, EnumMeta, etc
1837 elif obj is type(None):
1838 logger.trace(pickler, "T7: %s", obj)
1839 #XXX: pickler.save_reduce(type, (None,), obj=obj)
1840 pickler.write(GLOBAL + b'__builtin__\nNoneType\n')
1841 logger.trace(pickler, "# T7")
1842 elif obj is NotImplementedType:
1843 logger.trace(pickler, "T7: %s", obj)
1844 pickler.save_reduce(type, (NotImplemented,), obj=obj)
1845 logger.trace(pickler, "# T7")
1846 elif obj is EllipsisType:
1847 logger.trace(pickler, "T7: %s", obj)
1848 pickler.save_reduce(type, (Ellipsis,), obj=obj)
1849 logger.trace(pickler, "# T7")
1850 elif obj is EnumMeta:
1851 logger.trace(pickler, "T7: %s", obj)
1852 pickler.write(GLOBAL + b'enum\nEnumMeta\n')
1853 logger.trace(pickler, "# T7")
1854 elif obj is ExceptHookArgsType: #NOTE: must be after NoneType for pypy
1855 logger.trace(pickler, "T7: %s", obj)
1856 pickler.write(GLOBAL + b'threading\nExceptHookArgs\n')
1857 logger.trace(pickler, "# T7")
1859 else:
1860 _byref = getattr(pickler, '_byref', None)
1861 obj_recursive = id(obj) in getattr(pickler, '_postproc', ())
1862 incorrectly_named = not _locate_function(obj, pickler)
1863 if not _byref and not obj_recursive and incorrectly_named: # not a function, but the name was held over
1864 if postproc_list is None:
1865 postproc_list = []
1867 # thanks to Tom Stepleton pointing out pickler._session unneeded
1868 logger.trace(pickler, "T2: %s", obj)
1869 _dict, attrs = _get_typedict_type(obj, obj.__dict__.copy(), None, postproc_list) # copy dict proxy to a dict
1871 #print (_dict)
1872 #print ("%s\n%s" % (type(obj), obj.__name__))
1873 #print ("%s\n%s" % (obj.__bases__, obj.__dict__))
1874 slots = _dict.get('__slots__', ())
1875 if type(slots) == str:
1876 # __slots__ accepts a single string
1877 slots = (slots,)
1879 for name in slots:
1880 _dict.pop(name, None)
1882 if isinstance(obj, abc.ABCMeta):
1883 logger.trace(pickler, "ABC: %s", obj)
1884 _dict, attrs = _get_typedict_abc(obj, _dict, attrs, postproc_list)
1885 logger.trace(pickler, "# ABC")
1887 qualname = getattr(obj, '__qualname__', None)
1888 if attrs is not None:
1889 for k, v in attrs.items():
1890 postproc_list.append((setattr, (obj, k, v)))
1891 # TODO: Consider using the state argument to save_reduce?
1892 if qualname is not None:
1893 postproc_list.append((setattr, (obj, '__qualname__', qualname)))
1895 if not hasattr(obj, '__orig_bases__'):
1896 _save_with_postproc(pickler, (_create_type, (
1897 type(obj), obj.__name__, obj.__bases__, _dict
1898 )), obj=obj, postproc_list=postproc_list)
1899 else:
1900 # This case will always work, but might be overkill.
1901 _metadict = {
1902 'metaclass': type(obj)
1903 }
1905 if _dict:
1906 _dict_update = PartialType(_setitems, source=_dict)
1907 else:
1908 _dict_update = None
1910 _save_with_postproc(pickler, (new_class, (
1911 obj.__name__, obj.__orig_bases__, _metadict, _dict_update
1912 )), obj=obj, postproc_list=postproc_list)
1913 logger.trace(pickler, "# T2")
1914 else:
1915 obj_name = getattr(obj, '__qualname__', getattr(obj, '__name__', None))
1916 logger.trace(pickler, "T4: %s", obj)
1917 if incorrectly_named:
1918 warnings.warn(
1919 "Cannot locate reference to %r." % (obj,),
1920 PicklingWarning,
1921 stacklevel=3,
1922 )
1923 if obj_recursive:
1924 warnings.warn(
1925 "Cannot pickle %r: %s.%s has recursive self-references that "
1926 "trigger a RecursionError." % (obj, obj.__module__, obj_name),
1927 PicklingWarning,
1928 stacklevel=3,
1929 )
1930 #print (obj.__dict__)
1931 #print ("%s\n%s" % (type(obj), obj.__name__))
1932 #print ("%s\n%s" % (obj.__bases__, obj.__dict__))
1933 StockPickler.save_global(pickler, obj, name=obj_name)
1934 logger.trace(pickler, "# T4")
1935 return
1937@register(property)
1938@register(abc.abstractproperty)
1939def save_property(pickler, obj):
1940 logger.trace(pickler, "Pr: %s", obj)
1941 pickler.save_reduce(type(obj), (obj.fget, obj.fset, obj.fdel, obj.__doc__),
1942 obj=obj)
1943 logger.trace(pickler, "# Pr")
1945@register(staticmethod)
1946@register(classmethod)
1947@register(abc.abstractstaticmethod)
1948@register(abc.abstractclassmethod)
1949def save_classmethod(pickler, obj):
1950 logger.trace(pickler, "Cm: %s", obj)
1951 orig_func = obj.__func__
1953 # if type(obj.__dict__) is dict:
1954 # if obj.__dict__:
1955 # state = obj.__dict__
1956 # else:
1957 # state = None
1958 # else:
1959 # state = (None, {'__dict__', obj.__dict__})
1961 pickler.save_reduce(type(obj), (orig_func,), obj=obj)
1962 logger.trace(pickler, "# Cm")
1964@register(FunctionType)
1965def save_function(pickler, obj):
1966 if not _locate_function(obj, pickler):
1967 if type(obj.__code__) is not CodeType:
1968 # Some PyPy builtin functions have no module name, and thus are not
1969 # able to be located
1970 module_name = getattr(obj, '__module__', None)
1971 if module_name is None:
1972 module_name = __builtin__.__name__
1973 module = _import_module(module_name, safe=True)
1974 _pypy_builtin = False
1975 try:
1976 found, _ = _getattribute(module, obj.__qualname__)
1977 if getattr(found, '__func__', None) is obj:
1978 _pypy_builtin = True
1979 except AttributeError:
1980 pass
1982 if _pypy_builtin:
1983 logger.trace(pickler, "F3: %s", obj)
1984 pickler.save_reduce(getattr, (found, '__func__'), obj=obj)
1985 logger.trace(pickler, "# F3")
1986 return
1988 logger.trace(pickler, "F1: %s", obj)
1989 _recurse = getattr(pickler, '_recurse', None)
1990 _postproc = getattr(pickler, '_postproc', None)
1991 _main_modified = getattr(pickler, '_main_modified', None)
1992 _original_main = getattr(pickler, '_original_main', __builtin__)#'None'
1993 postproc_list = []
1994 if _recurse:
1995 # recurse to get all globals referred to by obj
1996 from .detect import globalvars
1997 globs_copy = globalvars(obj, recurse=True, builtin=True)
1999 # Add the name of the module to the globs dictionary to prevent
2000 # the duplication of the dictionary. Pickle the unpopulated
2001 # globals dictionary and set the remaining items after the function
2002 # is created to correctly handle recursion.
2003 globs = {'__name__': obj.__module__}
2004 else:
2005 globs_copy = obj.__globals__
2007 # If the globals is the __dict__ from the module being saved as a
2008 # session, substitute it by the dictionary being actually saved.
2009 if _main_modified and globs_copy is _original_main.__dict__:
2010 globs_copy = getattr(pickler, '_main', _original_main).__dict__
2011 globs = globs_copy
2012 # If the globals is a module __dict__, do not save it in the pickle.
2013 elif globs_copy is not None and obj.__module__ is not None and \
2014 getattr(_import_module(obj.__module__, True), '__dict__', None) is globs_copy:
2015 globs = globs_copy
2016 else:
2017 globs = {'__name__': obj.__module__}
2019 if globs_copy is not None and globs is not globs_copy:
2020 # In the case that the globals are copied, we need to ensure that
2021 # the globals dictionary is updated when all objects in the
2022 # dictionary are already created.
2023 glob_ids = {id(g) for g in globs_copy.values()}
2024 for stack_element in _postproc:
2025 if stack_element in glob_ids:
2026 _postproc[stack_element].append((_setitems, (globs, globs_copy)))
2027 break
2028 else:
2029 postproc_list.append((_setitems, (globs, globs_copy)))
2031 closure = obj.__closure__
2032 state_dict = {}
2033 for fattrname in ('__doc__', '__kwdefaults__', '__annotations__'):
2034 fattr = getattr(obj, fattrname, None)
2035 if fattr is not None:
2036 state_dict[fattrname] = fattr
2037 if obj.__qualname__ != obj.__name__:
2038 state_dict['__qualname__'] = obj.__qualname__
2039 if '__name__' not in globs or obj.__module__ != globs['__name__']:
2040 state_dict['__module__'] = obj.__module__
2042 state = obj.__dict__
2043 if type(state) is not dict:
2044 state_dict['__dict__'] = state
2045 state = None
2046 if state_dict:
2047 state = state, state_dict
2049 _save_with_postproc(pickler, (_create_function, (
2050 obj.__code__, globs, obj.__name__, obj.__defaults__,
2051 closure
2052 ), state), obj=obj, postproc_list=postproc_list)
2054 # Lift closure cell update to earliest function (#458)
2055 if _postproc:
2056 topmost_postproc = next(iter(_postproc.values()), None)
2057 if closure and topmost_postproc:
2058 for cell in closure:
2059 possible_postproc = (setattr, (cell, 'cell_contents', obj))
2060 try:
2061 topmost_postproc.remove(possible_postproc)
2062 except ValueError:
2063 continue
2065 # Change the value of the cell
2066 pickler.save_reduce(*possible_postproc)
2067 # pop None created by calling preprocessing step off stack
2068 pickler.write(POP)
2070 logger.trace(pickler, "# F1")
2071 else:
2072 logger.trace(pickler, "F2: %s", obj)
2073 name = getattr(obj, '__qualname__', getattr(obj, '__name__', None))
2074 StockPickler.save_global(pickler, obj, name=name)
2075 logger.trace(pickler, "# F2")
2076 return
2078if HAS_CTYPES and hasattr(ctypes, 'pythonapi'):
2079 _PyCapsule_New = ctypes.pythonapi.PyCapsule_New
2080 _PyCapsule_New.argtypes = (ctypes.c_void_p, ctypes.c_char_p, ctypes.c_void_p)
2081 _PyCapsule_New.restype = ctypes.py_object
2082 _PyCapsule_GetPointer = ctypes.pythonapi.PyCapsule_GetPointer
2083 _PyCapsule_GetPointer.argtypes = (ctypes.py_object, ctypes.c_char_p)
2084 _PyCapsule_GetPointer.restype = ctypes.c_void_p
2085 _PyCapsule_GetDestructor = ctypes.pythonapi.PyCapsule_GetDestructor
2086 _PyCapsule_GetDestructor.argtypes = (ctypes.py_object,)
2087 _PyCapsule_GetDestructor.restype = ctypes.c_void_p
2088 _PyCapsule_GetContext = ctypes.pythonapi.PyCapsule_GetContext
2089 _PyCapsule_GetContext.argtypes = (ctypes.py_object,)
2090 _PyCapsule_GetContext.restype = ctypes.c_void_p
2091 _PyCapsule_GetName = ctypes.pythonapi.PyCapsule_GetName
2092 _PyCapsule_GetName.argtypes = (ctypes.py_object,)
2093 _PyCapsule_GetName.restype = ctypes.c_char_p
2094 _PyCapsule_IsValid = ctypes.pythonapi.PyCapsule_IsValid
2095 _PyCapsule_IsValid.argtypes = (ctypes.py_object, ctypes.c_char_p)
2096 _PyCapsule_IsValid.restype = ctypes.c_bool
2097 _PyCapsule_SetContext = ctypes.pythonapi.PyCapsule_SetContext
2098 _PyCapsule_SetContext.argtypes = (ctypes.py_object, ctypes.c_void_p)
2099 _PyCapsule_SetDestructor = ctypes.pythonapi.PyCapsule_SetDestructor
2100 _PyCapsule_SetDestructor.argtypes = (ctypes.py_object, ctypes.c_void_p)
2101 _PyCapsule_SetName = ctypes.pythonapi.PyCapsule_SetName
2102 _PyCapsule_SetName.argtypes = (ctypes.py_object, ctypes.c_char_p)
2103 _PyCapsule_SetPointer = ctypes.pythonapi.PyCapsule_SetPointer
2104 _PyCapsule_SetPointer.argtypes = (ctypes.py_object, ctypes.c_void_p)
2105 #from _socket import CAPI as _testcapsule
2106 _testcapsule_name = b'dill._dill._testcapsule'
2107 _testcapsule = _PyCapsule_New(
2108 ctypes.cast(_PyCapsule_New, ctypes.c_void_p),
2109 ctypes.c_char_p(_testcapsule_name),
2110 None
2111 )
2112 PyCapsuleType = type(_testcapsule)
2113 @register(PyCapsuleType)
2114 def save_capsule(pickler, obj):
2115 logger.trace(pickler, "Cap: %s", obj)
2116 name = _PyCapsule_GetName(obj)
2117 #warnings.warn('Pickling a PyCapsule (%s) does not pickle any C data structures and could cause segmentation faults or other memory errors when unpickling.' % (name,), PicklingWarning)
2118 pointer = _PyCapsule_GetPointer(obj, name)
2119 context = _PyCapsule_GetContext(obj)
2120 destructor = _PyCapsule_GetDestructor(obj)
2121 pickler.save_reduce(_create_capsule, (pointer, name, context, destructor), obj=obj)
2122 logger.trace(pickler, "# Cap")
2123 _incedental_reverse_typemap['PyCapsuleType'] = PyCapsuleType
2124 _reverse_typemap['PyCapsuleType'] = PyCapsuleType
2125 _incedental_types.add(PyCapsuleType)
2126else:
2127 _testcapsule = None
2129@register(ContextType)
2130def save_context(pickler, obj):
2131 logger.trace(pickler, "Cx: %s", obj)
2132 pickler.save_reduce(ContextType, tuple(obj.items()), obj=obj)
2133 logger.trace(pickler, "# Cx")
2136#############################
2137# A quick fix for issue #500
2138# This should be removed when a better solution is found.
2140if hasattr(dataclasses, "_HAS_DEFAULT_FACTORY_CLASS"):
2141 @register(dataclasses._HAS_DEFAULT_FACTORY_CLASS)
2142 def save_dataclasses_HAS_DEFAULT_FACTORY_CLASS(pickler, obj):
2143 logger.trace(pickler, "DcHDF: %s", obj)
2144 pickler.write(GLOBAL + b"dataclasses\n_HAS_DEFAULT_FACTORY\n")
2145 logger.trace(pickler, "# DcHDF")
2147if hasattr(dataclasses, "MISSING"):
2148 @register(type(dataclasses.MISSING))
2149 def save_dataclasses_MISSING_TYPE(pickler, obj):
2150 logger.trace(pickler, "DcM: %s", obj)
2151 pickler.write(GLOBAL + b"dataclasses\nMISSING\n")
2152 logger.trace(pickler, "# DcM")
2154if hasattr(dataclasses, "KW_ONLY"):
2155 @register(type(dataclasses.KW_ONLY))
2156 def save_dataclasses_KW_ONLY_TYPE(pickler, obj):
2157 logger.trace(pickler, "DcKWO: %s", obj)
2158 pickler.write(GLOBAL + b"dataclasses\nKW_ONLY\n")
2159 logger.trace(pickler, "# DcKWO")
2161if hasattr(dataclasses, "_FIELD_BASE"):
2162 @register(dataclasses._FIELD_BASE)
2163 def save_dataclasses_FIELD_BASE(pickler, obj):
2164 logger.trace(pickler, "DcFB: %s", obj)
2165 pickler.write(GLOBAL + b"dataclasses\n" + obj.name.encode() + b"\n")
2166 logger.trace(pickler, "# DcFB")
2168#############################
2170# quick sanity checking
2171def pickles(obj,exact=False,safe=False,**kwds):
2172 """
2173 Quick check if object pickles with dill.
2175 If *exact=True* then an equality test is done to check if the reconstructed
2176 object matches the original object.
2178 If *safe=True* then any exception will raised in copy signal that the
2179 object is not picklable, otherwise only pickling errors will be trapped.
2181 Additional keyword arguments are as :func:`dumps` and :func:`loads`.
2182 """
2183 if safe: exceptions = (Exception,) # RuntimeError, ValueError
2184 else:
2185 exceptions = (TypeError, AssertionError, NotImplementedError, PicklingError, UnpicklingError)
2186 try:
2187 pik = copy(obj, **kwds)
2188 #FIXME: should check types match first, then check content if "exact"
2189 try:
2190 #FIXME: should be "(pik == obj).all()" for numpy comparison, though that'll fail if shapes differ
2191 result = bool(pik.all() == obj.all())
2192 except (AttributeError, TypeError):
2193 warnings.filterwarnings('ignore') #FIXME: be specific
2194 result = pik == obj
2195 if warnings.filters: del warnings.filters[0]
2196 if hasattr(result, 'toarray'): # for unusual types like sparse matrix
2197 result = result.toarray().all()
2198 if result: return True
2199 if not exact:
2200 result = type(pik) == type(obj)
2201 if result: return result
2202 # class instances might have been dumped with byref=False
2203 return repr(type(pik)) == repr(type(obj)) #XXX: InstanceType?
2204 return False
2205 except exceptions:
2206 return False
2208def check(obj, *args, **kwds):
2209 """
2210 Check pickling of an object across another process.
2212 *python* is the path to the python interpreter (defaults to sys.executable)
2214 Set *verbose=True* to print the unpickled object in the other process.
2216 Additional keyword arguments are as :func:`dumps` and :func:`loads`.
2217 """
2218 # == undocumented ==
2219 # python -- the string path or executable name of the selected python
2220 # verbose -- if True, be verbose about printing warning messages
2221 # all other args and kwds are passed to dill.dumps #FIXME: ignore on load
2222 verbose = kwds.pop('verbose', False)
2223 python = kwds.pop('python', None)
2224 if python is None:
2225 import sys
2226 python = sys.executable
2227 # type check
2228 isinstance(python, str)
2229 import subprocess
2230 fail = True
2231 try:
2232 _obj = dumps(obj, *args, **kwds)
2233 fail = False
2234 finally:
2235 if fail and verbose:
2236 print("DUMP FAILED")
2237 #FIXME: fails if python interpreter path contains spaces
2238 # Use the following instead (which also processes the 'ignore' keyword):
2239 # ignore = kwds.pop('ignore', None)
2240 # unpickle = "dill.loads(%s, ignore=%s)"%(repr(_obj), repr(ignore))
2241 # cmd = [python, "-c", "import dill; print(%s)"%unpickle]
2242 # msg = "SUCCESS" if not subprocess.call(cmd) else "LOAD FAILED"
2243 msg = "%s -c import dill; print(dill.loads(%s))" % (python, repr(_obj))
2244 msg = "SUCCESS" if not subprocess.call(msg.split(None,2)) else "LOAD FAILED"
2245 if verbose:
2246 print(msg)
2247 return
2249# use to protect against missing attributes
2250def is_dill(pickler, child=None):
2251 "check the dill-ness of your pickler"
2252 if child is False or not hasattr(pickler.__class__, 'mro'):
2253 return 'dill' in pickler.__module__
2254 return Pickler in pickler.__class__.mro()
2256def _extend():
2257 """extend pickle with all of dill's registered types"""
2258 # need to have pickle not choke on _main_module? use is_dill(pickler)
2259 for t,func in Pickler.dispatch.items():
2260 try:
2261 StockPickler.dispatch[t] = func
2262 except Exception: #TypeError, PicklingError, UnpicklingError
2263 logger.trace(pickler, "skip: %s", t)
2264 return
2266del diff, _use_diff, use_diff
2268# EOF