Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.10/site-packages/dill/detect.py: 11%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#!/usr/bin/env python
2#
3# Author: Mike McKerns (mmckerns @caltech and @uqfoundation)
4# Copyright (c) 2008-2016 California Institute of Technology.
5# Copyright (c) 2016-2025 The Uncertainty Quantification Foundation.
6# License: 3-clause BSD. The full license text is available at:
7# - https://github.com/uqfoundation/dill/blob/master/LICENSE
8"""
9Methods for detecting objects leading to pickling failures.
10"""
12import dis
13from inspect import ismethod, isfunction, istraceback, isframe, iscode
15from .pointers import parent, reference, at, parents, children
16from .logger import trace
18__all__ = ['baditems','badobjects','badtypes','code','errors','freevars',
19 'getmodule','globalvars','nestedcode','nestedglobals','outermost',
20 'referredglobals','referrednested','trace','varnames']
22def getmodule(object, _filename=None, force=False):
23 """get the module of the object"""
24 from inspect import getmodule as getmod
25 module = getmod(object, _filename)
26 if module or not force: return module
27 import builtins
28 from .source import getname
29 name = getname(object, force=True)
30 return builtins if name in vars(builtins).keys() else None
32def outermost(func): # is analogous to getsource(func,enclosing=True)
33 """get outermost enclosing object (i.e. the outer function in a closure)
35 NOTE: this is the object-equivalent of getsource(func, enclosing=True)
36 """
37 if ismethod(func):
38 _globals = func.__func__.__globals__ or {}
39 elif isfunction(func):
40 _globals = func.__globals__ or {}
41 else:
42 return #XXX: or raise? no matches
43 _globals = _globals.items()
44 # get the enclosing source
45 from .source import getsourcelines
46 try: lines,lnum = getsourcelines(func, enclosing=True)
47 except Exception: #TypeError, IOError
48 lines,lnum = [],None
49 code = ''.join(lines)
50 # get all possible names,objects that are named in the enclosing source
51 _locals = ((name,obj) for (name,obj) in _globals if name in code)
52 # now only save the objects that generate the enclosing block
53 for name,obj in _locals: #XXX: don't really need 'name'
54 try:
55 if getsourcelines(obj) == (lines,lnum): return obj
56 except Exception: #TypeError, IOError
57 pass
58 return #XXX: or raise? no matches
60def nestedcode(func, recurse=True): #XXX: or return dict of {co_name: co} ?
61 """get the code objects for any nested functions (e.g. in a closure)"""
62 func = code(func)
63 if not iscode(func): return [] #XXX: or raise? no matches
64 nested = set()
65 for co in func.co_consts:
66 if co is None: continue
67 co = code(co)
68 if co:
69 nested.add(co)
70 if recurse: nested |= set(nestedcode(co, recurse=True))
71 return list(nested)
73def code(func):
74 """get the code object for the given function or method
76 NOTE: use dill.source.getsource(CODEOBJ) to get the source code
77 """
78 if ismethod(func): func = func.__func__
79 if isfunction(func): func = func.__code__
80 if istraceback(func): func = func.tb_frame
81 if isframe(func): func = func.f_code
82 if iscode(func): return func
83 return
85#XXX: ugly: parse dis.dis for name after "<code object" in line and in globals?
86def referrednested(func, recurse=True): #XXX: return dict of {__name__: obj} ?
87 """get functions defined inside of func (e.g. inner functions in a closure)
89 NOTE: results may differ if the function has been executed or not.
90 If len(nestedcode(func)) > len(referrednested(func)), try calling func().
91 If possible, python builds code objects, but delays building functions
92 until func() is called.
93 """
94 import gc
95 funcs = set()
96 # get the code objects, and try to track down by referrence
97 for co in nestedcode(func, recurse):
98 # look for function objects that refer to the code object
99 for obj in gc.get_referrers(co):
100 # get methods
101 _ = getattr(obj, '__func__', None) # ismethod
102 if getattr(_, '__code__', None) is co: funcs.add(obj)
103 # get functions
104 elif getattr(obj, '__code__', None) is co: funcs.add(obj)
105 # get frame objects
106 elif getattr(obj, 'f_code', None) is co: funcs.add(obj)
107 # get code objects
108 elif hasattr(obj, 'co_code') and obj is co: funcs.add(obj)
109# frameobjs => func.__code__.co_varnames not in func.__code__.co_cellvars
110# funcobjs => func.__code__.co_cellvars not in func.__code__.co_varnames
111# frameobjs are not found, however funcobjs are...
112# (see: test_mixins.quad ... and test_mixins.wtf)
113# after execution, code objects get compiled, and then may be found by gc
114 return list(funcs)
117def freevars(func):
118 """get objects defined in enclosing code that are referred to by func
120 returns a dict of {name:object}"""
121 if ismethod(func): func = func.__func__
122 if isfunction(func):
123 closures = func.__closure__ or ()
124 func = func.__code__.co_freevars # get freevars
125 else:
126 return {}
128 def get_cell_contents():
129 for name, c in zip(func, closures):
130 try:
131 cell_contents = c.cell_contents
132 except ValueError: # cell is empty
133 continue
134 yield name, c.cell_contents
136 return dict(get_cell_contents())
138# thanks to Davies Liu for recursion of globals
139def nestedglobals(func, recurse=True):
140 """get the names of any globals found within func"""
141 func = code(func)
142 if func is None: return list()
143 import sys
144 from .temp import capture
145 CAN_NULL = sys.hexversion >= 0x30b00a7 # NULL may be prepended >= 3.11a7
146 names = set()
147 with capture('stdout') as out:
148 try:
149 dis.dis(func) #XXX: dis.dis(None) disassembles last traceback
150 except IndexError:
151 pass #FIXME: HACK for IS_PYPY (3.11)
152 for line in out.getvalue().splitlines():
153 if '_GLOBAL' in line:
154 name = line.split('(')[-1].split(')')[0]
155 if CAN_NULL:
156 names.add(name.replace('NULL + ', '').replace(' + NULL', ''))
157 else:
158 names.add(name)
159 for co in getattr(func, 'co_consts', tuple()):
160 if co and recurse and iscode(co):
161 names.update(nestedglobals(co, recurse=True))
162 return list(names)
164def referredglobals(func, recurse=True, builtin=False):
165 """get the names of objects in the global scope referred to by func"""
166 return globalvars(func, recurse, builtin).keys()
168def globalvars(func, recurse=True, builtin=False):
169 """get objects defined in global scope that are referred to by func
171 return a dict of {name:object}"""
172 if ismethod(func): func = func.__func__
173 if isfunction(func):
174 globs = vars(getmodule(sum)).copy() if builtin else {}
175 # get references from within closure
176 orig_func, func = func, set()
177 for obj in orig_func.__closure__ or {}:
178 try:
179 cell_contents = obj.cell_contents
180 except ValueError: # cell is empty
181 pass
182 else:
183 _vars = globalvars(cell_contents, recurse, builtin) or {}
184 func.update(_vars) #XXX: (above) be wary of infinte recursion?
185 globs.update(_vars)
186 # get globals
187 globs.update(orig_func.__globals__ or {})
188 # get names of references
189 if not recurse:
190 func.update(orig_func.__code__.co_names)
191 else:
192 func.update(nestedglobals(orig_func.__code__))
193 # find globals for all entries of func
194 for key in func.copy(): #XXX: unnecessary...?
195 nested_func = globs.get(key)
196 if nested_func is orig_func:
197 #func.remove(key) if key in func else None
198 continue #XXX: globalvars(func, False)?
199 func.update(globalvars(nested_func, True, builtin))
200 elif iscode(func):
201 globs = vars(getmodule(sum)).copy() if builtin else {}
202 #globs.update(globals())
203 if not recurse:
204 func = func.co_names # get names
205 else:
206 orig_func = func.co_name # to stop infinite recursion
207 func = set(nestedglobals(func))
208 # find globals for all entries of func
209 for key in func.copy(): #XXX: unnecessary...?
210 if key is orig_func:
211 #func.remove(key) if key in func else None
212 continue #XXX: globalvars(func, False)?
213 nested_func = globs.get(key)
214 func.update(globalvars(nested_func, True, builtin))
215 else:
216 return {}
217 #NOTE: if name not in __globals__, then we skip it...
218 return dict((name,globs[name]) for name in func if name in globs)
221def varnames(func):
222 """get names of variables defined by func
224 returns a tuple (local vars, local vars referrenced by nested functions)"""
225 func = code(func)
226 if not iscode(func):
227 return () #XXX: better ((),())? or None?
228 return func.co_varnames, func.co_cellvars
231def baditems(obj, exact=False, safe=False): #XXX: obj=globals() ?
232 """get items in object that fail to pickle"""
233 if not hasattr(obj,'__iter__'): # is not iterable
234 return [j for j in (badobjects(obj,0,exact,safe),) if j is not None]
235 obj = obj.values() if getattr(obj,'values',None) else obj
236 _obj = [] # can't use a set, as items may be unhashable
237 [_obj.append(badobjects(i,0,exact,safe)) for i in obj if i not in _obj]
238 return [j for j in _obj if j is not None]
241def badobjects(obj, depth=0, exact=False, safe=False):
242 """get objects that fail to pickle"""
243 from dill import pickles
244 if not depth:
245 if pickles(obj,exact,safe): return None
246 return obj
247 return dict(((attr, badobjects(getattr(obj,attr),depth-1,exact,safe)) \
248 for attr in dir(obj) if not pickles(getattr(obj,attr),exact,safe)))
250def badtypes(obj, depth=0, exact=False, safe=False):
251 """get types for objects that fail to pickle"""
252 from dill import pickles
253 if not depth:
254 if pickles(obj,exact,safe): return None
255 return type(obj)
256 return dict(((attr, badtypes(getattr(obj,attr),depth-1,exact,safe)) \
257 for attr in dir(obj) if not pickles(getattr(obj,attr),exact,safe)))
259def errors(obj, depth=0, exact=False, safe=False):
260 """get errors for objects that fail to pickle"""
261 from dill import pickles, copy
262 if not depth:
263 try:
264 pik = copy(obj)
265 if exact:
266 assert pik == obj, \
267 "Unpickling produces %s instead of %s" % (pik,obj)
268 assert type(pik) == type(obj), \
269 "Unpickling produces %s instead of %s" % (type(pik),type(obj))
270 return None
271 except Exception:
272 import sys
273 return sys.exc_info()[1]
274 _dict = {}
275 for attr in dir(obj):
276 try:
277 _attr = getattr(obj,attr)
278 except Exception:
279 import sys
280 _dict[attr] = sys.exc_info()[1]
281 continue
282 if not pickles(_attr,exact,safe):
283 _dict[attr] = errors(_attr,depth-1,exact,safe)
284 return _dict
287# EOF