Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/dill/detect.py: 11%
179 statements
« prev ^ index » next coverage.py v7.0.1, created at 2022-12-25 06:11 +0000
1#!/usr/bin/env python
2#
3# Author: Mike McKerns (mmckerns @caltech and @uqfoundation)
4# Copyright (c) 2008-2016 California Institute of Technology.
5# Copyright (c) 2016-2022 The Uncertainty Quantification Foundation.
6# License: 3-clause BSD. The full license text is available at:
7# - https://github.com/uqfoundation/dill/blob/master/LICENSE
8"""
9Methods for detecting objects leading to pickling failures.
10"""
12import dis
13from inspect import ismethod, isfunction, istraceback, isframe, iscode
15from .pointers import parent, reference, at, parents, children
16from .logger import trace
18__all__ = ['baditems','badobjects','badtypes','code','errors','freevars',
19 'getmodule','globalvars','nestedcode','nestedglobals','outermost',
20 'referredglobals','referrednested','trace','varnames']
22def getmodule(object, _filename=None, force=False):
23 """get the module of the object"""
24 from inspect import getmodule as getmod
25 module = getmod(object, _filename)
26 if module or not force: return module
27 import builtins
28 from .source import getname
29 name = getname(object, force=True)
30 return builtins if name in vars(builtins).keys() else None
32def outermost(func): # is analogous to getsource(func,enclosing=True)
33 """get outermost enclosing object (i.e. the outer function in a closure)
35 NOTE: this is the object-equivalent of getsource(func, enclosing=True)
36 """
37 if ismethod(func):
38 _globals = func.__func__.__globals__ or {}
39 elif isfunction(func):
40 _globals = func.__globals__ or {}
41 else:
42 return #XXX: or raise? no matches
43 _globals = _globals.items()
44 # get the enclosing source
45 from .source import getsourcelines
46 try: lines,lnum = getsourcelines(func, enclosing=True)
47 except Exception: #TypeError, IOError
48 lines,lnum = [],None
49 code = ''.join(lines)
50 # get all possible names,objects that are named in the enclosing source
51 _locals = ((name,obj) for (name,obj) in _globals if name in code)
52 # now only save the objects that generate the enclosing block
53 for name,obj in _locals: #XXX: don't really need 'name'
54 try:
55 if getsourcelines(obj) == (lines,lnum): return obj
56 except Exception: #TypeError, IOError
57 pass
58 return #XXX: or raise? no matches
60def nestedcode(func, recurse=True): #XXX: or return dict of {co_name: co} ?
61 """get the code objects for any nested functions (e.g. in a closure)"""
62 func = code(func)
63 if not iscode(func): return [] #XXX: or raise? no matches
64 nested = set()
65 for co in func.co_consts:
66 if co is None: continue
67 co = code(co)
68 if co:
69 nested.add(co)
70 if recurse: nested |= set(nestedcode(co, recurse=True))
71 return list(nested)
73def code(func):
74 """get the code object for the given function or method
76 NOTE: use dill.source.getsource(CODEOBJ) to get the source code
77 """
78 if ismethod(func): func = func.__func__
79 if isfunction(func): func = func.__code__
80 if istraceback(func): func = func.tb_frame
81 if isframe(func): func = func.f_code
82 if iscode(func): return func
83 return
85#XXX: ugly: parse dis.dis for name after "<code object" in line and in globals?
86def referrednested(func, recurse=True): #XXX: return dict of {__name__: obj} ?
87 """get functions defined inside of func (e.g. inner functions in a closure)
89 NOTE: results may differ if the function has been executed or not.
90 If len(nestedcode(func)) > len(referrednested(func)), try calling func().
91 If possible, python builds code objects, but delays building functions
92 until func() is called.
93 """
94 import gc
95 funcs = set()
96 # get the code objects, and try to track down by referrence
97 for co in nestedcode(func, recurse):
98 # look for function objects that refer to the code object
99 for obj in gc.get_referrers(co):
100 # get methods
101 _ = getattr(obj, '__func__', None) # ismethod
102 if getattr(_, '__code__', None) is co: funcs.add(obj)
103 # get functions
104 elif getattr(obj, '__code__', None) is co: funcs.add(obj)
105 # get frame objects
106 elif getattr(obj, 'f_code', None) is co: funcs.add(obj)
107 # get code objects
108 elif hasattr(obj, 'co_code') and obj is co: funcs.add(obj)
109# frameobjs => func.__code__.co_varnames not in func.__code__.co_cellvars
110# funcobjs => func.__code__.co_cellvars not in func.__code__.co_varnames
111# frameobjs are not found, however funcobjs are...
112# (see: test_mixins.quad ... and test_mixins.wtf)
113# after execution, code objects get compiled, and then may be found by gc
114 return list(funcs)
117def freevars(func):
118 """get objects defined in enclosing code that are referred to by func
120 returns a dict of {name:object}"""
121 if ismethod(func): func = func.__func__
122 if isfunction(func):
123 closures = func.__closure__ or ()
124 func = func.__code__.co_freevars # get freevars
125 else:
126 return {}
128 def get_cell_contents():
129 for name, c in zip(func, closures):
130 try:
131 cell_contents = c.cell_contents
132 except ValueError: # cell is empty
133 continue
134 yield name, c.cell_contents
136 return dict(get_cell_contents())
138# thanks to Davies Liu for recursion of globals
139def nestedglobals(func, recurse=True):
140 """get the names of any globals found within func"""
141 func = code(func)
142 if func is None: return list()
143 import sys
144 from .temp import capture
145 CAN_NULL = sys.hexversion >= 0x30b00a7 # NULL may be prepended >= 3.11a7
146 names = set()
147 with capture('stdout') as out:
148 dis.dis(func) #XXX: dis.dis(None) disassembles last traceback
149 for line in out.getvalue().splitlines():
150 if '_GLOBAL' in line:
151 name = line.split('(')[-1].split(')')[0]
152 if CAN_NULL:
153 names.add(name.replace('NULL + ', ''))
154 else:
155 names.add(name)
156 for co in getattr(func, 'co_consts', tuple()):
157 if co and recurse and iscode(co):
158 names.update(nestedglobals(co, recurse=True))
159 return list(names)
161def referredglobals(func, recurse=True, builtin=False):
162 """get the names of objects in the global scope referred to by func"""
163 return globalvars(func, recurse, builtin).keys()
165def globalvars(func, recurse=True, builtin=False):
166 """get objects defined in global scope that are referred to by func
168 return a dict of {name:object}"""
169 if ismethod(func): func = func.__func__
170 if isfunction(func):
171 globs = vars(getmodule(sum)).copy() if builtin else {}
172 # get references from within closure
173 orig_func, func = func, set()
174 for obj in orig_func.__closure__ or {}:
175 try:
176 cell_contents = obj.cell_contents
177 except ValueError: # cell is empty
178 pass
179 else:
180 _vars = globalvars(cell_contents, recurse, builtin) or {}
181 func.update(_vars) #XXX: (above) be wary of infinte recursion?
182 globs.update(_vars)
183 # get globals
184 globs.update(orig_func.__globals__ or {})
185 # get names of references
186 if not recurse:
187 func.update(orig_func.__code__.co_names)
188 else:
189 func.update(nestedglobals(orig_func.__code__))
190 # find globals for all entries of func
191 for key in func.copy(): #XXX: unnecessary...?
192 nested_func = globs.get(key)
193 if nested_func is orig_func:
194 #func.remove(key) if key in func else None
195 continue #XXX: globalvars(func, False)?
196 func.update(globalvars(nested_func, True, builtin))
197 elif iscode(func):
198 globs = vars(getmodule(sum)).copy() if builtin else {}
199 #globs.update(globals())
200 if not recurse:
201 func = func.co_names # get names
202 else:
203 orig_func = func.co_name # to stop infinite recursion
204 func = set(nestedglobals(func))
205 # find globals for all entries of func
206 for key in func.copy(): #XXX: unnecessary...?
207 if key is orig_func:
208 #func.remove(key) if key in func else None
209 continue #XXX: globalvars(func, False)?
210 nested_func = globs.get(key)
211 func.update(globalvars(nested_func, True, builtin))
212 else:
213 return {}
214 #NOTE: if name not in __globals__, then we skip it...
215 return dict((name,globs[name]) for name in func if name in globs)
218def varnames(func):
219 """get names of variables defined by func
221 returns a tuple (local vars, local vars referrenced by nested functions)"""
222 func = code(func)
223 if not iscode(func):
224 return () #XXX: better ((),())? or None?
225 return func.co_varnames, func.co_cellvars
228def baditems(obj, exact=False, safe=False): #XXX: obj=globals() ?
229 """get items in object that fail to pickle"""
230 if not hasattr(obj,'__iter__'): # is not iterable
231 return [j for j in (badobjects(obj,0,exact,safe),) if j is not None]
232 obj = obj.values() if getattr(obj,'values',None) else obj
233 _obj = [] # can't use a set, as items may be unhashable
234 [_obj.append(badobjects(i,0,exact,safe)) for i in obj if i not in _obj]
235 return [j for j in _obj if j is not None]
238def badobjects(obj, depth=0, exact=False, safe=False):
239 """get objects that fail to pickle"""
240 from dill import pickles
241 if not depth:
242 if pickles(obj,exact,safe): return None
243 return obj
244 return dict(((attr, badobjects(getattr(obj,attr),depth-1,exact,safe)) \
245 for attr in dir(obj) if not pickles(getattr(obj,attr),exact,safe)))
247def badtypes(obj, depth=0, exact=False, safe=False):
248 """get types for objects that fail to pickle"""
249 from dill import pickles
250 if not depth:
251 if pickles(obj,exact,safe): return None
252 return type(obj)
253 return dict(((attr, badtypes(getattr(obj,attr),depth-1,exact,safe)) \
254 for attr in dir(obj) if not pickles(getattr(obj,attr),exact,safe)))
256def errors(obj, depth=0, exact=False, safe=False):
257 """get errors for objects that fail to pickle"""
258 from dill import pickles, copy
259 if not depth:
260 try:
261 pik = copy(obj)
262 if exact:
263 assert pik == obj, \
264 "Unpickling produces %s instead of %s" % (pik,obj)
265 assert type(pik) == type(obj), \
266 "Unpickling produces %s instead of %s" % (type(pik),type(obj))
267 return None
268 except Exception:
269 import sys
270 return sys.exc_info()[1]
271 _dict = {}
272 for attr in dir(obj):
273 try:
274 _attr = getattr(obj,attr)
275 except Exception:
276 import sys
277 _dict[attr] = sys.exc_info()[1]
278 continue
279 if not pickles(_attr,exact,safe):
280 _dict[attr] = errors(_attr,depth-1,exact,safe)
281 return _dict
284# EOF