Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.10/site-packages/dill/source.py: 6%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#!/usr/bin/env python
2#
3# Author: Mike McKerns (mmckerns @caltech and @uqfoundation)
4# Copyright (c) 2008-2016 California Institute of Technology.
5# Copyright (c) 2016-2026 The Uncertainty Quantification Foundation.
6# License: 3-clause BSD. The full license text is available at:
7# - https://github.com/uqfoundation/dill/blob/master/LICENSE
8#
9# inspired by inspect.py from Python-2.7.6
10# inspect.py author: 'Ka-Ping Yee <ping@lfw.org>'
11# inspect.py merged into original dill.source by Mike McKerns 4/13/14
12"""
13Extensions to python's 'inspect' module, which can be used
14to retrieve information from live python objects. The methods
15defined in this module are augmented to facilitate access to
16source code of interactively defined functions and classes,
17as well as provide access to source code for objects defined
18in a file.
19"""
21__all__ = ['findsource', 'getsourcelines', 'getsource', 'indent', 'outdent', \
22 '_wrap', 'dumpsource', 'getname', '_namespace', 'getimport', \
23 '_importable', 'importable','isdynamic', 'isfrommain']
25import linecache
26import re
27from inspect import (getblock, getfile, getmodule, getsourcefile, indentsize,
28 isbuiltin, isclass, iscode, isframe, isfunction, ismethod,
29 ismodule, istraceback)
30from tokenize import TokenError
32from ._dill import IS_IPYTHON
35def isfrommain(obj):
36 "check if object was built in __main__"
37 module = getmodule(obj)
38 if module and module.__name__ == '__main__':
39 return True
40 return False
43def _is_repl_filename(file):
44 """True for interpreter buffers, including code.InteractiveConsole and IPython."""
45 if not file:
46 return True
47 name = file.replace('\\', '/').rsplit('/', 1)[-1]
48 return (
49 name in ('<stdin>', '<console>')
50 or name.startswith('<python-input-')
51 or name.startswith('<ipython-input-')
52 )
55def isdynamic(obj):
56 "check if object was built in the interpreter"
57 try: file = getfile(obj)
58 except TypeError: file = None
59 if _is_repl_filename(file) and isfrommain(obj):
60 return True
61 return False
64def _lambdacode(lhs, rhs):
65 """get the code object for 'lambda lhs : rhs', or None if it won't compile
67 the lambda is compiled instead of evaluated, so that nothing taken from
68 the raw line of code runs (e.g. a default argument expression)"""
69 try:
70 code = compile("lambda %s : %s" % (lhs,rhs), '<string>', 'eval')
71 except Exception:
72 return None
73 for const in code.co_consts:
74 if iscode(const): return const
75 return None
78def _matchlambda(func, line):
79 """check if lambda object 'func' matches raw line of code 'line'"""
80 from .detect import code as getcode
81 from .detect import freevars, globalvars, varnames
82 dummy = lambda : '__this_is_a_big_dummy_function__'
83 # process the line (removing leading whitespace, etc)
84 lhs,rhs = line.split('lambda ',1)[-1].split(":", 1) #FIXME: if !1 inputs
85 _ = _lambdacode(lhs,rhs)
86 if _ is None: _ = dummy
87 # get code objects, for comparison
88 _, code = getcode(_).co_code, getcode(func).co_code
89 # check if func is in closure
90 _f = [line.count(i) for i in freevars(func).keys()]
91 if not _f: # not in closure
92 # check if code matches
93 if _ == code: return True
94 return False
95 # weak check on freevars
96 if not all(_f): return False #XXX: VERY WEAK
97 # weak check on varnames and globalvars
98 _f = varnames(func)
99 _f = [line.count(i) for i in _f[0]+_f[1]]
100 if _f and not all(_f): return False #XXX: VERY WEAK
101 _f = [line.count(i) for i in globalvars(func).keys()]
102 if _f and not all(_f): return False #XXX: VERY WEAK
103 # check if func is a double lambda
104 if (line.count('lambda ') > 1) and (lhs in freevars(func).keys()):
105 _lhs,_rhs = rhs.split('lambda ',1)[-1].split(":",1) #FIXME: if !1 inputs
106 _f = _lambdacode(_lhs,_rhs)
107 if _f is None: _f = dummy
108 # get code objects, for comparison
109 _, code = getcode(_f).co_code, getcode(func).co_code
110 if len(_) != len(code): return False
111 #NOTE: should be same code same order, but except for 't' and '\x88'
112 _ = set((i,j) for (i,j) in zip(_,code) if i != j)
113 if len(_) != 1: return False #('t','\x88')
114 return True
115 # check indentsize
116 if not indentsize(line): return False #FIXME: is this a good check???
117 # check if code 'pattern' matches
118 #XXX: or pattern match against dis.dis(code)? (or use uncompyle2?)
119 _ = _.split(_[0]) # 't' #XXX: remove matching values if starts the same?
120 _f = code.split(code[0]) # '\x88'
121 #NOTE: should be same code different order, with different first element
122 _ = dict(re.match(r'([\W\D\S])(.*)', _[i]).groups() for i in range(1,len(_)))
123 _f = dict(re.match(r'([\W\D\S])(.*)', _f[i]).groups() for i in range(1,len(_f)))
124 if (_.keys() == _f.keys()) and (sorted(_.values()) == sorted(_f.values())):
125 return True
126 return False
129def findsource(object):
130 """Return the entire source file and starting line number for an object.
131 For interactively-defined objects, the 'file' is the interpreter's history.
133 The argument may be a module, class, method, function, traceback, frame,
134 or code object. The source code is returned as a list of all the lines
135 in the file and the line number indexes a line in that list. An IOError
136 is raised if the source code cannot be retrieved, while a TypeError is
137 raised for objects where the source code is unavailable (e.g. builtins)."""
139 module = getmodule(object)
140 try: file = getfile(module)
141 except TypeError: file = None
142 if file and file.endswith('_pyrepl/__main__.py'):
143 import sys #XXX: assumes only 3.14.0a0 > python > 3.12.99f0
144 file = None if (0x30e00a0 > sys.hexversion > 0x30c63f0) else file
145 is_module_main = module and module.__name__ == '__main__'
146 if is_module_main:
147 # Consoles such as python -m asyncio retain the bootstrap module's file.
148 # The callable's own filename still identifies its interactive source.
149 try:
150 is_module_main = (_is_repl_filename(file)
151 or _is_repl_filename(getfile(object)))
152 except TypeError:
153 is_module_main = _is_repl_filename(file)
154 if IS_IPYTHON and is_module_main:
155 #FIXME: quick fix for functions and classes in IPython interpreter
156 try:
157 file = getfile(object)
158 sourcefile = getsourcefile(object)
159 except TypeError:
160 if isclass(object):
161 for object_method in filter(isfunction, object.__dict__.values()):
162 # look for a method of the class
163 file_candidate = getfile(object_method)
164 if not file_candidate.startswith('<ipython-input-'):
165 continue
166 file = file_candidate
167 sourcefile = getsourcefile(object_method)
168 break
169 if file:
170 lines = linecache.getlines(file)
171 else:
172 # fallback to use history
173 history = '\n'.join(get_ipython().history_manager.input_hist_parsed)
174 lines = [line + '\n' for line in history.splitlines()]
175 # use readline when working in interpreter (i.e. __main__ and not file)
176 elif is_module_main:
177 try:
178 from _pyrepl.readline import _wrapper as readline
179 except ImportError:
180 try:
181 import readline
182 err = ''
183 except ImportError:
184 import sys
185 err = sys.exc_info()[1].args[0]
186 if sys.platform[:3] == 'win':
187 err += ", please install 'pyreadline'"
188 if err:
189 raise IOError(err)
190 lbuf = readline.get_current_history_length()
191 # PyREPL can store an entire multiline block in one history entry.
192 lines = [line+'\n' for i in range(1,lbuf+1)
193 for line in (readline.get_history_item(i)+'\n').split('\n')[:-1]]
194 else:
195 try: # special handling for class instances
196 if not isclass(object) and isclass(type(object)): # __class__
197 file = getfile(module)
198 sourcefile = getsourcefile(module)
199 else: # builtins fail with a TypeError
200 file = getfile(object)
201 sourcefile = getsourcefile(object)
202 except (TypeError, AttributeError): # fail with better error
203 file = getfile(object)
204 sourcefile = getsourcefile(object)
205 if not sourcefile and file[:1] + file[-1:] != '<>':
206 raise IOError('source code not available')
207 file = sourcefile if sourcefile else file
209 module = getmodule(object, file)
210 if module:
211 lines = linecache.getlines(file, module.__dict__)
212 else:
213 lines = linecache.getlines(file)
215 if not lines:
216 raise IOError('could not extract source code')
218 #FIXME: all below may fail if exec used (i.e. exec('f = lambda x:x') )
219 if ismodule(object):
220 return lines, 0
222 #NOTE: beneficial if search goes from end to start of buffer history
223 name = pat1 = obj = ''
224 pat2 = r'^(\s*@)'
225# pat1b = r'^(\s*%s\W*=)' % name #FIXME: finds 'f = decorate(f)', not exec
226 if ismethod(object):
227 name = object.__name__
228 if name == '<lambda>': pat1 = r'(.*(?<!\w)lambda(:|\s))'
229 else: pat1 = r'^(\s*(?:async\s+)?def\s)'
230 object = object.__func__
231 if isfunction(object):
232 name = object.__name__
233 if name == '<lambda>':
234 pat1 = r'(.*(?<!\w)lambda(:|\s))'
235 obj = object #XXX: better a copy?
236 else: pat1 = r'^(\s*(?:async\s+)?def\s)'
237 object = object.__code__
238 if istraceback(object):
239 object = object.tb_frame
240 if isframe(object):
241 object = object.f_code
242 if iscode(object):
243 if not hasattr(object, 'co_firstlineno'):
244 raise IOError('could not find function definition')
245 stdin = _is_repl_filename(object.co_filename)
246 if stdin:
247 if not name and object.co_name.isidentifier():
248 name = object.co_name
249 lnum = len(lines) - 1 # can't get lnum easily, so leverage pat
250 if not pat1: pat1 = r'^(\s*(?:async\s+)?def\s)|(.*(?<!\w)lambda(:|\s))|^(\s*@)'
251 else:
252 lnum = object.co_firstlineno - 1
253 pat1 = r'^(\s*(?:async\s+)?def\s)|(.*(?<!\w)lambda(:|\s))|^(\s*@)'
254 pat1 = re.compile(pat1); pat2 = re.compile(pat2)
255 #XXX: candidate_lnum = [n for n in range(lnum) if pat1.match(lines[n])]
256 while lnum > 0: #XXX: won't find decorators in <stdin> ?
257 line = lines[lnum]
258 if pat1.match(line):
259 if not stdin: break # co_firstlineno does the job
260 if name == '<lambda>': # hackery needed to confirm a match
261 if _matchlambda(obj, line): break
262 else: # not a lambda, just look for the name
263 definition = line
264 end = lnum
265 while definition.rstrip().endswith('\\') and end + 1 < len(lines):
266 end += 1
267 definition += lines[end]
268 if not name or re.search(r'^\s*(?:async\s+)?def(?:\s|\\\r?\n)+'
269 + re.escape(name) + r'(?=\s*[\[(\\])',
270 definition, re.M):
271 hats = 0
272 for _lnum in range(lnum-1,-1,-1):
273 if pat2.match(lines[_lnum]): hats += 1
274 else: break
275 lnum = lnum - hats
276 break
277 lnum = lnum - 1
278 return lines, lnum
280 try: # turn instances into classes
281 if not isclass(object) and isclass(type(object)): # __class__
282 object = object.__class__ #XXX: sometimes type(class) is better?
283 #XXX: we don't find how the instance was built
284 except AttributeError: pass
285 if isclass(object):
286 name = object.__name__
287 # __name__ is not required to be an identifier, so escape it to keep
288 # it a literal rather than letting it act as part of the pattern
289 pat = re.compile(r'^(\s*)class\s*' + re.escape(name) + r'\b')
290 # make some effort to find the best matching class definition:
291 # use the one with the least indentation, which is the one
292 # that's most probably not inside a function definition.
293 candidates = []
294 for i in range(len(lines)-1,-1,-1):
295 match = pat.match(lines[i])
296 if match:
297 # if it's at toplevel, it's already the best one
298 if lines[i][0] == 'c':
299 return lines, i
300 # else add whitespace to candidate list
301 candidates.append((match.group(1), i))
302 if candidates:
303 # this will sort by whitespace, and by line number,
304 # less whitespace first #XXX: should sort high lnum before low
305 candidates.sort()
306 return lines, candidates[0][1]
307 else:
308 raise IOError('could not find class definition')
309 raise IOError('could not find code object')
312def getblocks(object, lstrip=False, enclosing=False, locate=False):
313 """Return a list of source lines and starting line number for an object.
314 Interactively-defined objects refer to lines in the interpreter's history.
316 If enclosing=True, then also return any enclosing code.
317 If lstrip=True, ensure there is no indentation in the first line of code.
318 If locate=True, then also return the line number for the block of code.
320 DEPRECATED: use 'getsourcelines' instead
321 """
322 lines, lnum = findsource(object)
324 if ismodule(object):
325 if lstrip: lines = _outdent(lines)
326 return ([lines], [0]) if locate is True else [lines]
328 #XXX: 'enclosing' means: closures only? or classes and files?
329 indent = indentsize(lines[lnum])
330 block = getblock(lines[lnum:]) #XXX: catch any TokenError here?
332 if not enclosing or not indent:
333 if lstrip: block = _outdent(block)
334 return ([block], [lnum]) if locate is True else [block]
336 pat1 = r'^(\s*(?:async\s+)?def\s)|(.*(?<!\w)lambda(:|\s))'; pat1 = re.compile(pat1)
337 pat2 = r'^(\s*@)'; pat2 = re.compile(pat2)
338 #pat3 = r'^(\s*class\s)'; pat3 = re.compile(pat3) #XXX: enclosing class?
339 #FIXME: bound methods need enclosing class (and then instantiation)
340 # *or* somehow apply a partial using the instance
342 skip = 0
343 line = 0
344 blocks = []; _lnum = []
345 target = ''.join(block)
346 while line <= lnum: #XXX: repeat lnum? or until line < lnum?
347 # see if starts with ('def','lambda') and contains our target block
348 if pat1.match(lines[line]):
349 if not skip:
350 try: code = getblock(lines[line:])
351 except TokenError: code = [lines[line]]
352 if indentsize(lines[line]) > indent: #XXX: should be >= ?
353 line += len(code) - skip
354 elif target in ''.join(code):
355 blocks.append(code) # save code block as the potential winner
356 _lnum.append(line - skip) # save the line number for the match
357 line += len(code) - skip
358 else:
359 line += 1
360 skip = 0
361 # find skip: the number of consecutive decorators
362 elif pat2.match(lines[line]):
363 try: code = getblock(lines[line:])
364 except TokenError: code = [lines[line]]
365 skip = 1
366 for _line in code[1:]: # skip lines that are decorators
367 if not pat2.match(_line): break
368 skip += 1
369 line += skip
370 # no match: reset skip and go to the next line
371 else:
372 line +=1
373 skip = 0
375 if not blocks:
376 blocks = [block]
377 _lnum = [lnum]
378 if lstrip: blocks = [_outdent(block) for block in blocks]
379 # return last match
380 return (blocks, _lnum) if locate is True else blocks
383def getsourcelines(object, lstrip=False, enclosing=False):
384 """Return a list of source lines and starting line number for an object.
385 Interactively-defined objects refer to lines in the interpreter's history.
387 The argument may be a module, class, method, function, traceback, frame,
388 or code object. The source code is returned as a list of the lines
389 corresponding to the object and the line number indicates where in the
390 original source file the first line of code was found. An IOError is
391 raised if the source code cannot be retrieved, while a TypeError is
392 raised for objects where the source code is unavailable (e.g. builtins).
394 If lstrip=True, ensure there is no indentation in the first line of code.
395 If enclosing=True, then also return any enclosing code."""
396 code, n = getblocks(object, lstrip=lstrip, enclosing=enclosing, locate=True)
397 return code[-1], n[-1]
400#NOTE: broke backward compatibility 4/16/14 (was lstrip=True, force=True)
401def getsource(object, alias='', lstrip=False, enclosing=False, \
402 force=False, builtin=False):
403 """Return the text of the source code for an object. The source code for
404 interactively-defined objects are extracted from the interpreter's history.
406 The argument may be a module, class, method, function, traceback, frame,
407 or code object. The source code is returned as a single string. An
408 IOError is raised if the source code cannot be retrieved, while a
409 TypeError is raised for objects where the source code is unavailable
410 (e.g. builtins).
412 If alias is provided, then add a line of code that renames the object.
413 If lstrip=True, ensure there is no indentation in the first line of code.
414 If enclosing=True, then also return any enclosing code.
415 If force=True, catch (TypeError,IOError) and try to use import hooks.
416 If builtin=True, force an import for any builtins
417 """
418 # hascode denotes a callable
419 hascode = _hascode(object)
420 # is a class instance type (and not in builtins)
421 instance = _isinstance(object)
423 # get source lines; if fail, try to 'force' an import
424 try: # fails for builtins, and other assorted object types
425 lines, lnum = getsourcelines(object, enclosing=enclosing)
426 except (TypeError, IOError): # failed to get source, resort to import hooks
427 if not force: # don't try to get types that findsource can't get
428 raise
429 if not getmodule(object): # get things like 'None' and '1'
430 if not instance: return getimport(object, alias, builtin=builtin)
431 # special handling (numpy arrays, ...)
432 _import = getimport(object, builtin=builtin)
433 name = getname(object, force=True)
434 _alias = "%s = " % alias if alias else ""
435 if alias == name: _alias = ""
436 return _import+_alias+"%s\n" % name
437 else: #FIXME: could use a good bit of cleanup, since using getimport...
438 if not instance: return getimport(object, alias, builtin=builtin)
439 # now we are dealing with an instance...
440 name = object.__class__.__name__
441 module = object.__module__
442 if not name.isidentifier() or not all(i.isidentifier() for i in module.split('.')): #XXX: does excluding malicious code exclude valid use?
443 raise SyntaxError('invalid syntax')
444 if module in ['builtins','__builtin__']:
445 return getimport(object, alias, builtin=builtin)
446 else: #FIXME: leverage getimport? use 'from module import name'?
447 lines, lnum = ["%s = __import__('%s', fromlist=['%s']).%s\n" % (name,module,name,name)], 0
448 obj = eval(lines[0].lstrip(name + ' = '))
449 lines, lnum = getsourcelines(obj, enclosing=enclosing)
451 # strip leading indent (helps ensure can be imported)
452 if lstrip or alias:
453 lines = _outdent(lines)
455 # instantiate, if there's a nice repr #XXX: BAD IDEA???
456 if instance: #and force: #XXX: move into findsource or getsourcelines ?
457 if '(' in repr(object): lines.append('%r\n' % object)
458 #else: #XXX: better to somehow to leverage __reduce__ ?
459 # reconstructor,args = object.__reduce__()
460 # _ = reconstructor(*args)
461 else: # fall back to serialization #XXX: bad idea?
462 #XXX: better not duplicate work? #XXX: better new/enclose=True?
463 lines = dumpsource(object, alias='', new=force, enclose=False)
464 lines, lnum = [line+'\n' for line in lines.split('\n')][:-1], 0
465 #else: object.__code__ # raise AttributeError
467 # add an alias to the source code
468 if alias:
469 if hascode:
470 skip = 0
471 for line in lines: # skip lines that are decorators
472 if not line.startswith('@'): break
473 skip += 1
474 #XXX: use regex from findsource / getsourcelines ?
475 if re.match(r'^\s*(?:async\s+)?def\s', lines[skip]): # we have a function
476 if alias != object.__name__:
477 lines.append('\n%s = %s\n' % (alias, object.__name__))
478 elif 'lambda ' in lines[skip]: # we have a lambda
479 if alias != lines[skip].split('=')[0].strip():
480 lines[skip] = '%s = %s' % (alias, lines[skip])
481 else: # ...try to use the object's name
482 if alias != object.__name__:
483 lines.append('\n%s = %s\n' % (alias, object.__name__))
484 else: # class or class instance
485 if instance:
486 if alias != lines[-1].split('=')[0].strip():
487 lines[-1] = ('%s = ' % alias) + lines[-1]
488 else:
489 name = getname(object, force=True) or object.__name__
490 if alias != name:
491 lines.append('\n%s = %s\n' % (alias, name))
492 return ''.join(lines)
495def _hascode(object):
496 '''True if object has an attribute that stores it's __code__'''
497 return getattr(object,'__code__',None) or getattr(object,'func_code',None)
499def _isinstance(object):
500 '''True if object is a class instance type (and is not a builtin)'''
501 if _hascode(object) or isclass(object) or ismodule(object):
502 return False
503 if istraceback(object) or isframe(object) or iscode(object):
504 return False
505 # special handling (numpy arrays, ...)
506 if not getmodule(object) and getmodule(type(object)).__name__ in ['numpy']:
507 return True
508# # check if is instance of a builtin
509# if not getmodule(object) and getmodule(type(object)).__name__ in ['__builtin__','builtins']:
510# return False
511 _types = ('<class ',"<type 'instance'>")
512 if not repr(type(object)).startswith(_types): #FIXME: weak hack
513 return False
514 if not getmodule(object) or object.__module__ in ['builtins','__builtin__'] or getname(object, force=True) in ['array']:
515 return False
516 return True # by process of elimination... it's what we want
519def _intypes(object):
520 '''check if object is in the 'types' module'''
521 import types
522 # allow user to pass in object or object.__name__
523 if type(object) is not type(''):
524 object = getname(object, force=True)
525 if object == 'ellipsis': object = 'EllipsisType'
526 return True if hasattr(types, object) else False
529def _isstring(object): #XXX: isstringlike better?
530 '''check if object is a string-like type'''
531 return isinstance(object, (str, bytes))
534def indent(code, spaces=4):
535 '''indent a block of code with whitespace (default is 4 spaces)'''
536 indent = indentsize(code)
537 from numbers import Integral
538 if isinstance(spaces, Integral): spaces = ' '*spaces
539 # if '\t' is provided, will indent with a tab
540 nspaces = indentsize(spaces)
541 # blank lines (etc) need to be ignored
542 lines = code.split('\n')
543## stq = "'''"; dtq = '"""'
544## in_stq = in_dtq = False
545 for i in range(len(lines)):
546 #FIXME: works... but shouldn't indent 2nd+ lines of multiline doc
547 _indent = indentsize(lines[i])
548 if indent > _indent: continue
549 lines[i] = spaces+lines[i]
550## #FIXME: may fail when stq and dtq in same line (depends on ordering)
551## nstq, ndtq = lines[i].count(stq), lines[i].count(dtq)
552## if not in_dtq and not in_stq:
553## lines[i] = spaces+lines[i] # we indent
554## # entering a comment block
555## if nstq%2: in_stq = not in_stq
556## if ndtq%2: in_dtq = not in_dtq
557## # leaving a comment block
558## elif in_dtq and ndtq%2: in_dtq = not in_dtq
559## elif in_stq and nstq%2: in_stq = not in_stq
560## else: pass
561 if lines[-1].strip() == '': lines[-1] = ''
562 return '\n'.join(lines)
565def _outdent(lines, spaces=None, all=True):
566 '''outdent lines of code, accounting for docs and line continuations'''
567 indent = indentsize(lines[0])
568 if spaces is None or spaces > indent or spaces < 0: spaces = indent
569 for i in range(len(lines) if all else 1):
570 #FIXME: works... but shouldn't outdent 2nd+ lines of multiline doc
571 _indent = indentsize(lines[i])
572 if spaces > _indent: _spaces = _indent
573 else: _spaces = spaces
574 lines[i] = lines[i][_spaces:]
575 return lines
577def outdent(code, spaces=None, all=True):
578 '''outdent a block of code (default is to strip all leading whitespace)'''
579 indent = indentsize(code)
580 if spaces is None or spaces > indent or spaces < 0: spaces = indent
581 #XXX: will this delete '\n' in some cases?
582 if not all: return code[spaces:]
583 return '\n'.join(_outdent(code.split('\n'), spaces=spaces, all=all))
586# _wrap provides an wrapper to correctly exec and load into locals
587__globals__ = globals()
588__locals__ = locals()
589def _wrap(f):
590 """ encapsulate a function and it's __import__ """
591 def func(*args, **kwds):
592 try:
593 # _ = eval(getsource(f, force=True)) #XXX: safer but less robust
594 exec(getimportable(f, alias='_'), __globals__, __locals__)
595 except Exception:
596 raise ImportError('cannot import name ' + f.__name__)
597 return _(*args, **kwds)
598 func.__name__ = f.__name__
599 func.__doc__ = f.__doc__
600 return func
603def _enclose(object, alias=''): #FIXME: needs alias to hold returned object
604 """create a function enclosure around the source of some object"""
605 #XXX: dummy and stub should append a random string
606 dummy = '__this_is_a_big_dummy_enclosing_function__'
607 stub = '__this_is_a_stub_variable__'
608 code = 'def %s():\n' % dummy
609 code += indent(getsource(object, alias=stub, lstrip=True, force=True))
610 code += indent('return %s\n' % stub)
611 if alias: code += '%s = ' % alias
612 code += '%s(); del %s\n' % (dummy, dummy)
613 #code += "globals().pop('%s',lambda :None)()\n" % dummy
614 return code
617def dumpsource(object, alias='', new=False, enclose=True):
618 """'dump to source', where the code includes a pickled object.
620 If new=True and object is a class instance, then create a new
621 instance using the unpacked class source code. If enclose, then
622 create the object inside a function enclosure (thus minimizing
623 any global namespace pollution).
624 """
625 from dill import dumps
626 pik = repr(dumps(object))
627 code = 'import dill\n'
628 if enclose:
629 stub = '__this_is_a_stub_variable__' #XXX: *must* be same _enclose.stub
630 pre = '%s = ' % stub
631 new = False #FIXME: new=True doesn't work with enclose=True
632 else:
633 stub = alias
634 pre = '%s = ' % stub if alias else alias
636 # if a 'new' instance is not needed, then just dump and load
637 if not new or not _isinstance(object):
638 code += pre + 'dill.loads(%s)\n' % pik
639 else: #XXX: other cases where source code is needed???
640 code += getsource(object.__class__, alias='', lstrip=True, force=True)
641 mod = repr(object.__module__) # should have a module (no builtins here)
642 code += pre + 'dill.loads(%s.replace(b%s,bytes(__name__,"UTF-8")))\n' % (pik,mod)
643 #code += 'del %s' % object.__class__.__name__ #NOTE: kills any existing!
645 if enclose:
646 # generation of the 'enclosure'
647 dummy = '__this_is_a_big_dummy_object__'
648 dummy = _enclose(dummy, alias=alias)
649 # hack to replace the 'dummy' with the 'real' code
650 dummy = dummy.split('\n')
651 code = dummy[0]+'\n' + indent(code) + '\n'.join(dummy[-3:])
653 return code #XXX: better 'dumpsourcelines', returning list of lines?
656def getname(obj, force=False, fqn=False): #XXX: throw(?) to raise error on fail?
657 """get the name of the object. for lambdas, get the name of the pointer """
658 if fqn: return '.'.join(_namespace(obj)) #NOTE: returns 'type'
659 module = getmodule(obj)
660 if not module: # things like "None" and "1"
661 if not force: return None #NOTE: returns 'instance' NOT 'type' #FIXME?
662 # handle some special cases
663 if hasattr(obj, 'dtype') and not obj.shape:
664 return getname(obj.__class__) + "(" + repr(obj.tolist()) + ")"
665 return repr(obj)
666 try:
667 #XXX: 'wrong' for decorators and curried functions ?
668 # if obj.func_closure: ...use logic from getimportable, etc ?
669 name = obj.__name__
670 if name == '<lambda>':
671 return getsource(obj).split('=',1)[0].strip()
672 # handle some special cases
673 if module.__name__ in ['builtins','__builtin__']:
674 if name == 'ellipsis': name = 'EllipsisType'
675 return name
676 except AttributeError: #XXX: better to just throw AttributeError ?
677 if not force: return None
678 name = repr(obj)
679 if name.startswith('<'): # or name.split('('):
680 return None
681 return name
684def _namespace(obj):
685 """_namespace(obj); return namespace hierarchy (as a list of names)
686 for the given object. For an instance, find the class hierarchy.
688 For example:
690 >>> from functools import partial
691 >>> p = partial(int, base=2)
692 >>> _namespace(p)
693 [\'functools\', \'partial\']
694 """
695 # mostly for functions and modules and such
696 #FIXME: 'wrong' for decorators and curried functions
697 try: #XXX: needs some work and testing on different types
698 module = qual = str(getmodule(obj)).split()[1].strip('>').strip('"').strip("'")
699 qual = qual.split('.')
700 if ismodule(obj):
701 return qual
702 # get name of a lambda, function, etc
703 name = getname(obj) or obj.__name__ # failing, raise AttributeError
704 # check special cases (NoneType, ...)
705 if module in ['builtins','__builtin__']: # BuiltinFunctionType
706 if _intypes(name): return ['types'] + [name]
707 return qual + [name] #XXX: can be wrong for some aliased objects
708 except Exception: pass
709 # special case: numpy.inf and numpy.nan (we don't want them as floats)
710 if str(obj) in ['inf','nan','Inf','NaN']: # is more, but are they needed?
711 return ['numpy'] + [str(obj)]
712 # mostly for classes and class instances and such
713 module = getattr(obj.__class__, '__module__', None)
714 qual = str(obj.__class__)
715 try: qual = qual[qual.index("'")+1:-2]
716 except ValueError: pass # str(obj.__class__) made the 'try' unnecessary
717 qual = qual.split(".")
718 if module in ['builtins','__builtin__']:
719 # check special cases (NoneType, Ellipsis, ...)
720 if qual[-1] == 'ellipsis': qual[-1] = 'EllipsisType'
721 if _intypes(qual[-1]): module = 'types' #XXX: BuiltinFunctionType
722 qual = [module] + qual
723 return qual
726#NOTE: 05/25/14 broke backward compatibility: added 'alias' as 3rd argument
727def _getimport(head, tail, alias='', verify=True, builtin=False):
728 """helper to build a likely import string from head and tail of namespace.
729 ('head','tail') are used in the following context: "from head import tail"
731 If verify=True, then test the import string before returning it.
732 If builtin=True, then force an import for builtins where possible.
733 If alias is provided, then rename the object on import.
734 """
735 # special handling for a few common types
736 if tail in ['Ellipsis', 'NotImplemented'] and head in ['types']:
737 head = len.__module__
738 elif tail in ['None'] and head in ['types']:
739 _alias = '%s = ' % alias if alias else ''
740 if alias == tail: _alias = ''
741 return _alias+'%s\n' % tail
742 # we don't need to import from builtins, so return ''
743# elif tail in ['NoneType','int','float','long','complex']: return '' #XXX: ?
744 if head in ['builtins','__builtin__']:
745 # special cases (NoneType, Ellipsis, ...) #XXX: BuiltinFunctionType
746 if tail == 'ellipsis': tail = 'EllipsisType'
747 if _intypes(tail): head = 'types'
748 elif not builtin:
749 _alias = '%s = ' % alias if alias else ''
750 if alias == tail: _alias = ''
751 return _alias+'%s\n' % tail
752 else: pass # handle builtins below
753 # get likely import string
754 if not head: _str = "import %s" % tail
755 else: _str = "from %s import %s" % (head, tail)
756 _alias = " as %s\n" % alias if alias else "\n"
757 if alias == tail: _alias = "\n"
758 _str += _alias
759 # FIXME: fails on most decorators, currying, and such...
760 # (could look for magic __wrapped__ or __func__ attr)
761 # (could fix in 'namespace' to check obj for closure)
762 if verify and not head.startswith('dill.'):# weird behavior for dill
763 #print(_str)
764 if not all(i.isidentifier() for i in tail.split('.')) \
765 or (head and not all(i.isidentifier() for i in head.split('.'))): #XXX: as in getsource
766 raise SyntaxError('invalid syntax')
767 try: exec(_str) #XXX: check if == obj? (name collision)
768 except ImportError: #XXX: better top-down or bottom-up recursion?
769 _head = head.rsplit(".",1)[0] #(or get all, then compare == obj?)
770 if not _head: raise
771 if _head != head:
772 _str = _getimport(_head, tail, alias, verify)
773 return _str
776#XXX: rename builtin to force? vice versa? verify to force? (as in getsource)
777#NOTE: 05/25/14 broke backward compatibility: added 'alias' as 2nd argument
778def getimport(obj, alias='', verify=True, builtin=False, enclosing=False):
779 """get the likely import string for the given object
781 obj is the object to inspect
782 If verify=True, then test the import string before returning it.
783 If builtin=True, then force an import for builtins where possible.
784 If enclosing=True, get the import for the outermost enclosing callable.
785 If alias is provided, then rename the object on import.
786 """
787 if enclosing:
788 from .detect import outermost
789 _obj = outermost(obj)
790 obj = _obj if _obj else obj
791 # get the namespace
792 qual = _namespace(obj)
793 head = '.'.join(qual[:-1])
794 tail = qual[-1]
795 # for named things... with a nice repr #XXX: move into _namespace?
796 try: # look for '<...>' and be mindful it might be in lists, dicts, etc...
797 name = repr(obj).split('<',1)[1].split('>',1)[1]
798 name = None # we have a 'object'-style repr
799 except Exception: # it's probably something 'importable'
800 if head in ['builtins','__builtin__']:
801 name = repr(obj) #XXX: catch [1,2], (1,2), set([1,2])... others?
802 elif _isinstance(obj):
803 name = getname(obj, force=True).split('(')[0]
804 else:
805 name = repr(obj).split('(')[0]
806 #if not repr(obj).startswith('<'): name = repr(obj).split('(')[0]
807 #else: name = None
808 if name: # try using name instead of tail
809 try: return _getimport(head, name, alias, verify, builtin)
810 except ImportError: pass
811 except SyntaxError:
812 if head in ['builtins','__builtin__']:
813 _alias = '%s = ' % alias if alias else ''
814 if alias == name: _alias = ''
815 return _alias+'%s\n' % name
816 else: pass
817 try:
818 #if type(obj) is type(abs): _builtin = builtin # BuiltinFunctionType
819 #else: _builtin = False
820 return _getimport(head, tail, alias, verify, builtin)
821 except ImportError:
822 raise # could do some checking against obj
823 except SyntaxError:
824 if head in ['builtins','__builtin__']:
825 _alias = '%s = ' % alias if alias else ''
826 if alias == tail: _alias = ''
827 return _alias+'%s\n' % tail
828 raise # could do some checking against obj
831def _importable(obj, alias='', source=None, enclosing=False, force=True, \
832 builtin=True, lstrip=True):
833 """get an import string (or the source code) for the given object
835 This function will attempt to discover the name of the object, or the repr
836 of the object, or the source code for the object. To attempt to force
837 discovery of the source code, use source=True, to attempt to force the
838 use of an import, use source=False; otherwise an import will be sought
839 for objects not defined in __main__. The intent is to build a string
840 that can be imported from a python file. obj is the object to inspect.
841 If alias is provided, then rename the object with the given alias.
843 If source=True, use these options:
844 If enclosing=True, then also return any enclosing code.
845 If force=True, catch (TypeError,IOError) and try to use import hooks.
846 If lstrip=True, ensure there is no indentation in the first line of code.
848 If source=False, use these options:
849 If enclosing=True, get the import for the outermost enclosing callable.
850 If force=True, then don't test the import string before returning it.
851 If builtin=True, then force an import for builtins where possible.
852 """
853 if source is None:
854 source = True if isfrommain(obj) else False
855 if source: # first try to get the source
856 try:
857 return getsource(obj, alias, enclosing=enclosing, \
858 force=force, lstrip=lstrip, builtin=builtin)
859 except Exception: pass
860 try:
861 if not _isinstance(obj):
862 return getimport(obj, alias, enclosing=enclosing, \
863 verify=(not force), builtin=builtin)
864 # first 'get the import', then 'get the instance'
865 _import = getimport(obj, enclosing=enclosing, \
866 verify=(not force), builtin=builtin)
867 name = getname(obj, force=True)
868 if not name:
869 raise AttributeError("object has no attribute '__name__'")
870 _alias = "%s = " % alias if alias else ""
871 if alias == name: _alias = ""
872 return _import+_alias+"%s\n" % name
874 except Exception: pass
875 if not source: # try getsource, only if it hasn't been tried yet
876 try:
877 return getsource(obj, alias, enclosing=enclosing, \
878 force=force, lstrip=lstrip, builtin=builtin)
879 except Exception: pass
880 # get the name (of functions, lambdas, and classes)
881 # or hope that obj can be built from the __repr__
882 #XXX: what to do about class instances and such?
883 obj = getname(obj, force=force)
884 # we either have __repr__ or __name__ (or None)
885 if not obj or obj.startswith('<'):
886 raise AttributeError("object has no atribute '__name__'")
887 _alias = '%s = ' % alias if alias else ''
888 if alias == obj: _alias = ''
889 return _alias+'%s\n' % obj
890 #XXX: possible failsafe... (for example, for instances when source=False)
891 # "import dill; result = dill.loads(<pickled_object>); # repr(<object>)"
893def _closuredimport(func, alias='', builtin=False):
894 """get import for closured objects; return a dict of 'name' and 'import'"""
895 import re
896 from .detect import freevars, outermost
897 free_vars = freevars(func)
898 func_vars = {}
899 # split into 'funcs' and 'non-funcs'
900 for name,obj in list(free_vars.items()):
901 if not isfunction(obj): continue
902 # get import for 'funcs'
903 fobj = free_vars.pop(name)
904 src = getsource(fobj)
905 if src.lstrip().startswith('@'): # we have a decorator
906 src = getimport(fobj, alias=alias, builtin=builtin)
907 else: # we have to "hack" a bit... and maybe be lucky
908 encl = outermost(func)
909 # pattern: 'func = enclosing(fobj'
910 pat = r'.*[\w\s]=\s*'+re.escape(getname(encl))+r'\('+re.escape(getname(fobj))
911 mod = getname(getmodule(encl))
912 #HACK: get file containing 'outer' function; is func there?
913 lines,_ = findsource(encl)
914 candidate = [line for line in lines if getname(encl) in line and \
915 re.match(pat, line)]
916 if not candidate:
917 mod = getname(getmodule(fobj))
918 #HACK: get file containing 'inner' function; is func there?
919 lines,_ = findsource(fobj)
920 candidate = [line for line in lines \
921 if getname(fobj) in line and re.match(pat, line)]
922 if not len(candidate): raise TypeError('import could not be found')
923 candidate = candidate[-1]
924 name = candidate.split('=',1)[0].split()[-1].strip()
925 src = _getimport(mod, name, alias=alias, builtin=builtin)
926 func_vars[name] = src
927 if not func_vars:
928 name = outermost(func)
929 mod = getname(getmodule(name))
930 if not mod or name is func: # then it can be handled by getimport
931 name = getname(func, force=True) #XXX: better key?
932 src = getimport(func, alias=alias, builtin=builtin)
933 else:
934 lines,_ = findsource(name)
935 # pattern: 'func = enclosing('
936 candidate = [line for line in lines if getname(name) in line and \
937 re.match(r'.*[\w\s]=\s*'+re.escape(getname(name))+r'\(', line)]
938 if not len(candidate): raise TypeError('import could not be found')
939 candidate = candidate[-1]
940 name = candidate.split('=',1)[0].split()[-1].strip()
941 src = _getimport(mod, name, alias=alias, builtin=builtin)
942 func_vars[name] = src
943 return func_vars
945#XXX: should be able to use __qualname__
946def _closuredsource(func, alias=''):
947 """get source code for closured objects; return a dict of 'name'
948 and 'code blocks'"""
949 #FIXME: this entire function is a messy messy HACK
950 # - pollutes global namespace
951 # - fails if name of freevars are reused
952 # - can unnecessarily duplicate function code
953 from .detect import freevars
954 free_vars = freevars(func)
955 func_vars = {}
956 # split into 'funcs' and 'non-funcs'
957 for name,obj in list(free_vars.items()):
958 if not isfunction(obj):
959 # get source for 'non-funcs'
960 free_vars[name] = getsource(obj, force=True, alias=name)
961 continue
962 # get source for 'funcs'
963 fobj = free_vars.pop(name)
964 src = getsource(fobj, alias) # DO NOT include dependencies
965 # if source doesn't start with '@', use name as the alias
966 if not src.lstrip().startswith('@'): #FIXME: 'enclose' in dummy;
967 src = importable(fobj,alias=name)# wrong ref 'name'
968 org = getsource(func, alias, enclosing=False, lstrip=True)
969 src = (src, org) # undecorated first, then target
970 else: #NOTE: reproduces the code!
971 org = getsource(func, enclosing=True, lstrip=False)
972 src = importable(fobj, alias, source=True) # include dependencies
973 src = (org, src) # target first, then decorated
974 func_vars[name] = src
975 src = ''.join(free_vars.values())
976 if not func_vars: #FIXME: 'enclose' in dummy; wrong ref 'name'
977 org = getsource(func, alias, force=True, enclosing=False, lstrip=True)
978 src = (src, org) # variables first, then target
979 else:
980 src = (src, None) # just variables (better '' instead of None?)
981 func_vars[None] = src
982 # FIXME: remove duplicates (however, order is important...)
983 return func_vars
985def importable(obj, alias='', source=None, builtin=True):
986 """get an importable string (i.e. source code or the import string)
987 for the given object, including any required objects from the enclosing
988 and global scope
990 This function will attempt to discover the name of the object, or the repr
991 of the object, or the source code for the object. To attempt to force
992 discovery of the source code, use source=True, to attempt to force the
993 use of an import, use source=False; otherwise an import will be sought
994 for objects not defined in __main__. The intent is to build a string
995 that can be imported from a python file.
997 obj is the object to inspect. If alias is provided, then rename the
998 object with the given alias. If builtin=True, then force an import for
999 builtins where possible.
1000 """
1001 #NOTE: we always 'force', and 'lstrip' as necessary
1002 #NOTE: for 'enclosing', use importable(outermost(obj))
1003 if source is None:
1004 source = True if isfrommain(obj) else False
1005 elif builtin and isbuiltin(obj):
1006 source = False
1007 tried_source = tried_import = False
1008 while True:
1009 if not source: # we want an import
1010 try:
1011 if _isinstance(obj): # for instances, punt to _importable
1012 return _importable(obj, alias, source=False, builtin=builtin)
1013 src = _closuredimport(obj, alias=alias, builtin=builtin)
1014 if len(src) == 0:
1015 raise NotImplementedError('not implemented')
1016 if len(src) > 1:
1017 raise NotImplementedError('not implemented')
1018 return list(src.values())[0]
1019 except Exception:
1020 if tried_source: raise
1021 tried_import = True
1022 # we want the source
1023 try:
1024 src = _closuredsource(obj, alias=alias)
1025 if len(src) == 0:
1026 raise NotImplementedError('not implemented')
1027 # groan... an inline code stitcher
1028 def _code_stitcher(block):
1029 "stitch together the strings in tuple 'block'"
1030 if block[0] and block[-1]: block = '\n'.join(block)
1031 elif block[0]: block = block[0]
1032 elif block[-1]: block = block[-1]
1033 else: block = ''
1034 return block
1035 # get free_vars first
1036 _src = _code_stitcher(src.pop(None))
1037 _src = [_src] if _src else []
1038 # get func_vars
1039 for xxx in src.values():
1040 xxx = _code_stitcher(xxx)
1041 if xxx: _src.append(xxx)
1042 # make a single source string
1043 if not len(_src):
1044 src = ''
1045 elif len(_src) == 1:
1046 src = _src[0]
1047 else:
1048 src = '\n'.join(_src)
1049 # get source code of objects referred to by obj in global scope
1050 from .detect import globalvars
1051 obj = globalvars(obj) #XXX: don't worry about alias? recurse? etc?
1052 obj = list(getsource(_obj,name,force=True) for (name,_obj) in obj.items() if not isbuiltin(_obj))
1053 obj = '\n'.join(obj) if obj else ''
1054 # combine all referred-to source (global then enclosing)
1055 if not obj: return src
1056 if not src: return obj
1057 return obj + src
1058 except Exception:
1059 if tried_import: raise
1060 tried_source = True
1061 source = not source
1062 # should never get here
1063 return
1066# backward compatibility
1067def getimportable(obj, alias='', byname=True, explicit=False):
1068 return importable(obj,alias,source=(not byname),builtin=explicit)
1069 #return outdent(_importable(obj,alias,source=(not byname),builtin=explicit))
1070def likely_import(obj, passive=False, explicit=False):
1071 return getimport(obj, verify=(not passive), builtin=explicit)
1072def _likely_import(first, last, passive=False, explicit=True):
1073 return _getimport(first, last, verify=(not passive), builtin=explicit)
1074_get_name = getname
1075getblocks_from_history = getblocks
1079# EOF