Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.10/site-packages/dill/source.py: 6%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#!/usr/bin/env python
2#
3# Author: Mike McKerns (mmckerns @caltech and @uqfoundation)
4# Copyright (c) 2008-2016 California Institute of Technology.
5# Copyright (c) 2016-2025 The Uncertainty Quantification Foundation.
6# License: 3-clause BSD. The full license text is available at:
7# - https://github.com/uqfoundation/dill/blob/master/LICENSE
8#
9# inspired by inspect.py from Python-2.7.6
10# inspect.py author: 'Ka-Ping Yee <ping@lfw.org>'
11# inspect.py merged into original dill.source by Mike McKerns 4/13/14
12"""
13Extensions to python's 'inspect' module, which can be used
14to retrieve information from live python objects. The methods
15defined in this module are augmented to facilitate access to
16source code of interactively defined functions and classes,
17as well as provide access to source code for objects defined
18in a file.
19"""
21__all__ = ['findsource', 'getsourcelines', 'getsource', 'indent', 'outdent', \
22 '_wrap', 'dumpsource', 'getname', '_namespace', 'getimport', \
23 '_importable', 'importable','isdynamic', 'isfrommain']
25import linecache
26import re
27from inspect import (getblock, getfile, getmodule, getsourcefile, indentsize,
28 isbuiltin, isclass, iscode, isframe, isfunction, ismethod,
29 ismodule, istraceback)
30from tokenize import TokenError
32from ._dill import IS_IPYTHON
35def isfrommain(obj):
36 "check if object was built in __main__"
37 module = getmodule(obj)
38 if module and module.__name__ == '__main__':
39 return True
40 return False
43def isdynamic(obj):
44 "check if object was built in the interpreter"
45 try: file = getfile(obj)
46 except TypeError: file = None
47 if file == '<stdin>' and isfrommain(obj):
48 return True
49 return False
52def _matchlambda(func, line):
53 """check if lambda object 'func' matches raw line of code 'line'"""
54 from .detect import code as getcode
55 from .detect import freevars, globalvars, varnames
56 dummy = lambda : '__this_is_a_big_dummy_function__'
57 # process the line (removing leading whitespace, etc)
58 lhs,rhs = line.split('lambda ',1)[-1].split(":", 1) #FIXME: if !1 inputs
59 try: #FIXME: unsafe
60 _ = eval("lambda %s : %s" % (lhs,rhs), globals(),locals())
61 except Exception: _ = dummy
62 # get code objects, for comparison
63 _, code = getcode(_).co_code, getcode(func).co_code
64 # check if func is in closure
65 _f = [line.count(i) for i in freevars(func).keys()]
66 if not _f: # not in closure
67 # check if code matches
68 if _ == code: return True
69 return False
70 # weak check on freevars
71 if not all(_f): return False #XXX: VERY WEAK
72 # weak check on varnames and globalvars
73 _f = varnames(func)
74 _f = [line.count(i) for i in _f[0]+_f[1]]
75 if _f and not all(_f): return False #XXX: VERY WEAK
76 _f = [line.count(i) for i in globalvars(func).keys()]
77 if _f and not all(_f): return False #XXX: VERY WEAK
78 # check if func is a double lambda
79 if (line.count('lambda ') > 1) and (lhs in freevars(func).keys()):
80 _lhs,_rhs = rhs.split('lambda ',1)[-1].split(":",1) #FIXME: if !1 inputs
81 try: #FIXME: unsafe
82 _f = eval("lambda %s : %s" % (_lhs,_rhs), globals(),locals())
83 except Exception: _f = dummy
84 # get code objects, for comparison
85 _, code = getcode(_f).co_code, getcode(func).co_code
86 if len(_) != len(code): return False
87 #NOTE: should be same code same order, but except for 't' and '\x88'
88 _ = set((i,j) for (i,j) in zip(_,code) if i != j)
89 if len(_) != 1: return False #('t','\x88')
90 return True
91 # check indentsize
92 if not indentsize(line): return False #FIXME: is this a good check???
93 # check if code 'pattern' matches
94 #XXX: or pattern match against dis.dis(code)? (or use uncompyle2?)
95 _ = _.split(_[0]) # 't' #XXX: remove matching values if starts the same?
96 _f = code.split(code[0]) # '\x88'
97 #NOTE: should be same code different order, with different first element
98 _ = dict(re.match(r'([\W\D\S])(.*)', _[i]).groups() for i in range(1,len(_)))
99 _f = dict(re.match(r'([\W\D\S])(.*)', _f[i]).groups() for i in range(1,len(_f)))
100 if (_.keys() == _f.keys()) and (sorted(_.values()) == sorted(_f.values())):
101 return True
102 return False
105def findsource(object):
106 """Return the entire source file and starting line number for an object.
107 For interactively-defined objects, the 'file' is the interpreter's history.
109 The argument may be a module, class, method, function, traceback, frame,
110 or code object. The source code is returned as a list of all the lines
111 in the file and the line number indexes a line in that list. An IOError
112 is raised if the source code cannot be retrieved, while a TypeError is
113 raised for objects where the source code is unavailable (e.g. builtins)."""
115 module = getmodule(object)
116 try: file = getfile(module)
117 except TypeError: file = None
118 is_module_main = (module and module.__name__ == '__main__' and not file)
119 if IS_IPYTHON and is_module_main:
120 #FIXME: quick fix for functions and classes in IPython interpreter
121 try:
122 file = getfile(object)
123 sourcefile = getsourcefile(object)
124 except TypeError:
125 if isclass(object):
126 for object_method in filter(isfunction, object.__dict__.values()):
127 # look for a method of the class
128 file_candidate = getfile(object_method)
129 if not file_candidate.startswith('<ipython-input-'):
130 continue
131 file = file_candidate
132 sourcefile = getsourcefile(object_method)
133 break
134 if file:
135 lines = linecache.getlines(file)
136 else:
137 # fallback to use history
138 history = '\n'.join(get_ipython().history_manager.input_hist_parsed)
139 lines = [line + '\n' for line in history.splitlines()]
140 # use readline when working in interpreter (i.e. __main__ and not file)
141 elif is_module_main:
142 try:
143 import readline
144 err = ''
145 except ImportError:
146 import sys
147 err = sys.exc_info()[1].args[0]
148 if sys.platform[:3] == 'win':
149 err += ", please install 'pyreadline'"
150 if err:
151 raise IOError(err)
152 lbuf = readline.get_current_history_length()
153 lines = [readline.get_history_item(i)+'\n' for i in range(1,lbuf+1)]
154 else:
155 try: # special handling for class instances
156 if not isclass(object) and isclass(type(object)): # __class__
157 file = getfile(module)
158 sourcefile = getsourcefile(module)
159 else: # builtins fail with a TypeError
160 file = getfile(object)
161 sourcefile = getsourcefile(object)
162 except (TypeError, AttributeError): # fail with better error
163 file = getfile(object)
164 sourcefile = getsourcefile(object)
165 if not sourcefile and file[:1] + file[-1:] != '<>':
166 raise IOError('source code not available')
167 file = sourcefile if sourcefile else file
169 module = getmodule(object, file)
170 if module:
171 lines = linecache.getlines(file, module.__dict__)
172 else:
173 lines = linecache.getlines(file)
175 if not lines:
176 raise IOError('could not extract source code')
178 #FIXME: all below may fail if exec used (i.e. exec('f = lambda x:x') )
179 if ismodule(object):
180 return lines, 0
182 #NOTE: beneficial if search goes from end to start of buffer history
183 name = pat1 = obj = ''
184 pat2 = r'^(\s*@)'
185# pat1b = r'^(\s*%s\W*=)' % name #FIXME: finds 'f = decorate(f)', not exec
186 if ismethod(object):
187 name = object.__name__
188 if name == '<lambda>': pat1 = r'(.*(?<!\w)lambda(:|\s))'
189 else: pat1 = r'^(\s*def\s)'
190 object = object.__func__
191 if isfunction(object):
192 name = object.__name__
193 if name == '<lambda>':
194 pat1 = r'(.*(?<!\w)lambda(:|\s))'
195 obj = object #XXX: better a copy?
196 else: pat1 = r'^(\s*def\s)'
197 object = object.__code__
198 if istraceback(object):
199 object = object.tb_frame
200 if isframe(object):
201 object = object.f_code
202 if iscode(object):
203 if not hasattr(object, 'co_firstlineno'):
204 raise IOError('could not find function definition')
205 stdin = object.co_filename == '<stdin>'
206 if stdin:
207 lnum = len(lines) - 1 # can't get lnum easily, so leverage pat
208 if not pat1: pat1 = r'^(\s*def\s)|(.*(?<!\w)lambda(:|\s))|^(\s*@)'
209 else:
210 lnum = object.co_firstlineno - 1
211 pat1 = r'^(\s*def\s)|(.*(?<!\w)lambda(:|\s))|^(\s*@)'
212 pat1 = re.compile(pat1); pat2 = re.compile(pat2)
213 #XXX: candidate_lnum = [n for n in range(lnum) if pat1.match(lines[n])]
214 while lnum > 0: #XXX: won't find decorators in <stdin> ?
215 line = lines[lnum]
216 if pat1.match(line):
217 if not stdin: break # co_firstlineno does the job
218 if name == '<lambda>': # hackery needed to confirm a match
219 if _matchlambda(obj, line): break
220 else: # not a lambda, just look for the name
221 if name in line: # need to check for decorator...
222 hats = 0
223 for _lnum in range(lnum-1,-1,-1):
224 if pat2.match(lines[_lnum]): hats += 1
225 else: break
226 lnum = lnum - hats
227 break
228 lnum = lnum - 1
229 return lines, lnum
231 try: # turn instances into classes
232 if not isclass(object) and isclass(type(object)): # __class__
233 object = object.__class__ #XXX: sometimes type(class) is better?
234 #XXX: we don't find how the instance was built
235 except AttributeError: pass
236 if isclass(object):
237 name = object.__name__
238 pat = re.compile(r'^(\s*)class\s*' + name + r'\b')
239 # make some effort to find the best matching class definition:
240 # use the one with the least indentation, which is the one
241 # that's most probably not inside a function definition.
242 candidates = []
243 for i in range(len(lines)-1,-1,-1):
244 match = pat.match(lines[i])
245 if match:
246 # if it's at toplevel, it's already the best one
247 if lines[i][0] == 'c':
248 return lines, i
249 # else add whitespace to candidate list
250 candidates.append((match.group(1), i))
251 if candidates:
252 # this will sort by whitespace, and by line number,
253 # less whitespace first #XXX: should sort high lnum before low
254 candidates.sort()
255 return lines, candidates[0][1]
256 else:
257 raise IOError('could not find class definition')
258 raise IOError('could not find code object')
261def getblocks(object, lstrip=False, enclosing=False, locate=False):
262 """Return a list of source lines and starting line number for an object.
263 Interactively-defined objects refer to lines in the interpreter's history.
265 If enclosing=True, then also return any enclosing code.
266 If lstrip=True, ensure there is no indentation in the first line of code.
267 If locate=True, then also return the line number for the block of code.
269 DEPRECATED: use 'getsourcelines' instead
270 """
271 lines, lnum = findsource(object)
273 if ismodule(object):
274 if lstrip: lines = _outdent(lines)
275 return ([lines], [0]) if locate is True else [lines]
277 #XXX: 'enclosing' means: closures only? or classes and files?
278 indent = indentsize(lines[lnum])
279 block = getblock(lines[lnum:]) #XXX: catch any TokenError here?
281 if not enclosing or not indent:
282 if lstrip: block = _outdent(block)
283 return ([block], [lnum]) if locate is True else [block]
285 pat1 = r'^(\s*def\s)|(.*(?<!\w)lambda(:|\s))'; pat1 = re.compile(pat1)
286 pat2 = r'^(\s*@)'; pat2 = re.compile(pat2)
287 #pat3 = r'^(\s*class\s)'; pat3 = re.compile(pat3) #XXX: enclosing class?
288 #FIXME: bound methods need enclosing class (and then instantiation)
289 # *or* somehow apply a partial using the instance
291 skip = 0
292 line = 0
293 blocks = []; _lnum = []
294 target = ''.join(block)
295 while line <= lnum: #XXX: repeat lnum? or until line < lnum?
296 # see if starts with ('def','lambda') and contains our target block
297 if pat1.match(lines[line]):
298 if not skip:
299 try: code = getblock(lines[line:])
300 except TokenError: code = [lines[line]]
301 if indentsize(lines[line]) > indent: #XXX: should be >= ?
302 line += len(code) - skip
303 elif target in ''.join(code):
304 blocks.append(code) # save code block as the potential winner
305 _lnum.append(line - skip) # save the line number for the match
306 line += len(code) - skip
307 else:
308 line += 1
309 skip = 0
310 # find skip: the number of consecutive decorators
311 elif pat2.match(lines[line]):
312 try: code = getblock(lines[line:])
313 except TokenError: code = [lines[line]]
314 skip = 1
315 for _line in code[1:]: # skip lines that are decorators
316 if not pat2.match(_line): break
317 skip += 1
318 line += skip
319 # no match: reset skip and go to the next line
320 else:
321 line +=1
322 skip = 0
324 if not blocks:
325 blocks = [block]
326 _lnum = [lnum]
327 if lstrip: blocks = [_outdent(block) for block in blocks]
328 # return last match
329 return (blocks, _lnum) if locate is True else blocks
332def getsourcelines(object, lstrip=False, enclosing=False):
333 """Return a list of source lines and starting line number for an object.
334 Interactively-defined objects refer to lines in the interpreter's history.
336 The argument may be a module, class, method, function, traceback, frame,
337 or code object. The source code is returned as a list of the lines
338 corresponding to the object and the line number indicates where in the
339 original source file the first line of code was found. An IOError is
340 raised if the source code cannot be retrieved, while a TypeError is
341 raised for objects where the source code is unavailable (e.g. builtins).
343 If lstrip=True, ensure there is no indentation in the first line of code.
344 If enclosing=True, then also return any enclosing code."""
345 code, n = getblocks(object, lstrip=lstrip, enclosing=enclosing, locate=True)
346 return code[-1], n[-1]
349#NOTE: broke backward compatibility 4/16/14 (was lstrip=True, force=True)
350def getsource(object, alias='', lstrip=False, enclosing=False, \
351 force=False, builtin=False):
352 """Return the text of the source code for an object. The source code for
353 interactively-defined objects are extracted from the interpreter's history.
355 The argument may be a module, class, method, function, traceback, frame,
356 or code object. The source code is returned as a single string. An
357 IOError is raised if the source code cannot be retrieved, while a
358 TypeError is raised for objects where the source code is unavailable
359 (e.g. builtins).
361 If alias is provided, then add a line of code that renames the object.
362 If lstrip=True, ensure there is no indentation in the first line of code.
363 If enclosing=True, then also return any enclosing code.
364 If force=True, catch (TypeError,IOError) and try to use import hooks.
365 If builtin=True, force an import for any builtins
366 """
367 # hascode denotes a callable
368 hascode = _hascode(object)
369 # is a class instance type (and not in builtins)
370 instance = _isinstance(object)
372 # get source lines; if fail, try to 'force' an import
373 try: # fails for builtins, and other assorted object types
374 lines, lnum = getsourcelines(object, enclosing=enclosing)
375 except (TypeError, IOError): # failed to get source, resort to import hooks
376 if not force: # don't try to get types that findsource can't get
377 raise
378 if not getmodule(object): # get things like 'None' and '1'
379 if not instance: return getimport(object, alias, builtin=builtin)
380 # special handling (numpy arrays, ...)
381 _import = getimport(object, builtin=builtin)
382 name = getname(object, force=True)
383 _alias = "%s = " % alias if alias else ""
384 if alias == name: _alias = ""
385 return _import+_alias+"%s\n" % name
386 else: #FIXME: could use a good bit of cleanup, since using getimport...
387 if not instance: return getimport(object, alias, builtin=builtin)
388 # now we are dealing with an instance...
389 name = object.__class__.__name__
390 module = object.__module__
391 if not name.isidentifier() or not all(i.isidentifier() for i in module.split('.')): #XXX: does exclusing malicious code exclude valid use?
392 raise SyntaxError('invalid syntax')
393 if module in ['builtins','__builtin__']:
394 return getimport(object, alias, builtin=builtin)
395 else: #FIXME: leverage getimport? use 'from module import name'?
396 lines, lnum = ["%s = __import__('%s', fromlist=['%s']).%s\n" % (name,module,name,name)], 0
397 obj = eval(lines[0].lstrip(name + ' = '))
398 lines, lnum = getsourcelines(obj, enclosing=enclosing)
400 # strip leading indent (helps ensure can be imported)
401 if lstrip or alias:
402 lines = _outdent(lines)
404 # instantiate, if there's a nice repr #XXX: BAD IDEA???
405 if instance: #and force: #XXX: move into findsource or getsourcelines ?
406 if '(' in repr(object): lines.append('%r\n' % object)
407 #else: #XXX: better to somehow to leverage __reduce__ ?
408 # reconstructor,args = object.__reduce__()
409 # _ = reconstructor(*args)
410 else: # fall back to serialization #XXX: bad idea?
411 #XXX: better not duplicate work? #XXX: better new/enclose=True?
412 lines = dumpsource(object, alias='', new=force, enclose=False)
413 lines, lnum = [line+'\n' for line in lines.split('\n')][:-1], 0
414 #else: object.__code__ # raise AttributeError
416 # add an alias to the source code
417 if alias:
418 if hascode:
419 skip = 0
420 for line in lines: # skip lines that are decorators
421 if not line.startswith('@'): break
422 skip += 1
423 #XXX: use regex from findsource / getsourcelines ?
424 if lines[skip].lstrip().startswith('def '): # we have a function
425 if alias != object.__name__:
426 lines.append('\n%s = %s\n' % (alias, object.__name__))
427 elif 'lambda ' in lines[skip]: # we have a lambda
428 if alias != lines[skip].split('=')[0].strip():
429 lines[skip] = '%s = %s' % (alias, lines[skip])
430 else: # ...try to use the object's name
431 if alias != object.__name__:
432 lines.append('\n%s = %s\n' % (alias, object.__name__))
433 else: # class or class instance
434 if instance:
435 if alias != lines[-1].split('=')[0].strip():
436 lines[-1] = ('%s = ' % alias) + lines[-1]
437 else:
438 name = getname(object, force=True) or object.__name__
439 if alias != name:
440 lines.append('\n%s = %s\n' % (alias, name))
441 return ''.join(lines)
444def _hascode(object):
445 '''True if object has an attribute that stores it's __code__'''
446 return getattr(object,'__code__',None) or getattr(object,'func_code',None)
448def _isinstance(object):
449 '''True if object is a class instance type (and is not a builtin)'''
450 if _hascode(object) or isclass(object) or ismodule(object):
451 return False
452 if istraceback(object) or isframe(object) or iscode(object):
453 return False
454 # special handling (numpy arrays, ...)
455 if not getmodule(object) and getmodule(type(object)).__name__ in ['numpy']:
456 return True
457# # check if is instance of a builtin
458# if not getmodule(object) and getmodule(type(object)).__name__ in ['__builtin__','builtins']:
459# return False
460 _types = ('<class ',"<type 'instance'>")
461 if not repr(type(object)).startswith(_types): #FIXME: weak hack
462 return False
463 if not getmodule(object) or object.__module__ in ['builtins','__builtin__'] or getname(object, force=True) in ['array']:
464 return False
465 return True # by process of elimination... it's what we want
468def _intypes(object):
469 '''check if object is in the 'types' module'''
470 import types
471 # allow user to pass in object or object.__name__
472 if type(object) is not type(''):
473 object = getname(object, force=True)
474 if object == 'ellipsis': object = 'EllipsisType'
475 return True if hasattr(types, object) else False
478def _isstring(object): #XXX: isstringlike better?
479 '''check if object is a string-like type'''
480 return isinstance(object, (str, bytes))
483def indent(code, spaces=4):
484 '''indent a block of code with whitespace (default is 4 spaces)'''
485 indent = indentsize(code)
486 from numbers import Integral
487 if isinstance(spaces, Integral): spaces = ' '*spaces
488 # if '\t' is provided, will indent with a tab
489 nspaces = indentsize(spaces)
490 # blank lines (etc) need to be ignored
491 lines = code.split('\n')
492## stq = "'''"; dtq = '"""'
493## in_stq = in_dtq = False
494 for i in range(len(lines)):
495 #FIXME: works... but shouldn't indent 2nd+ lines of multiline doc
496 _indent = indentsize(lines[i])
497 if indent > _indent: continue
498 lines[i] = spaces+lines[i]
499## #FIXME: may fail when stq and dtq in same line (depends on ordering)
500## nstq, ndtq = lines[i].count(stq), lines[i].count(dtq)
501## if not in_dtq and not in_stq:
502## lines[i] = spaces+lines[i] # we indent
503## # entering a comment block
504## if nstq%2: in_stq = not in_stq
505## if ndtq%2: in_dtq = not in_dtq
506## # leaving a comment block
507## elif in_dtq and ndtq%2: in_dtq = not in_dtq
508## elif in_stq and nstq%2: in_stq = not in_stq
509## else: pass
510 if lines[-1].strip() == '': lines[-1] = ''
511 return '\n'.join(lines)
514def _outdent(lines, spaces=None, all=True):
515 '''outdent lines of code, accounting for docs and line continuations'''
516 indent = indentsize(lines[0])
517 if spaces is None or spaces > indent or spaces < 0: spaces = indent
518 for i in range(len(lines) if all else 1):
519 #FIXME: works... but shouldn't outdent 2nd+ lines of multiline doc
520 _indent = indentsize(lines[i])
521 if spaces > _indent: _spaces = _indent
522 else: _spaces = spaces
523 lines[i] = lines[i][_spaces:]
524 return lines
526def outdent(code, spaces=None, all=True):
527 '''outdent a block of code (default is to strip all leading whitespace)'''
528 indent = indentsize(code)
529 if spaces is None or spaces > indent or spaces < 0: spaces = indent
530 #XXX: will this delete '\n' in some cases?
531 if not all: return code[spaces:]
532 return '\n'.join(_outdent(code.split('\n'), spaces=spaces, all=all))
535# _wrap provides an wrapper to correctly exec and load into locals
536__globals__ = globals()
537__locals__ = locals()
538def _wrap(f):
539 """ encapsulate a function and it's __import__ """
540 def func(*args, **kwds):
541 try:
542 # _ = eval(getsource(f, force=True)) #XXX: safer but less robust
543 exec(getimportable(f, alias='_'), __globals__, __locals__)
544 except Exception:
545 raise ImportError('cannot import name ' + f.__name__)
546 return _(*args, **kwds)
547 func.__name__ = f.__name__
548 func.__doc__ = f.__doc__
549 return func
552def _enclose(object, alias=''): #FIXME: needs alias to hold returned object
553 """create a function enclosure around the source of some object"""
554 #XXX: dummy and stub should append a random string
555 dummy = '__this_is_a_big_dummy_enclosing_function__'
556 stub = '__this_is_a_stub_variable__'
557 code = 'def %s():\n' % dummy
558 code += indent(getsource(object, alias=stub, lstrip=True, force=True))
559 code += indent('return %s\n' % stub)
560 if alias: code += '%s = ' % alias
561 code += '%s(); del %s\n' % (dummy, dummy)
562 #code += "globals().pop('%s',lambda :None)()\n" % dummy
563 return code
566def dumpsource(object, alias='', new=False, enclose=True):
567 """'dump to source', where the code includes a pickled object.
569 If new=True and object is a class instance, then create a new
570 instance using the unpacked class source code. If enclose, then
571 create the object inside a function enclosure (thus minimizing
572 any global namespace pollution).
573 """
574 from dill import dumps
575 pik = repr(dumps(object))
576 code = 'import dill\n'
577 if enclose:
578 stub = '__this_is_a_stub_variable__' #XXX: *must* be same _enclose.stub
579 pre = '%s = ' % stub
580 new = False #FIXME: new=True doesn't work with enclose=True
581 else:
582 stub = alias
583 pre = '%s = ' % stub if alias else alias
585 # if a 'new' instance is not needed, then just dump and load
586 if not new or not _isinstance(object):
587 code += pre + 'dill.loads(%s)\n' % pik
588 else: #XXX: other cases where source code is needed???
589 code += getsource(object.__class__, alias='', lstrip=True, force=True)
590 mod = repr(object.__module__) # should have a module (no builtins here)
591 code += pre + 'dill.loads(%s.replace(b%s,bytes(__name__,"UTF-8")))\n' % (pik,mod)
592 #code += 'del %s' % object.__class__.__name__ #NOTE: kills any existing!
594 if enclose:
595 # generation of the 'enclosure'
596 dummy = '__this_is_a_big_dummy_object__'
597 dummy = _enclose(dummy, alias=alias)
598 # hack to replace the 'dummy' with the 'real' code
599 dummy = dummy.split('\n')
600 code = dummy[0]+'\n' + indent(code) + '\n'.join(dummy[-3:])
602 return code #XXX: better 'dumpsourcelines', returning list of lines?
605def getname(obj, force=False, fqn=False): #XXX: throw(?) to raise error on fail?
606 """get the name of the object. for lambdas, get the name of the pointer """
607 if fqn: return '.'.join(_namespace(obj)) #NOTE: returns 'type'
608 module = getmodule(obj)
609 if not module: # things like "None" and "1"
610 if not force: return None #NOTE: returns 'instance' NOT 'type' #FIXME?
611 # handle some special cases
612 if hasattr(obj, 'dtype') and not obj.shape:
613 return getname(obj.__class__) + "(" + repr(obj.tolist()) + ")"
614 return repr(obj)
615 try:
616 #XXX: 'wrong' for decorators and curried functions ?
617 # if obj.func_closure: ...use logic from getimportable, etc ?
618 name = obj.__name__
619 if name == '<lambda>':
620 return getsource(obj).split('=',1)[0].strip()
621 # handle some special cases
622 if module.__name__ in ['builtins','__builtin__']:
623 if name == 'ellipsis': name = 'EllipsisType'
624 return name
625 except AttributeError: #XXX: better to just throw AttributeError ?
626 if not force: return None
627 name = repr(obj)
628 if name.startswith('<'): # or name.split('('):
629 return None
630 return name
633def _namespace(obj):
634 """_namespace(obj); return namespace hierarchy (as a list of names)
635 for the given object. For an instance, find the class hierarchy.
637 For example:
639 >>> from functools import partial
640 >>> p = partial(int, base=2)
641 >>> _namespace(p)
642 [\'functools\', \'partial\']
643 """
644 # mostly for functions and modules and such
645 #FIXME: 'wrong' for decorators and curried functions
646 try: #XXX: needs some work and testing on different types
647 module = qual = str(getmodule(obj)).split()[1].strip('>').strip('"').strip("'")
648 qual = qual.split('.')
649 if ismodule(obj):
650 return qual
651 # get name of a lambda, function, etc
652 name = getname(obj) or obj.__name__ # failing, raise AttributeError
653 # check special cases (NoneType, ...)
654 if module in ['builtins','__builtin__']: # BuiltinFunctionType
655 if _intypes(name): return ['types'] + [name]
656 return qual + [name] #XXX: can be wrong for some aliased objects
657 except Exception: pass
658 # special case: numpy.inf and numpy.nan (we don't want them as floats)
659 if str(obj) in ['inf','nan','Inf','NaN']: # is more, but are they needed?
660 return ['numpy'] + [str(obj)]
661 # mostly for classes and class instances and such
662 module = getattr(obj.__class__, '__module__', None)
663 qual = str(obj.__class__)
664 try: qual = qual[qual.index("'")+1:-2]
665 except ValueError: pass # str(obj.__class__) made the 'try' unnecessary
666 qual = qual.split(".")
667 if module in ['builtins','__builtin__']:
668 # check special cases (NoneType, Ellipsis, ...)
669 if qual[-1] == 'ellipsis': qual[-1] = 'EllipsisType'
670 if _intypes(qual[-1]): module = 'types' #XXX: BuiltinFunctionType
671 qual = [module] + qual
672 return qual
675#NOTE: 05/25/14 broke backward compatibility: added 'alias' as 3rd argument
676def _getimport(head, tail, alias='', verify=True, builtin=False):
677 """helper to build a likely import string from head and tail of namespace.
678 ('head','tail') are used in the following context: "from head import tail"
680 If verify=True, then test the import string before returning it.
681 If builtin=True, then force an import for builtins where possible.
682 If alias is provided, then rename the object on import.
683 """
684 # special handling for a few common types
685 if tail in ['Ellipsis', 'NotImplemented'] and head in ['types']:
686 head = len.__module__
687 elif tail in ['None'] and head in ['types']:
688 _alias = '%s = ' % alias if alias else ''
689 if alias == tail: _alias = ''
690 return _alias+'%s\n' % tail
691 # we don't need to import from builtins, so return ''
692# elif tail in ['NoneType','int','float','long','complex']: return '' #XXX: ?
693 if head in ['builtins','__builtin__']:
694 # special cases (NoneType, Ellipsis, ...) #XXX: BuiltinFunctionType
695 if tail == 'ellipsis': tail = 'EllipsisType'
696 if _intypes(tail): head = 'types'
697 elif not builtin:
698 _alias = '%s = ' % alias if alias else ''
699 if alias == tail: _alias = ''
700 return _alias+'%s\n' % tail
701 else: pass # handle builtins below
702 # get likely import string
703 if not head: _str = "import %s" % tail
704 else: _str = "from %s import %s" % (head, tail)
705 _alias = " as %s\n" % alias if alias else "\n"
706 if alias == tail: _alias = "\n"
707 _str += _alias
708 # FIXME: fails on most decorators, currying, and such...
709 # (could look for magic __wrapped__ or __func__ attr)
710 # (could fix in 'namespace' to check obj for closure)
711 if verify and not head.startswith('dill.'):# weird behavior for dill
712 #print(_str)
713 try: exec(_str) #XXX: check if == obj? (name collision)
714 except ImportError: #XXX: better top-down or bottom-up recursion?
715 _head = head.rsplit(".",1)[0] #(or get all, then compare == obj?)
716 if not _head: raise
717 if _head != head:
718 _str = _getimport(_head, tail, alias, verify)
719 return _str
722#XXX: rename builtin to force? vice versa? verify to force? (as in getsource)
723#NOTE: 05/25/14 broke backward compatibility: added 'alias' as 2nd argument
724def getimport(obj, alias='', verify=True, builtin=False, enclosing=False):
725 """get the likely import string for the given object
727 obj is the object to inspect
728 If verify=True, then test the import string before returning it.
729 If builtin=True, then force an import for builtins where possible.
730 If enclosing=True, get the import for the outermost enclosing callable.
731 If alias is provided, then rename the object on import.
732 """
733 if enclosing:
734 from .detect import outermost
735 _obj = outermost(obj)
736 obj = _obj if _obj else obj
737 # get the namespace
738 qual = _namespace(obj)
739 head = '.'.join(qual[:-1])
740 tail = qual[-1]
741 # for named things... with a nice repr #XXX: move into _namespace?
742 try: # look for '<...>' and be mindful it might be in lists, dicts, etc...
743 name = repr(obj).split('<',1)[1].split('>',1)[1]
744 name = None # we have a 'object'-style repr
745 except Exception: # it's probably something 'importable'
746 if head in ['builtins','__builtin__']:
747 name = repr(obj) #XXX: catch [1,2], (1,2), set([1,2])... others?
748 elif _isinstance(obj):
749 name = getname(obj, force=True).split('(')[0]
750 else:
751 name = repr(obj).split('(')[0]
752 #if not repr(obj).startswith('<'): name = repr(obj).split('(')[0]
753 #else: name = None
754 if name: # try using name instead of tail
755 try: return _getimport(head, name, alias, verify, builtin)
756 except ImportError: pass
757 except SyntaxError:
758 if head in ['builtins','__builtin__']:
759 _alias = '%s = ' % alias if alias else ''
760 if alias == name: _alias = ''
761 return _alias+'%s\n' % name
762 else: pass
763 try:
764 #if type(obj) is type(abs): _builtin = builtin # BuiltinFunctionType
765 #else: _builtin = False
766 return _getimport(head, tail, alias, verify, builtin)
767 except ImportError:
768 raise # could do some checking against obj
769 except SyntaxError:
770 if head in ['builtins','__builtin__']:
771 _alias = '%s = ' % alias if alias else ''
772 if alias == tail: _alias = ''
773 return _alias+'%s\n' % tail
774 raise # could do some checking against obj
777def _importable(obj, alias='', source=None, enclosing=False, force=True, \
778 builtin=True, lstrip=True):
779 """get an import string (or the source code) for the given object
781 This function will attempt to discover the name of the object, or the repr
782 of the object, or the source code for the object. To attempt to force
783 discovery of the source code, use source=True, to attempt to force the
784 use of an import, use source=False; otherwise an import will be sought
785 for objects not defined in __main__. The intent is to build a string
786 that can be imported from a python file. obj is the object to inspect.
787 If alias is provided, then rename the object with the given alias.
789 If source=True, use these options:
790 If enclosing=True, then also return any enclosing code.
791 If force=True, catch (TypeError,IOError) and try to use import hooks.
792 If lstrip=True, ensure there is no indentation in the first line of code.
794 If source=False, use these options:
795 If enclosing=True, get the import for the outermost enclosing callable.
796 If force=True, then don't test the import string before returning it.
797 If builtin=True, then force an import for builtins where possible.
798 """
799 if source is None:
800 source = True if isfrommain(obj) else False
801 if source: # first try to get the source
802 try:
803 return getsource(obj, alias, enclosing=enclosing, \
804 force=force, lstrip=lstrip, builtin=builtin)
805 except Exception: pass
806 try:
807 if not _isinstance(obj):
808 return getimport(obj, alias, enclosing=enclosing, \
809 verify=(not force), builtin=builtin)
810 # first 'get the import', then 'get the instance'
811 _import = getimport(obj, enclosing=enclosing, \
812 verify=(not force), builtin=builtin)
813 name = getname(obj, force=True)
814 if not name:
815 raise AttributeError("object has no atribute '__name__'")
816 _alias = "%s = " % alias if alias else ""
817 if alias == name: _alias = ""
818 return _import+_alias+"%s\n" % name
820 except Exception: pass
821 if not source: # try getsource, only if it hasn't been tried yet
822 try:
823 return getsource(obj, alias, enclosing=enclosing, \
824 force=force, lstrip=lstrip, builtin=builtin)
825 except Exception: pass
826 # get the name (of functions, lambdas, and classes)
827 # or hope that obj can be built from the __repr__
828 #XXX: what to do about class instances and such?
829 obj = getname(obj, force=force)
830 # we either have __repr__ or __name__ (or None)
831 if not obj or obj.startswith('<'):
832 raise AttributeError("object has no atribute '__name__'")
833 _alias = '%s = ' % alias if alias else ''
834 if alias == obj: _alias = ''
835 return _alias+'%s\n' % obj
836 #XXX: possible failsafe... (for example, for instances when source=False)
837 # "import dill; result = dill.loads(<pickled_object>); # repr(<object>)"
839def _closuredimport(func, alias='', builtin=False):
840 """get import for closured objects; return a dict of 'name' and 'import'"""
841 import re
842 from .detect import freevars, outermost
843 free_vars = freevars(func)
844 func_vars = {}
845 # split into 'funcs' and 'non-funcs'
846 for name,obj in list(free_vars.items()):
847 if not isfunction(obj): continue
848 # get import for 'funcs'
849 fobj = free_vars.pop(name)
850 src = getsource(fobj)
851 if src.lstrip().startswith('@'): # we have a decorator
852 src = getimport(fobj, alias=alias, builtin=builtin)
853 else: # we have to "hack" a bit... and maybe be lucky
854 encl = outermost(func)
855 # pattern: 'func = enclosing(fobj'
856 pat = r'.*[\w\s]=\s*'+getname(encl)+r'\('+getname(fobj)
857 mod = getname(getmodule(encl))
858 #HACK: get file containing 'outer' function; is func there?
859 lines,_ = findsource(encl)
860 candidate = [line for line in lines if getname(encl) in line and \
861 re.match(pat, line)]
862 if not candidate:
863 mod = getname(getmodule(fobj))
864 #HACK: get file containing 'inner' function; is func there?
865 lines,_ = findsource(fobj)
866 candidate = [line for line in lines \
867 if getname(fobj) in line and re.match(pat, line)]
868 if not len(candidate): raise TypeError('import could not be found')
869 candidate = candidate[-1]
870 name = candidate.split('=',1)[0].split()[-1].strip()
871 src = _getimport(mod, name, alias=alias, builtin=builtin)
872 func_vars[name] = src
873 if not func_vars:
874 name = outermost(func)
875 mod = getname(getmodule(name))
876 if not mod or name is func: # then it can be handled by getimport
877 name = getname(func, force=True) #XXX: better key?
878 src = getimport(func, alias=alias, builtin=builtin)
879 else:
880 lines,_ = findsource(name)
881 # pattern: 'func = enclosing('
882 candidate = [line for line in lines if getname(name) in line and \
883 re.match(r'.*[\w\s]=\s*'+getname(name)+r'\(', line)]
884 if not len(candidate): raise TypeError('import could not be found')
885 candidate = candidate[-1]
886 name = candidate.split('=',1)[0].split()[-1].strip()
887 src = _getimport(mod, name, alias=alias, builtin=builtin)
888 func_vars[name] = src
889 return func_vars
891#XXX: should be able to use __qualname__
892def _closuredsource(func, alias=''):
893 """get source code for closured objects; return a dict of 'name'
894 and 'code blocks'"""
895 #FIXME: this entire function is a messy messy HACK
896 # - pollutes global namespace
897 # - fails if name of freevars are reused
898 # - can unnecessarily duplicate function code
899 from .detect import freevars
900 free_vars = freevars(func)
901 func_vars = {}
902 # split into 'funcs' and 'non-funcs'
903 for name,obj in list(free_vars.items()):
904 if not isfunction(obj):
905 # get source for 'non-funcs'
906 free_vars[name] = getsource(obj, force=True, alias=name)
907 continue
908 # get source for 'funcs'
909 fobj = free_vars.pop(name)
910 src = getsource(fobj, alias) # DO NOT include dependencies
911 # if source doesn't start with '@', use name as the alias
912 if not src.lstrip().startswith('@'): #FIXME: 'enclose' in dummy;
913 src = importable(fobj,alias=name)# wrong ref 'name'
914 org = getsource(func, alias, enclosing=False, lstrip=True)
915 src = (src, org) # undecorated first, then target
916 else: #NOTE: reproduces the code!
917 org = getsource(func, enclosing=True, lstrip=False)
918 src = importable(fobj, alias, source=True) # include dependencies
919 src = (org, src) # target first, then decorated
920 func_vars[name] = src
921 src = ''.join(free_vars.values())
922 if not func_vars: #FIXME: 'enclose' in dummy; wrong ref 'name'
923 org = getsource(func, alias, force=True, enclosing=False, lstrip=True)
924 src = (src, org) # variables first, then target
925 else:
926 src = (src, None) # just variables (better '' instead of None?)
927 func_vars[None] = src
928 # FIXME: remove duplicates (however, order is important...)
929 return func_vars
931def importable(obj, alias='', source=None, builtin=True):
932 """get an importable string (i.e. source code or the import string)
933 for the given object, including any required objects from the enclosing
934 and global scope
936 This function will attempt to discover the name of the object, or the repr
937 of the object, or the source code for the object. To attempt to force
938 discovery of the source code, use source=True, to attempt to force the
939 use of an import, use source=False; otherwise an import will be sought
940 for objects not defined in __main__. The intent is to build a string
941 that can be imported from a python file.
943 obj is the object to inspect. If alias is provided, then rename the
944 object with the given alias. If builtin=True, then force an import for
945 builtins where possible.
946 """
947 #NOTE: we always 'force', and 'lstrip' as necessary
948 #NOTE: for 'enclosing', use importable(outermost(obj))
949 if source is None:
950 source = True if isfrommain(obj) else False
951 elif builtin and isbuiltin(obj):
952 source = False
953 tried_source = tried_import = False
954 while True:
955 if not source: # we want an import
956 try:
957 if _isinstance(obj): # for instances, punt to _importable
958 return _importable(obj, alias, source=False, builtin=builtin)
959 src = _closuredimport(obj, alias=alias, builtin=builtin)
960 if len(src) == 0:
961 raise NotImplementedError('not implemented')
962 if len(src) > 1:
963 raise NotImplementedError('not implemented')
964 return list(src.values())[0]
965 except Exception:
966 if tried_source: raise
967 tried_import = True
968 # we want the source
969 try:
970 src = _closuredsource(obj, alias=alias)
971 if len(src) == 0:
972 raise NotImplementedError('not implemented')
973 # groan... an inline code stitcher
974 def _code_stitcher(block):
975 "stitch together the strings in tuple 'block'"
976 if block[0] and block[-1]: block = '\n'.join(block)
977 elif block[0]: block = block[0]
978 elif block[-1]: block = block[-1]
979 else: block = ''
980 return block
981 # get free_vars first
982 _src = _code_stitcher(src.pop(None))
983 _src = [_src] if _src else []
984 # get func_vars
985 for xxx in src.values():
986 xxx = _code_stitcher(xxx)
987 if xxx: _src.append(xxx)
988 # make a single source string
989 if not len(_src):
990 src = ''
991 elif len(_src) == 1:
992 src = _src[0]
993 else:
994 src = '\n'.join(_src)
995 # get source code of objects referred to by obj in global scope
996 from .detect import globalvars
997 obj = globalvars(obj) #XXX: don't worry about alias? recurse? etc?
998 obj = list(getsource(_obj,name,force=True) for (name,_obj) in obj.items() if not isbuiltin(_obj))
999 obj = '\n'.join(obj) if obj else ''
1000 # combine all referred-to source (global then enclosing)
1001 if not obj: return src
1002 if not src: return obj
1003 return obj + src
1004 except Exception:
1005 if tried_import: raise
1006 tried_source = True
1007 source = not source
1008 # should never get here
1009 return
1012# backward compatibility
1013def getimportable(obj, alias='', byname=True, explicit=False):
1014 return importable(obj,alias,source=(not byname),builtin=explicit)
1015 #return outdent(_importable(obj,alias,source=(not byname),builtin=explicit))
1016def likely_import(obj, passive=False, explicit=False):
1017 return getimport(obj, verify=(not passive), builtin=explicit)
1018def _likely_import(first, last, passive=False, explicit=True):
1019 return _getimport(first, last, verify=(not passive), builtin=explicit)
1020_get_name = getname
1021getblocks_from_history = getblocks
1025# EOF