Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""Implementation of code management magic functions.
2"""
3#-----------------------------------------------------------------------------
4# Copyright (c) 2012 The IPython Development Team.
5#
6# Distributed under the terms of the Modified BSD License.
7#
8# The full license is in the file COPYING.txt, distributed with this software.
9#-----------------------------------------------------------------------------
11#-----------------------------------------------------------------------------
12# Imports
13#-----------------------------------------------------------------------------
15# Stdlib
16import inspect
17import io
18import os
19import re
20import sys
21import ast
22from itertools import chain
23from urllib.request import Request, urlopen
24from urllib.parse import urlencode
25from pathlib import Path
27# Our own packages
28from IPython.core.error import TryNext, StdinNotImplementedError, UsageError
29from IPython.core.macro import Macro
30from IPython.core.magic import Magics, magics_class, line_magic
31from IPython.core.oinspect import find_file, find_source_lines
32from IPython.core.release import version
33from IPython.testing.skipdoctest import skip_doctest
34from IPython.utils.contexts import preserve_keys
35from IPython.utils.path import get_py_filename
36from warnings import warn
37from logging import error
38from IPython.utils.text import get_text_list
40#-----------------------------------------------------------------------------
41# Magic implementation classes
42#-----------------------------------------------------------------------------
44# Used for exception handling in magic_edit
45class MacroToEdit(ValueError): pass
47ipython_input_pat = re.compile(r"<ipython\-input\-(\d+)-[a-z\d]+>$")
49# To match, e.g. 8-10 1:5 :10 3-
50range_re = re.compile(r"""
51(?P<start>\d+)?
52((?P<sep>[\-:])
53 (?P<end>\d+)?)?
54$""", re.VERBOSE)
57def extract_code_ranges(ranges_str):
58 """Turn a string of range for %%load into 2-tuples of (start, stop)
59 ready to use as a slice of the content split by lines.
61 Examples
62 --------
63 list(extract_input_ranges("5-10 2"))
64 [(4, 10), (1, 2)]
65 """
66 for range_str in ranges_str.split():
67 rmatch = range_re.match(range_str)
68 if not rmatch:
69 continue
70 sep = rmatch.group("sep")
71 start = rmatch.group("start")
72 end = rmatch.group("end")
74 if sep == '-':
75 start = int(start) - 1 if start else None
76 end = int(end) if end else None
77 elif sep == ':':
78 start = int(start) - 1 if start else None
79 end = int(end) - 1 if end else None
80 else:
81 end = int(start)
82 start = int(start) - 1
83 yield (start, end)
86def extract_symbols(code, symbols):
87 """
88 Return a tuple (blocks, not_found)
89 where ``blocks`` is a list of code fragments
90 for each symbol parsed from code, and ``not_found`` are
91 symbols not found in the code.
93 For example::
95 In [1]: code = '''a = 10
96 ...: def b(): return 42
97 ...: class A: pass'''
99 In [2]: extract_symbols(code, 'A,b,z')
100 Out[2]: (['class A: pass\\n', 'def b(): return 42\\n'], ['z'])
101 """
102 symbols = symbols.split(',')
104 # this will raise SyntaxError if code isn't valid Python
105 py_code = ast.parse(code)
107 marks = [(getattr(s, 'name', None), s.lineno) for s in py_code.body]
108 code = code.split('\n')
110 symbols_lines = {}
112 # we already know the start_lineno of each symbol (marks).
113 # To find each end_lineno, we traverse in reverse order until each
114 # non-blank line
115 end = len(code)
116 for name, start in reversed(marks):
117 while not code[end - 1].strip():
118 end -= 1
119 if name:
120 symbols_lines[name] = (start - 1, end)
121 end = start - 1
123 # Now symbols_lines is a map
124 # {'symbol_name': (start_lineno, end_lineno), ...}
126 # fill a list with chunks of codes for each requested symbol
127 blocks = []
128 not_found = []
129 for symbol in symbols:
130 if symbol in symbols_lines:
131 start, end = symbols_lines[symbol]
132 blocks.append('\n'.join(code[start:end]) + '\n')
133 else:
134 not_found.append(symbol)
136 return blocks, not_found
138def strip_initial_indent(lines):
139 """For %load, strip indent from lines until finding an unindented line.
141 https://github.com/ipython/ipython/issues/9775
142 """
143 indent_re = re.compile(r'\s+')
145 it = iter(lines)
146 first_line = next(it)
147 indent_match = indent_re.match(first_line)
149 if indent_match:
150 # First line was indented
151 indent = indent_match.group()
152 yield first_line[len(indent):]
154 for line in it:
155 if line.startswith(indent):
156 yield line[len(indent) :]
157 elif line in ("\n", "\r\n") or len(line) == 0:
158 yield line
159 else:
160 # Less indented than the first line - stop dedenting
161 yield line
162 break
163 else:
164 yield first_line
166 # Pass the remaining lines through without dedenting
167 for line in it:
168 yield line
171class InteractivelyDefined(Exception):
172 """Exception for interactively defined variable in magic_edit"""
173 def __init__(self, index):
174 self.index = index
177@magics_class
178class CodeMagics(Magics):
179 """Magics related to code management (loading, saving, editing, ...)."""
181 def __init__(self, *args, **kwargs):
182 self._knowntemps = set()
183 super(CodeMagics, self).__init__(*args, **kwargs)
185 @line_magic
186 def save(self, parameter_s=''):
187 """Save a set of lines or a macro to a given filename.
189 Usage:\\
190 %save [options] filename [history]
192 Options:
194 -r: use 'raw' input. By default, the 'processed' history is used,
195 so that magics are loaded in their transformed version to valid
196 Python. If this option is given, the raw input as typed at the
197 command line is used instead.
199 -f: force overwrite. If file exists, %save will prompt for overwrite
200 unless -f is given.
202 -a: append to the file instead of overwriting it.
204 The history argument uses the same syntax as %history for input ranges,
205 then saves the lines to the filename you specify.
207 If no ranges are specified, saves history of the current session up to
208 this point.
210 It adds a '.py' extension to the file if you don't do so yourself, and
211 it asks for confirmation before overwriting existing files.
213 If `-r` option is used, the default extension is `.ipy`.
214 """
216 opts,args = self.parse_options(parameter_s,'fra',mode='list')
217 if not args:
218 raise UsageError('Missing filename.')
219 raw = 'r' in opts
220 force = 'f' in opts
221 append = 'a' in opts
222 mode = 'a' if append else 'w'
223 ext = '.ipy' if raw else '.py'
224 fname, codefrom = args[0], " ".join(args[1:])
225 if not fname.endswith(('.py','.ipy')):
226 fname += ext
227 fname = os.path.expanduser(fname)
228 file_exists = os.path.isfile(fname)
229 if file_exists and not force and not append:
230 try:
231 overwrite = self.shell.ask_yes_no('File `%s` exists. Overwrite (y/[N])? ' % fname, default='n')
232 except StdinNotImplementedError:
233 print("File `%s` exists. Use `%%save -f %s` to force overwrite" % (fname, parameter_s))
234 return
235 if not overwrite :
236 print('Operation cancelled.')
237 return
238 try:
239 cmds = self.shell.find_user_code(codefrom,raw)
240 except (TypeError, ValueError) as e:
241 print(e.args[0])
242 return
243 with io.open(fname, mode, encoding="utf-8") as f:
244 if not file_exists or not append:
245 f.write("# coding: utf-8\n")
246 f.write(cmds)
247 # make sure we end on a newline
248 if not cmds.endswith('\n'):
249 f.write('\n')
250 print('The following commands were written to file `%s`:' % fname)
251 print(cmds)
253 @line_magic
254 def pastebin(self, parameter_s=''):
255 """Upload code to dpaste.com, returning the URL.
257 Usage:\\
258 %pastebin [-d "Custom description"][-e 24] 1-7
260 The argument can be an input history range, a filename, or the name of a
261 string or macro.
263 If no arguments are given, uploads the history of this session up to
264 this point.
266 Options:
268 -d: Pass a custom description. The default will say
269 "Pasted from IPython".
270 -e: Pass number of days for the link to be expired.
271 The default will be 7 days.
272 """
273 opts, args = self.parse_options(parameter_s, "d:e:")
275 try:
276 code = self.shell.find_user_code(args)
277 except (ValueError, TypeError) as e:
278 print(e.args[0])
279 return
281 expiry_days = 7
282 try:
283 expiry_days = int(opts.get("e", 7))
284 except ValueError as e:
285 print(e.args[0].capitalize())
286 return
287 if expiry_days < 1 or expiry_days > 365:
288 print("Expiry days should be in range of 1 to 365")
289 return
291 post_data = urlencode(
292 {
293 "title": opts.get("d", "Pasted from IPython"),
294 "syntax": "python",
295 "content": code,
296 "expiry_days": expiry_days,
297 }
298 ).encode("utf-8")
300 request = Request(
301 "https://dpaste.com/api/v2/",
302 headers={"User-Agent": "IPython v{}".format(version)},
303 )
304 response = urlopen(request, post_data)
305 return response.headers.get('Location')
307 @line_magic
308 def loadpy(self, arg_s):
309 """Alias of `%load`
311 `%loadpy` has gained some flexibility and dropped the requirement of a `.py`
312 extension. So it has been renamed simply into %load. You can look at
313 `%load`'s docstring for more info.
314 """
315 self.load(arg_s)
317 @line_magic
318 def load(self, arg_s):
319 """Load code into the current frontend.
321 Usage:\\
322 %load [options] source
324 where source can be a filename, URL, input history range, macro, or
325 element in the user namespace
327 If no arguments are given, loads the history of this session up to this
328 point.
330 Options:
332 -r <lines>: Specify lines or ranges of lines to load from the source.
333 Ranges could be specified as x-y (x..y) or in python-style x:y
334 (x..(y-1)). Both limits x and y can be left blank (meaning the
335 beginning and end of the file, respectively).
337 -s <symbols>: Specify function or classes to load from python source.
339 -y : Don't ask confirmation for loading source above 200 000 characters.
341 -n : Include the user's namespace when searching for source code.
343 This magic command can either take a local filename, a URL, an history
344 range (see %history) or a macro as argument, it will prompt for
345 confirmation before loading source with more than 200 000 characters, unless
346 -y flag is passed or if the frontend does not support raw_input::
348 %load
349 %load myscript.py
350 %load 7-27
351 %load myMacro
352 %load http://www.example.com/myscript.py
353 %load -r 5-10 myscript.py
354 %load -r 10-20,30,40: foo.py
355 %load -s MyClass,wonder_function myscript.py
356 %load -n MyClass
357 %load -n my_module.wonder_function
358 """
359 opts,args = self.parse_options(arg_s,'yns:r:')
360 search_ns = 'n' in opts
361 contents = self.shell.find_user_code(args, search_ns=search_ns)
363 if 's' in opts:
364 try:
365 blocks, not_found = extract_symbols(contents, opts['s'])
366 except SyntaxError:
367 # non python code
368 error("Unable to parse the input as valid Python code")
369 return
371 if len(not_found) == 1:
372 warn('The symbol `%s` was not found' % not_found[0])
373 elif len(not_found) > 1:
374 warn('The symbols %s were not found' % get_text_list(not_found,
375 wrap_item_with='`')
376 )
378 contents = '\n'.join(blocks)
380 if 'r' in opts:
381 ranges = opts['r'].replace(',', ' ')
382 lines = contents.split('\n')
383 slices = extract_code_ranges(ranges)
384 contents = [lines[slice(*slc)] for slc in slices]
385 contents = '\n'.join(strip_initial_indent(chain.from_iterable(contents)))
387 l = len(contents)
389 # 200 000 is ~ 2500 full 80 character lines
390 # so in average, more than 5000 lines
391 if l > 200000 and 'y' not in opts:
392 try:
393 ans = self.shell.ask_yes_no(("The text you're trying to load seems pretty big"\
394 " (%d characters). Continue (y/[N]) ?" % l), default='n' )
395 except StdinNotImplementedError:
396 #assume yes if raw input not implemented
397 ans = True
399 if ans is False :
400 print('Operation cancelled.')
401 return
403 contents = "# %load {}\n".format(arg_s) + contents
405 self.shell.set_next_input(contents, replace=True)
407 @staticmethod
408 def _find_edit_target(shell, args, opts, last_call):
409 """Utility method used by magic_edit to find what to edit."""
411 def make_filename(arg):
412 "Make a filename from the given args"
413 try:
414 filename = get_py_filename(arg)
415 except IOError:
416 # If it ends with .py but doesn't already exist, assume we want
417 # a new file.
418 if arg.endswith('.py'):
419 filename = arg
420 else:
421 filename = None
422 return filename
424 # Set a few locals from the options for convenience:
425 opts_prev = 'p' in opts
426 opts_raw = 'r' in opts
428 # custom exceptions
429 class DataIsObject(Exception): pass
431 # Default line number value
432 lineno = opts.get('n',None)
434 if opts_prev:
435 args = '_%s' % last_call[0]
436 if args not in shell.user_ns:
437 args = last_call[1]
439 # by default this is done with temp files, except when the given
440 # arg is a filename
441 use_temp = True
443 data = ''
445 # First, see if the arguments should be a filename.
446 filename = make_filename(args)
447 if filename:
448 use_temp = False
449 elif args:
450 # Mode where user specifies ranges of lines, like in %macro.
451 data = shell.extract_input_lines(args, opts_raw)
452 if not data:
453 try:
454 # Load the parameter given as a variable. If not a string,
455 # process it as an object instead (below)
457 # print('*** args',args,'type',type(args)) # dbg
458 data = eval(args, shell.user_ns)
459 if not isinstance(data, str):
460 raise DataIsObject
462 except (NameError,SyntaxError):
463 # given argument is not a variable, try as a filename
464 filename = make_filename(args)
465 if filename is None:
466 warn("Argument given (%s) can't be found as a variable "
467 "or as a filename." % args)
468 return (None, None, None)
469 use_temp = False
471 except DataIsObject as e:
472 # macros have a special edit function
473 if isinstance(data, Macro):
474 raise MacroToEdit(data) from e
476 # For objects, try to edit the file where they are defined
477 filename = find_file(data)
478 if filename:
479 if 'fakemodule' in filename.lower() and \
480 inspect.isclass(data):
481 # class created by %edit? Try to find source
482 # by looking for method definitions instead, the
483 # __module__ in those classes is FakeModule.
484 attrs = [getattr(data, aname) for aname in dir(data)]
485 for attr in attrs:
486 if not inspect.ismethod(attr):
487 continue
488 filename = find_file(attr)
489 if filename and \
490 'fakemodule' not in filename.lower():
491 # change the attribute to be the edit
492 # target instead
493 data = attr
494 break
496 m = ipython_input_pat.match(os.path.basename(filename))
497 if m:
498 raise InteractivelyDefined(int(m.groups()[0])) from e
500 datafile = 1
501 if filename is None:
502 filename = make_filename(args)
503 datafile = 1
504 if filename is not None:
505 # only warn about this if we get a real name
506 warn('Could not find file where `%s` is defined.\n'
507 'Opening a file named `%s`' % (args, filename))
508 # Now, make sure we can actually read the source (if it was
509 # in a temp file it's gone by now).
510 if datafile:
511 if lineno is None:
512 lineno = find_source_lines(data)
513 if lineno is None:
514 filename = make_filename(args)
515 if filename is None:
516 warn('The file where `%s` was defined '
517 'cannot be read or found.' % data)
518 return (None, None, None)
519 use_temp = False
521 if use_temp:
522 filename = shell.mktempfile(data)
523 print('IPython will make a temporary file named:',filename)
525 # use last_call to remember the state of the previous call, but don't
526 # let it be clobbered by successive '-p' calls.
527 try:
528 last_call[0] = shell.displayhook.prompt_count
529 if not opts_prev:
530 last_call[1] = args
531 except:
532 pass
535 return filename, lineno, use_temp
537 def _edit_macro(self,mname,macro):
538 """open an editor with the macro data in a file"""
539 filename = self.shell.mktempfile(macro.value)
540 self.shell.hooks.editor(filename)
542 # and make a new macro object, to replace the old one
543 mvalue = Path(filename).read_text(encoding="utf-8")
544 self.shell.user_ns[mname] = Macro(mvalue)
546 @skip_doctest
547 @line_magic
548 def edit(self, parameter_s='',last_call=['','']):
549 """Bring up an editor and execute the resulting code.
551 Usage:
552 %edit [options] [args]
554 %edit runs IPython's editor hook. The default version of this hook is
555 set to call the editor specified by your $EDITOR environment variable.
556 If this isn't found, it will default to vi under Linux/Unix and to
557 notepad under Windows. See the end of this docstring for how to change
558 the editor hook.
560 You can also set the value of this editor via the
561 ``TerminalInteractiveShell.editor`` option in your configuration file.
562 This is useful if you wish to use a different editor from your typical
563 default with IPython (and for Windows users who typically don't set
564 environment variables).
566 This command allows you to conveniently edit multi-line code right in
567 your IPython session.
569 If called without arguments, %edit opens up an empty editor with a
570 temporary file and will execute the contents of this file when you
571 close it (don't forget to save it!).
574 Options:
576 -n <number>: open the editor at a specified line number. By default,
577 the IPython editor hook uses the unix syntax 'editor +N filename', but
578 you can configure this by providing your own modified hook if your
579 favorite editor supports line-number specifications with a different
580 syntax.
582 -p: this will call the editor with the same data as the previous time
583 it was used, regardless of how long ago (in your current session) it
584 was.
586 -r: use 'raw' input. This option only applies to input taken from the
587 user's history. By default, the 'processed' history is used, so that
588 magics are loaded in their transformed version to valid Python. If
589 this option is given, the raw input as typed as the command line is
590 used instead. When you exit the editor, it will be executed by
591 IPython's own processor.
593 -x: do not execute the edited code immediately upon exit. This is
594 mainly useful if you are editing programs which need to be called with
595 command line arguments, which you can then do using %run.
598 Arguments:
600 If arguments are given, the following possibilities exist:
602 - If the argument is a filename, IPython will load that into the
603 editor. It will execute its contents with execfile() when you exit,
604 loading any code in the file into your interactive namespace.
606 - The arguments are ranges of input history, e.g. "7 ~1/4-6".
607 The syntax is the same as in the %history magic.
609 - If the argument is a string variable, its contents are loaded
610 into the editor. You can thus edit any string which contains
611 python code (including the result of previous edits).
613 - If the argument is the name of an object (other than a string),
614 IPython will try to locate the file where it was defined and open the
615 editor at the point where it is defined. You can use `%edit function`
616 to load an editor exactly at the point where 'function' is defined,
617 edit it and have the file be executed automatically.
619 - If the object is a macro (see %macro for details), this opens up your
620 specified editor with a temporary file containing the macro's data.
621 Upon exit, the macro is reloaded with the contents of the file.
623 Note: opening at an exact line is only supported under Unix, and some
624 editors (like kedit and gedit up to Gnome 2.8) do not understand the
625 '+NUMBER' parameter necessary for this feature. Good editors like
626 (X)Emacs, vi, jed, pico and joe all do.
628 After executing your code, %edit will return as output the code you
629 typed in the editor (except when it was an existing file). This way
630 you can reload the code in further invocations of %edit as a variable,
631 via _<NUMBER> or Out[<NUMBER>], where <NUMBER> is the prompt number of
632 the output.
634 Note that %edit is also available through the alias %ed.
636 This is an example of creating a simple function inside the editor and
637 then modifying it. First, start up the editor::
639 In [1]: edit
640 Editing... done. Executing edited code...
641 Out[1]: 'def foo():\\n print("foo() was defined in an editing
642 session")\\n'
644 We can then call the function foo()::
646 In [2]: foo()
647 foo() was defined in an editing session
649 Now we edit foo. IPython automatically loads the editor with the
650 (temporary) file where foo() was previously defined::
652 In [3]: edit foo
653 Editing... done. Executing edited code...
655 And if we call foo() again we get the modified version::
657 In [4]: foo()
658 foo() has now been changed!
660 Here is an example of how to edit a code snippet successive
661 times. First we call the editor::
663 In [5]: edit
664 Editing... done. Executing edited code...
665 hello
666 Out[5]: "print('hello')\\n"
668 Now we call it again with the previous output (stored in _)::
670 In [6]: edit _
671 Editing... done. Executing edited code...
672 hello world
673 Out[6]: "print('hello world')\\n"
675 Now we call it with the output #8 (stored in _8, also as Out[8])::
677 In [7]: edit _8
678 Editing... done. Executing edited code...
679 hello again
680 Out[7]: "print('hello again')\\n"
683 Changing the default editor hook:
685 If you wish to write your own editor hook, you can put it in a
686 configuration file which you load at startup time. The default hook
687 is defined in the IPython.core.hooks module, and you can use that as a
688 starting example for further modifications. That file also has
689 general instructions on how to set a new hook for use once you've
690 defined it."""
691 opts,args = self.parse_options(parameter_s,'prxn:')
693 try:
694 filename, lineno, is_temp = self._find_edit_target(self.shell,
695 args, opts, last_call)
696 except MacroToEdit as e:
697 self._edit_macro(args, e.args[0])
698 return
699 except InteractivelyDefined as e:
700 print("Editing In[%i]" % e.index)
701 args = str(e.index)
702 filename, lineno, is_temp = self._find_edit_target(self.shell,
703 args, opts, last_call)
704 if filename is None:
705 # nothing was found, warnings have already been issued,
706 # just give up.
707 return
709 if is_temp:
710 self._knowntemps.add(filename)
711 elif (filename in self._knowntemps):
712 is_temp = True
715 # do actual editing here
716 print('Editing...', end=' ')
717 sys.stdout.flush()
718 filepath = Path(filename)
719 try:
720 # Quote filenames that may have spaces in them when opening
721 # the editor
722 quoted = filename = str(filepath.absolute())
723 if " " in quoted:
724 quoted = "'%s'" % quoted
725 self.shell.hooks.editor(quoted, lineno)
726 except TryNext:
727 warn('Could not open editor')
728 return
730 # XXX TODO: should this be generalized for all string vars?
731 # For now, this is special-cased to blocks created by cpaste
732 if args.strip() == "pasted_block":
733 self.shell.user_ns["pasted_block"] = filepath.read_text(encoding="utf-8")
735 if 'x' in opts: # -x prevents actual execution
736 print()
737 else:
738 print('done. Executing edited code...')
739 with preserve_keys(self.shell.user_ns, '__file__'):
740 if not is_temp:
741 self.shell.user_ns["__file__"] = filename
742 if "r" in opts: # Untranslated IPython code
743 source = filepath.read_text(encoding="utf-8")
744 self.shell.run_cell(source, store_history=False)
745 else:
746 self.shell.safe_execfile(filename, self.shell.user_ns,
747 self.shell.user_ns)
749 if is_temp:
750 try:
751 return filepath.read_text(encoding="utf-8")
752 except IOError as msg:
753 if Path(msg.filename) == filepath:
754 warn('File not found. Did you forget to save?')
755 return
756 else:
757 self.shell.showtraceback()