Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.9/dist-packages/IPython/core/inputtransformer.py: 33%

266 statements  

« prev     ^ index     » next       coverage.py v7.3.1, created at 2023-09-25 06:05 +0000

1"""DEPRECATED: Input transformer classes to support IPython special syntax. 

2 

3This module was deprecated in IPython 7.0, in favour of inputtransformer2. 

4 

5This includes the machinery to recognise and transform ``%magic`` commands, 

6``!system`` commands, ``help?`` querying, prompt stripping, and so forth. 

7""" 

8import abc 

9import functools 

10import re 

11import tokenize 

12from tokenize import untokenize, TokenError 

13from io import StringIO 

14 

15from IPython.core.splitinput import LineInfo 

16from IPython.utils import tokenutil 

17 

18#----------------------------------------------------------------------------- 

19# Globals 

20#----------------------------------------------------------------------------- 

21 

22# The escape sequences that define the syntax transformations IPython will 

23# apply to user input. These can NOT be just changed here: many regular 

24# expressions and other parts of the code may use their hardcoded values, and 

25# for all intents and purposes they constitute the 'IPython syntax', so they 

26# should be considered fixed. 

27 

28ESC_SHELL = '!' # Send line to underlying system shell 

29ESC_SH_CAP = '!!' # Send line to system shell and capture output 

30ESC_HELP = '?' # Find information about object 

31ESC_HELP2 = '??' # Find extra-detailed information about object 

32ESC_MAGIC = '%' # Call magic function 

33ESC_MAGIC2 = '%%' # Call cell-magic function 

34ESC_QUOTE = ',' # Split args on whitespace, quote each as string and call 

35ESC_QUOTE2 = ';' # Quote all args as a single string, call 

36ESC_PAREN = '/' # Call first argument with rest of line as arguments 

37 

38ESC_SEQUENCES = [ESC_SHELL, ESC_SH_CAP, ESC_HELP ,\ 

39 ESC_HELP2, ESC_MAGIC, ESC_MAGIC2,\ 

40 ESC_QUOTE, ESC_QUOTE2, ESC_PAREN ] 

41 

42 

43class InputTransformer(metaclass=abc.ABCMeta): 

44 """Abstract base class for line-based input transformers.""" 

45 

46 @abc.abstractmethod 

47 def push(self, line): 

48 """Send a line of input to the transformer, returning the transformed 

49 input or None if the transformer is waiting for more input. 

50 

51 Must be overridden by subclasses. 

52 

53 Implementations may raise ``SyntaxError`` if the input is invalid. No 

54 other exceptions may be raised. 

55 """ 

56 pass 

57 

58 @abc.abstractmethod 

59 def reset(self): 

60 """Return, transformed any lines that the transformer has accumulated, 

61 and reset its internal state. 

62 

63 Must be overridden by subclasses. 

64 """ 

65 pass 

66 

67 @classmethod 

68 def wrap(cls, func): 

69 """Can be used by subclasses as a decorator, to return a factory that 

70 will allow instantiation with the decorated object. 

71 """ 

72 @functools.wraps(func) 

73 def transformer_factory(**kwargs): 

74 return cls(func, **kwargs) 

75 

76 return transformer_factory 

77 

78class StatelessInputTransformer(InputTransformer): 

79 """Wrapper for a stateless input transformer implemented as a function.""" 

80 def __init__(self, func): 

81 self.func = func 

82 

83 def __repr__(self): 

84 return "StatelessInputTransformer(func={0!r})".format(self.func) 

85 

86 def push(self, line): 

87 """Send a line of input to the transformer, returning the 

88 transformed input.""" 

89 return self.func(line) 

90 

91 def reset(self): 

92 """No-op - exists for compatibility.""" 

93 pass 

94 

95class CoroutineInputTransformer(InputTransformer): 

96 """Wrapper for an input transformer implemented as a coroutine.""" 

97 def __init__(self, coro, **kwargs): 

98 # Prime it 

99 self.coro = coro(**kwargs) 

100 next(self.coro) 

101 

102 def __repr__(self): 

103 return "CoroutineInputTransformer(coro={0!r})".format(self.coro) 

104 

105 def push(self, line): 

106 """Send a line of input to the transformer, returning the 

107 transformed input or None if the transformer is waiting for more 

108 input. 

109 """ 

110 return self.coro.send(line) 

111 

112 def reset(self): 

113 """Return, transformed any lines that the transformer has 

114 accumulated, and reset its internal state. 

115 """ 

116 return self.coro.send(None) 

117 

118class TokenInputTransformer(InputTransformer): 

119 """Wrapper for a token-based input transformer. 

120  

121 func should accept a list of tokens (5-tuples, see tokenize docs), and 

122 return an iterable which can be passed to tokenize.untokenize(). 

123 """ 

124 def __init__(self, func): 

125 self.func = func 

126 self.buf = [] 

127 self.reset_tokenizer() 

128 

129 def reset_tokenizer(self): 

130 it = iter(self.buf) 

131 self.tokenizer = tokenutil.generate_tokens_catch_errors(it.__next__) 

132 

133 def push(self, line): 

134 self.buf.append(line + '\n') 

135 if all(l.isspace() for l in self.buf): 

136 return self.reset() 

137 

138 tokens = [] 

139 stop_at_NL = False 

140 try: 

141 for intok in self.tokenizer: 

142 tokens.append(intok) 

143 t = intok[0] 

144 if t == tokenize.NEWLINE or (stop_at_NL and t == tokenize.NL): 

145 # Stop before we try to pull a line we don't have yet 

146 break 

147 elif t == tokenize.ERRORTOKEN: 

148 stop_at_NL = True 

149 except TokenError: 

150 # Multi-line statement - stop and try again with the next line 

151 self.reset_tokenizer() 

152 return None 

153 

154 return self.output(tokens) 

155 

156 def output(self, tokens): 

157 self.buf.clear() 

158 self.reset_tokenizer() 

159 return untokenize(self.func(tokens)).rstrip('\n') 

160 

161 def reset(self): 

162 l = ''.join(self.buf) 

163 self.buf.clear() 

164 self.reset_tokenizer() 

165 if l: 

166 return l.rstrip('\n') 

167 

168class assemble_python_lines(TokenInputTransformer): 

169 def __init__(self): 

170 super(assemble_python_lines, self).__init__(None) 

171 

172 def output(self, tokens): 

173 return self.reset() 

174 

175@CoroutineInputTransformer.wrap 

176def assemble_logical_lines(): 

177 r"""Join lines following explicit line continuations (\)""" 

178 line = '' 

179 while True: 

180 line = (yield line) 

181 if not line or line.isspace(): 

182 continue 

183 

184 parts = [] 

185 while line is not None: 

186 if line.endswith('\\') and (not has_comment(line)): 

187 parts.append(line[:-1]) 

188 line = (yield None) # Get another line 

189 else: 

190 parts.append(line) 

191 break 

192 

193 # Output 

194 line = ''.join(parts) 

195 

196# Utilities 

197def _make_help_call(target, esc, lspace): 

198 """Prepares a pinfo(2)/psearch call from a target name and the escape 

199 (i.e. ? or ??)""" 

200 method = 'pinfo2' if esc == '??' \ 

201 else 'psearch' if '*' in target \ 

202 else 'pinfo' 

203 arg = " ".join([method, target]) 

204 #Prepare arguments for get_ipython().run_line_magic(magic_name, magic_args) 

205 t_magic_name, _, t_magic_arg_s = arg.partition(' ') 

206 t_magic_name = t_magic_name.lstrip(ESC_MAGIC) 

207 return "%sget_ipython().run_line_magic(%r, %r)" % ( 

208 lspace, 

209 t_magic_name, 

210 t_magic_arg_s, 

211 ) 

212 

213 

214# These define the transformations for the different escape characters. 

215def _tr_system(line_info): 

216 "Translate lines escaped with: !" 

217 cmd = line_info.line.lstrip().lstrip(ESC_SHELL) 

218 return '%sget_ipython().system(%r)' % (line_info.pre, cmd) 

219 

220def _tr_system2(line_info): 

221 "Translate lines escaped with: !!" 

222 cmd = line_info.line.lstrip()[2:] 

223 return '%sget_ipython().getoutput(%r)' % (line_info.pre, cmd) 

224 

225def _tr_help(line_info): 

226 "Translate lines escaped with: ?/??" 

227 # A naked help line should just fire the intro help screen 

228 if not line_info.line[1:]: 

229 return 'get_ipython().show_usage()' 

230 

231 return _make_help_call(line_info.ifun, line_info.esc, line_info.pre) 

232 

233def _tr_magic(line_info): 

234 "Translate lines escaped with: %" 

235 tpl = '%sget_ipython().run_line_magic(%r, %r)' 

236 if line_info.line.startswith(ESC_MAGIC2): 

237 return line_info.line 

238 cmd = ' '.join([line_info.ifun, line_info.the_rest]).strip() 

239 #Prepare arguments for get_ipython().run_line_magic(magic_name, magic_args) 

240 t_magic_name, _, t_magic_arg_s = cmd.partition(' ') 

241 t_magic_name = t_magic_name.lstrip(ESC_MAGIC) 

242 return tpl % (line_info.pre, t_magic_name, t_magic_arg_s) 

243 

244def _tr_quote(line_info): 

245 "Translate lines escaped with: ," 

246 return '%s%s("%s")' % (line_info.pre, line_info.ifun, 

247 '", "'.join(line_info.the_rest.split()) ) 

248 

249def _tr_quote2(line_info): 

250 "Translate lines escaped with: ;" 

251 return '%s%s("%s")' % (line_info.pre, line_info.ifun, 

252 line_info.the_rest) 

253 

254def _tr_paren(line_info): 

255 "Translate lines escaped with: /" 

256 return '%s%s(%s)' % (line_info.pre, line_info.ifun, 

257 ", ".join(line_info.the_rest.split())) 

258 

259tr = { ESC_SHELL : _tr_system, 

260 ESC_SH_CAP : _tr_system2, 

261 ESC_HELP : _tr_help, 

262 ESC_HELP2 : _tr_help, 

263 ESC_MAGIC : _tr_magic, 

264 ESC_QUOTE : _tr_quote, 

265 ESC_QUOTE2 : _tr_quote2, 

266 ESC_PAREN : _tr_paren } 

267 

268@StatelessInputTransformer.wrap 

269def escaped_commands(line): 

270 """Transform escaped commands - %magic, !system, ?help + various autocalls. 

271 """ 

272 if not line or line.isspace(): 

273 return line 

274 lineinf = LineInfo(line) 

275 if lineinf.esc not in tr: 

276 return line 

277 

278 return tr[lineinf.esc](lineinf) 

279 

280_initial_space_re = re.compile(r'\s*') 

281 

282_help_end_re = re.compile(r"""(%{0,2} 

283 (?!\d)[\w*]+ # Variable name 

284 (\.(?!\d)[\w*]+)* # .etc.etc 

285 ) 

286 (\?\??)$ # ? or ?? 

287 """, 

288 re.VERBOSE) 

289 

290# Extra pseudotokens for multiline strings and data structures 

291_MULTILINE_STRING = object() 

292_MULTILINE_STRUCTURE = object() 

293 

294def _line_tokens(line): 

295 """Helper for has_comment and ends_in_comment_or_string.""" 

296 readline = StringIO(line).readline 

297 toktypes = set() 

298 try: 

299 for t in tokenutil.generate_tokens_catch_errors(readline): 

300 toktypes.add(t[0]) 

301 except TokenError as e: 

302 # There are only two cases where a TokenError is raised. 

303 if 'multi-line string' in e.args[0]: 

304 toktypes.add(_MULTILINE_STRING) 

305 else: 

306 toktypes.add(_MULTILINE_STRUCTURE) 

307 return toktypes 

308 

309def has_comment(src): 

310 """Indicate whether an input line has (i.e. ends in, or is) a comment. 

311 

312 This uses tokenize, so it can distinguish comments from # inside strings. 

313 

314 Parameters 

315 ---------- 

316 src : string 

317 A single line input string. 

318 

319 Returns 

320 ------- 

321 comment : bool 

322 True if source has a comment. 

323 """ 

324 return (tokenize.COMMENT in _line_tokens(src)) 

325 

326def ends_in_comment_or_string(src): 

327 """Indicates whether or not an input line ends in a comment or within 

328 a multiline string. 

329 

330 Parameters 

331 ---------- 

332 src : string 

333 A single line input string. 

334 

335 Returns 

336 ------- 

337 comment : bool 

338 True if source ends in a comment or multiline string. 

339 """ 

340 toktypes = _line_tokens(src) 

341 return (tokenize.COMMENT in toktypes) or (_MULTILINE_STRING in toktypes) 

342 

343 

344@StatelessInputTransformer.wrap 

345def help_end(line): 

346 """Translate lines with ?/?? at the end""" 

347 m = _help_end_re.search(line) 

348 if m is None or ends_in_comment_or_string(line): 

349 return line 

350 target = m.group(1) 

351 esc = m.group(3) 

352 lspace = _initial_space_re.match(line).group(0) 

353 

354 return _make_help_call(target, esc, lspace) 

355 

356 

357@CoroutineInputTransformer.wrap 

358def cellmagic(end_on_blank_line=False): 

359 """Captures & transforms cell magics. 

360 

361 After a cell magic is started, this stores up any lines it gets until it is 

362 reset (sent None). 

363 """ 

364 tpl = 'get_ipython().run_cell_magic(%r, %r, %r)' 

365 cellmagic_help_re = re.compile(r'%%\w+\?') 

366 line = '' 

367 while True: 

368 line = (yield line) 

369 # consume leading empty lines 

370 while not line: 

371 line = (yield line) 

372 

373 if not line.startswith(ESC_MAGIC2): 

374 # This isn't a cell magic, idle waiting for reset then start over 

375 while line is not None: 

376 line = (yield line) 

377 continue 

378 

379 if cellmagic_help_re.match(line): 

380 # This case will be handled by help_end 

381 continue 

382 

383 first = line 

384 body = [] 

385 line = (yield None) 

386 while (line is not None) and \ 

387 ((line.strip() != '') or not end_on_blank_line): 

388 body.append(line) 

389 line = (yield None) 

390 

391 # Output 

392 magic_name, _, first = first.partition(' ') 

393 magic_name = magic_name.lstrip(ESC_MAGIC2) 

394 line = tpl % (magic_name, first, u'\n'.join(body)) 

395 

396 

397def _strip_prompts(prompt_re, initial_re=None, turnoff_re=None): 

398 """Remove matching input prompts from a block of input. 

399 

400 Parameters 

401 ---------- 

402 prompt_re : regular expression 

403 A regular expression matching any input prompt (including continuation) 

404 initial_re : regular expression, optional 

405 A regular expression matching only the initial prompt, but not continuation. 

406 If no initial expression is given, prompt_re will be used everywhere. 

407 Used mainly for plain Python prompts, where the continuation prompt 

408 ``...`` is a valid Python expression in Python 3, so shouldn't be stripped. 

409 

410 Notes 

411 ----- 

412 If `initial_re` and `prompt_re differ`, 

413 only `initial_re` will be tested against the first line. 

414 If any prompt is found on the first two lines, 

415 prompts will be stripped from the rest of the block. 

416 """ 

417 if initial_re is None: 

418 initial_re = prompt_re 

419 line = '' 

420 while True: 

421 line = (yield line) 

422 

423 # First line of cell 

424 if line is None: 

425 continue 

426 out, n1 = initial_re.subn('', line, count=1) 

427 if turnoff_re and not n1: 

428 if turnoff_re.match(line): 

429 # We're in e.g. a cell magic; disable this transformer for 

430 # the rest of the cell. 

431 while line is not None: 

432 line = (yield line) 

433 continue 

434 

435 line = (yield out) 

436 

437 if line is None: 

438 continue 

439 # check for any prompt on the second line of the cell, 

440 # because people often copy from just after the first prompt, 

441 # so we might not see it in the first line. 

442 out, n2 = prompt_re.subn('', line, count=1) 

443 line = (yield out) 

444 

445 if n1 or n2: 

446 # Found a prompt in the first two lines - check for it in 

447 # the rest of the cell as well. 

448 while line is not None: 

449 line = (yield prompt_re.sub('', line, count=1)) 

450 

451 else: 

452 # Prompts not in input - wait for reset 

453 while line is not None: 

454 line = (yield line) 

455 

456@CoroutineInputTransformer.wrap 

457def classic_prompt(): 

458 """Strip the >>>/... prompts of the Python interactive shell.""" 

459 # FIXME: non-capturing version (?:...) usable? 

460 prompt_re = re.compile(r'^(>>>|\.\.\.)( |$)') 

461 initial_re = re.compile(r'^>>>( |$)') 

462 # Any %magic/!system is IPython syntax, so we needn't look for >>> prompts 

463 turnoff_re = re.compile(r'^[%!]') 

464 return _strip_prompts(prompt_re, initial_re, turnoff_re) 

465 

466@CoroutineInputTransformer.wrap 

467def ipy_prompt(): 

468 """Strip IPython's In [1]:/...: prompts.""" 

469 # FIXME: non-capturing version (?:...) usable? 

470 prompt_re = re.compile(r'^(In \[\d+\]: |\s*\.{3,}: ?)') 

471 # Disable prompt stripping inside cell magics 

472 turnoff_re = re.compile(r'^%%') 

473 return _strip_prompts(prompt_re, turnoff_re=turnoff_re) 

474 

475 

476@CoroutineInputTransformer.wrap 

477def leading_indent(): 

478 """Remove leading indentation. 

479 

480 If the first line starts with a spaces or tabs, the same whitespace will be 

481 removed from each following line until it is reset. 

482 """ 

483 space_re = re.compile(r'^[ \t]+') 

484 line = '' 

485 while True: 

486 line = (yield line) 

487 

488 if line is None: 

489 continue 

490 

491 m = space_re.match(line) 

492 if m: 

493 space = m.group(0) 

494 while line is not None: 

495 if line.startswith(space): 

496 line = line[len(space):] 

497 line = (yield line) 

498 else: 

499 # No leading spaces - wait for reset 

500 while line is not None: 

501 line = (yield line) 

502 

503 

504_assign_pat = \ 

505r'''(?P<lhs>(\s*) 

506 ([\w\.]+) # Initial identifier 

507 (\s*,\s* 

508 \*?[\w\.]+)* # Further identifiers for unpacking 

509 \s*?,? # Trailing comma 

510 ) 

511 \s*=\s* 

512''' 

513 

514assign_system_re = re.compile(r'{}!\s*(?P<cmd>.*)'.format(_assign_pat), re.VERBOSE) 

515assign_system_template = '%s = get_ipython().getoutput(%r)' 

516@StatelessInputTransformer.wrap 

517def assign_from_system(line): 

518 """Transform assignment from system commands (e.g. files = !ls)""" 

519 m = assign_system_re.match(line) 

520 if m is None: 

521 return line 

522 

523 return assign_system_template % m.group('lhs', 'cmd') 

524 

525assign_magic_re = re.compile(r'{}%\s*(?P<cmd>.*)'.format(_assign_pat), re.VERBOSE) 

526assign_magic_template = '%s = get_ipython().run_line_magic(%r, %r)' 

527@StatelessInputTransformer.wrap 

528def assign_from_magic(line): 

529 """Transform assignment from magic commands (e.g. a = %who_ls)""" 

530 m = assign_magic_re.match(line) 

531 if m is None: 

532 return line 

533 #Prepare arguments for get_ipython().run_line_magic(magic_name, magic_args) 

534 m_lhs, m_cmd = m.group('lhs', 'cmd') 

535 t_magic_name, _, t_magic_arg_s = m_cmd.partition(' ') 

536 t_magic_name = t_magic_name.lstrip(ESC_MAGIC) 

537 return assign_magic_template % (m_lhs, t_magic_name, t_magic_arg_s)