Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/IPython/utils/PyColorize.py: 25%

1# -*- coding: utf-8 -*- 

2""" 

3Class and program to colorize python source code for ANSI terminals. 

4 

5Based on an HTML code highlighter by Jurgen Hermann found at: 

6http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/52298 

7 

8Modifications by Fernando Perez (fperez@colorado.edu). 

9 

10Information on the original HTML highlighter follows: 

11 

12MoinMoin - Python Source Parser 

13 

14Title: Colorize Python source using the built-in tokenizer 

15 

16Submitter: Jurgen Hermann 

17Last Updated:2001/04/06 

18 

19Version no:1.2 

20 

21Description: 

22 

23This code is part of MoinMoin (http://moin.sourceforge.net/) and converts 

24Python source code to HTML markup, rendering comments, keywords, 

25operators, numeric and string literals in different colors. 

26 

27It shows how to use the built-in keyword, token and tokenize modules to 

28scan Python source code and re-emit it with no changes to its original 

29formatting (which is the hard part). 

30""" 

31 

32__all__ = ['ANSICodeColors', 'Parser'] 

33 

34_scheme_default = 'Linux' 

35 

36 

37# Imports 

38import keyword 

39import os 

40import sys 

41import token 

42import tokenize 

43 

44generate_tokens = tokenize.generate_tokens 

45 

46from IPython.utils.coloransi import TermColors, InputTermColors,ColorScheme, ColorSchemeTable 

47from .colorable import Colorable 

48from io import StringIO 

49 

50############################################################################# 

51### Python Source Parser (does Highlighting) 

52############################################################################# 

53 

54_KEYWORD = token.NT_OFFSET + 1 

55_TEXT = token.NT_OFFSET + 2 

56 

57#**************************************************************************** 

58# Builtin color schemes 

59 

60Colors = TermColors # just a shorthand 

61 

62# Build a few color schemes 

63NoColor = ColorScheme( 

64 'NoColor',{ 

65 'header' : Colors.NoColor, 

66 token.NUMBER : Colors.NoColor, 

67 token.OP : Colors.NoColor, 

68 token.STRING : Colors.NoColor, 

69 tokenize.COMMENT : Colors.NoColor, 

70 token.NAME : Colors.NoColor, 

71 token.ERRORTOKEN : Colors.NoColor, 

72 

73 _KEYWORD : Colors.NoColor, 

74 _TEXT : Colors.NoColor, 

75 

76 'in_prompt' : InputTermColors.NoColor, # Input prompt 

77 'in_number' : InputTermColors.NoColor, # Input prompt number 

78 'in_prompt2' : InputTermColors.NoColor, # Continuation prompt 

79 'in_normal' : InputTermColors.NoColor, # color off (usu. Colors.Normal) 

80 

81 'out_prompt' : Colors.NoColor, # Output prompt 

82 'out_number' : Colors.NoColor, # Output prompt number 

83 

84 'normal' : Colors.NoColor # color off (usu. Colors.Normal) 

85 } ) 

86 

87LinuxColors = ColorScheme( 

88 'Linux',{ 

89 'header' : Colors.LightRed, 

90 token.NUMBER : Colors.LightCyan, 

91 token.OP : Colors.Yellow, 

92 token.STRING : Colors.LightBlue, 

93 tokenize.COMMENT : Colors.LightRed, 

94 token.NAME : Colors.Normal, 

95 token.ERRORTOKEN : Colors.Red, 

96 

97 _KEYWORD : Colors.LightGreen, 

98 _TEXT : Colors.Yellow, 

99 

100 'in_prompt' : InputTermColors.Green, 

101 'in_number' : InputTermColors.LightGreen, 

102 'in_prompt2' : InputTermColors.Green, 

103 'in_normal' : InputTermColors.Normal, # color off (usu. Colors.Normal) 

104 

105 'out_prompt' : Colors.Red, 

106 'out_number' : Colors.LightRed, 

107 

108 'normal' : Colors.Normal # color off (usu. Colors.Normal) 

109 } ) 

110 

111NeutralColors = ColorScheme( 

112 'Neutral',{ 

113 'header' : Colors.Red, 

114 token.NUMBER : Colors.Cyan, 

115 token.OP : Colors.Blue, 

116 token.STRING : Colors.Blue, 

117 tokenize.COMMENT : Colors.Red, 

118 token.NAME : Colors.Normal, 

119 token.ERRORTOKEN : Colors.Red, 

120 

121 _KEYWORD : Colors.Green, 

122 _TEXT : Colors.Blue, 

123 

124 'in_prompt' : InputTermColors.Blue, 

125 'in_number' : InputTermColors.LightBlue, 

126 'in_prompt2' : InputTermColors.Blue, 

127 'in_normal' : InputTermColors.Normal, # color off (usu. Colors.Normal) 

128 

129 'out_prompt' : Colors.Red, 

130 'out_number' : Colors.LightRed, 

131 

132 'normal' : Colors.Normal # color off (usu. Colors.Normal) 

133 } ) 

134 

135# Hack: the 'neutral' colours are not very visible on a dark background on 

136# Windows. Since Windows command prompts have a dark background by default, and 

137# relatively few users are likely to alter that, we will use the 'Linux' colours, 

138# designed for a dark background, as the default on Windows. Changing it here 

139# avoids affecting the prompt colours rendered by prompt_toolkit, where the 

140# neutral defaults do work OK. 

141 

142if os.name == 'nt': 

143 NeutralColors = LinuxColors.copy(name='Neutral') 

144 

145LightBGColors = ColorScheme( 

146 'LightBG',{ 

147 'header' : Colors.Red, 

148 token.NUMBER : Colors.Cyan, 

149 token.OP : Colors.Blue, 

150 token.STRING : Colors.Blue, 

151 tokenize.COMMENT : Colors.Red, 

152 token.NAME : Colors.Normal, 

153 token.ERRORTOKEN : Colors.Red, 

154 

155 

156 _KEYWORD : Colors.Green, 

157 _TEXT : Colors.Blue, 

158 

159 'in_prompt' : InputTermColors.Blue, 

160 'in_number' : InputTermColors.LightBlue, 

161 'in_prompt2' : InputTermColors.Blue, 

162 'in_normal' : InputTermColors.Normal, # color off (usu. Colors.Normal) 

163 

164 'out_prompt' : Colors.Red, 

165 'out_number' : Colors.LightRed, 

166 

167 'normal' : Colors.Normal # color off (usu. Colors.Normal) 

168 } ) 

169 

170# Build table of color schemes (needed by the parser) 

171ANSICodeColors = ColorSchemeTable([NoColor,LinuxColors,LightBGColors, NeutralColors], 

172 _scheme_default) 

173 

174Undefined = object() 

175 

176class Parser(Colorable): 

177 """ Format colored Python source. 

178 """ 

179 

180 def __init__(self, color_table=None, out = sys.stdout, parent=None, style=None): 

181 """ Create a parser with a specified color table and output channel. 

182 

183 Call format() to process code. 

184 """ 

185 

186 super(Parser, self).__init__(parent=parent) 

187 

188 self.color_table = color_table if color_table else ANSICodeColors 

189 self.out = out 

190 self.pos = None 

191 self.lines = None 

192 self.raw = None 

193 if not style: 

194 self.style = self.default_style 

195 else: 

196 self.style = style 

197 

198 

199 def format(self, raw, out=None, scheme=Undefined): 

200 import warnings 

201 if scheme is not Undefined: 

202 warnings.warn('The `scheme` argument of IPython.utils.PyColorize:Parser.format is deprecated since IPython 6.0.' 

203 'It will have no effect. Set the parser `style` directly.', 

204 stacklevel=2) 

205 return self.format2(raw, out)[0] 

206 

207 def format2(self, raw, out = None): 

208 """ Parse and send the colored source. 

209 

210 If out and scheme are not specified, the defaults (given to 

211 constructor) are used. 

212 

213 out should be a file-type object. Optionally, out can be given as the 

214 string 'str' and the parser will automatically return the output in a 

215 string.""" 

216 

217 string_output = 0 

218 if out == 'str' or self.out == 'str' or \ 

219 isinstance(self.out, StringIO): 

220 # XXX - I don't really like this state handling logic, but at this 

221 # point I don't want to make major changes, so adding the 

222 # isinstance() check is the simplest I can do to ensure correct 

223 # behavior. 

224 out_old = self.out 

225 self.out = StringIO() 

226 string_output = 1 

227 elif out is not None: 

228 self.out = out 

229 else: 

230 raise ValueError('`out` or `self.out` should be file-like or the value `"str"`') 

231 

232 # Fast return of the unmodified input for NoColor scheme 

233 if self.style == 'NoColor': 

234 error = False 

235 self.out.write(raw) 

236 if string_output: 

237 return raw, error 

238 return None, error 

239 

240 # local shorthands 

241 colors = self.color_table[self.style].colors 

242 self.colors = colors # put in object so __call__ sees it 

243 

244 # Remove trailing whitespace and normalize tabs 

245 self.raw = raw.expandtabs().rstrip() 

246 

247 # store line offsets in self.lines 

248 self.lines = [0, 0] 

249 pos = 0 

250 raw_find = self.raw.find 

251 lines_append = self.lines.append 

252 while True: 

253 pos = raw_find('\n', pos) + 1 

254 if not pos: 

255 break 

256 lines_append(pos) 

257 lines_append(len(self.raw)) 

258 

259 # parse the source and write it 

260 self.pos = 0 

261 text = StringIO(self.raw) 

262 

263 error = False 

264 try: 

265 for atoken in generate_tokens(text.readline): 

266 self(*atoken) 

267 except tokenize.TokenError as ex: 

268 msg = ex.args[0] 

269 line = ex.args[1][0] 

270 self.out.write("%s\n\n*** ERROR: %s%s%s\n" % 

271 (colors[token.ERRORTOKEN], 

272 msg, self.raw[self.lines[line]:], 

273 colors.normal) 

274 ) 

275 error = True 

276 self.out.write(colors.normal+'\n') 

277 if string_output: 

278 output = self.out.getvalue() 

279 self.out = out_old 

280 return (output, error) 

281 return (None, error) 

282 

283 

284 def _inner_call_(self, toktype, toktext, start_pos): 

285 """like call but write to a temporary buffer""" 

286 buff = StringIO() 

287 srow, scol = start_pos 

288 colors = self.colors 

289 owrite = buff.write 

290 

291 # line separator, so this works across platforms 

292 linesep = os.linesep 

293 

294 # calculate new positions 

295 oldpos = self.pos 

296 newpos = self.lines[srow] + scol 

297 self.pos = newpos + len(toktext) 

298 

299 # send the original whitespace, if needed 

300 if newpos > oldpos: 

301 owrite(self.raw[oldpos:newpos]) 

302 

303 # skip indenting tokens 

304 if toktype in [token.INDENT, token.DEDENT]: 

305 self.pos = newpos 

306 buff.seek(0) 

307 return buff.read() 

308 

309 # map token type to a color group 

310 if token.LPAR <= toktype <= token.OP: 

311 toktype = token.OP 

312 elif toktype == token.NAME and keyword.iskeyword(toktext): 

313 toktype = _KEYWORD 

314 color = colors.get(toktype, colors[_TEXT]) 

315 

316 # Triple quoted strings must be handled carefully so that backtracking 

317 # in pagers works correctly. We need color terminators on _each_ line. 

318 if linesep in toktext: 

319 toktext = toktext.replace(linesep, '%s%s%s' % 

320 (colors.normal,linesep,color)) 

321 

322 # send text 

323 owrite('%s%s%s' % (color,toktext,colors.normal)) 

324 buff.seek(0) 

325 return buff.read() 

326 

327 

328 def __call__(self, toktype, toktext, start_pos, end_pos, line): 

329 """ Token handler, with syntax highlighting.""" 

330 self.out.write( 

331 self._inner_call_(toktype, toktext, start_pos))