Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/pygments/lexers/python.py: 58%
124 statements
« prev ^ index » next coverage.py v7.2.2, created at 2023-03-26 06:07 +0000
1"""
2 pygments.lexers.python
3 ~~~~~~~~~~~~~~~~~~~~~~
5 Lexers for Python and related languages.
7 :copyright: Copyright 2006-2022 by the Pygments team, see AUTHORS.
8 :license: BSD, see LICENSE for details.
9"""
11import re
12import keyword
14from pygments.lexer import Lexer, RegexLexer, include, bygroups, using, \
15 default, words, combined, do_insertions, this, line_re
16from pygments.util import get_bool_opt, shebang_matches
17from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
18 Number, Punctuation, Generic, Other, Error, Whitespace
19from pygments import unistring as uni
21__all__ = ['PythonLexer', 'PythonConsoleLexer', 'PythonTracebackLexer',
22 'Python2Lexer', 'Python2TracebackLexer',
23 'CythonLexer', 'DgLexer', 'NumPyLexer']
26class PythonLexer(RegexLexer):
27 """
28 For Python source code (version 3.x).
30 .. versionadded:: 0.10
32 .. versionchanged:: 2.5
33 This is now the default ``PythonLexer``. It is still available as the
34 alias ``Python3Lexer``.
35 """
37 name = 'Python'
38 url = 'http://www.python.org'
39 aliases = ['python', 'py', 'sage', 'python3', 'py3']
40 filenames = [
41 '*.py',
42 '*.pyw',
43 # Type stubs
44 '*.pyi',
45 # Jython
46 '*.jy',
47 # Sage
48 '*.sage',
49 # SCons
50 '*.sc',
51 'SConstruct',
52 'SConscript',
53 # Skylark/Starlark (used by Bazel, Buck, and Pants)
54 '*.bzl',
55 'BUCK',
56 'BUILD',
57 'BUILD.bazel',
58 'WORKSPACE',
59 # Twisted Application infrastructure
60 '*.tac',
61 ]
62 mimetypes = ['text/x-python', 'application/x-python',
63 'text/x-python3', 'application/x-python3']
65 uni_name = "[%s][%s]*" % (uni.xid_start, uni.xid_continue)
67 def innerstring_rules(ttype):
68 return [
69 # the old style '%s' % (...) string formatting (still valid in Py3)
70 (r'%(\(\w+\))?[-#0 +]*([0-9]+|[*])?(\.([0-9]+|[*]))?'
71 '[hlL]?[E-GXc-giorsaux%]', String.Interpol),
72 # the new style '{}'.format(...) string formatting
73 (r'\{'
74 r'((\w+)((\.\w+)|(\[[^\]]+\]))*)?' # field name
75 r'(\![sra])?' # conversion
76 r'(\:(.?[<>=\^])?[-+ ]?#?0?(\d+)?,?(\.\d+)?[E-GXb-gnosx%]?)?'
77 r'\}', String.Interpol),
79 # backslashes, quotes and formatting signs must be parsed one at a time
80 (r'[^\\\'"%{\n]+', ttype),
81 (r'[\'"\\]', ttype),
82 # unhandled string formatting sign
83 (r'%|(\{{1,2})', ttype)
84 # newlines are an error (use "nl" state)
85 ]
87 def fstring_rules(ttype):
88 return [
89 # Assuming that a '}' is the closing brace after format specifier.
90 # Sadly, this means that we won't detect syntax error. But it's
91 # more important to parse correct syntax correctly, than to
92 # highlight invalid syntax.
93 (r'\}', String.Interpol),
94 (r'\{', String.Interpol, 'expr-inside-fstring'),
95 # backslashes, quotes and formatting signs must be parsed one at a time
96 (r'[^\\\'"{}\n]+', ttype),
97 (r'[\'"\\]', ttype),
98 # newlines are an error (use "nl" state)
99 ]
101 tokens = {
102 'root': [
103 (r'\n', Whitespace),
104 (r'^(\s*)([rRuUbB]{,2})("""(?:.|\n)*?""")',
105 bygroups(Whitespace, String.Affix, String.Doc)),
106 (r"^(\s*)([rRuUbB]{,2})('''(?:.|\n)*?''')",
107 bygroups(Whitespace, String.Affix, String.Doc)),
108 (r'\A#!.+$', Comment.Hashbang),
109 (r'#.*$', Comment.Single),
110 (r'\\\n', Text),
111 (r'\\', Text),
112 include('keywords'),
113 include('soft-keywords'),
114 (r'(def)((?:\s|\\\s)+)', bygroups(Keyword, Text), 'funcname'),
115 (r'(class)((?:\s|\\\s)+)', bygroups(Keyword, Text), 'classname'),
116 (r'(from)((?:\s|\\\s)+)', bygroups(Keyword.Namespace, Text),
117 'fromimport'),
118 (r'(import)((?:\s|\\\s)+)', bygroups(Keyword.Namespace, Text),
119 'import'),
120 include('expr'),
121 ],
122 'expr': [
123 # raw f-strings
124 ('(?i)(rf|fr)(""")',
125 bygroups(String.Affix, String.Double),
126 combined('rfstringescape', 'tdqf')),
127 ("(?i)(rf|fr)(''')",
128 bygroups(String.Affix, String.Single),
129 combined('rfstringescape', 'tsqf')),
130 ('(?i)(rf|fr)(")',
131 bygroups(String.Affix, String.Double),
132 combined('rfstringescape', 'dqf')),
133 ("(?i)(rf|fr)(')",
134 bygroups(String.Affix, String.Single),
135 combined('rfstringescape', 'sqf')),
136 # non-raw f-strings
137 ('([fF])(""")', bygroups(String.Affix, String.Double),
138 combined('fstringescape', 'tdqf')),
139 ("([fF])(''')", bygroups(String.Affix, String.Single),
140 combined('fstringescape', 'tsqf')),
141 ('([fF])(")', bygroups(String.Affix, String.Double),
142 combined('fstringescape', 'dqf')),
143 ("([fF])(')", bygroups(String.Affix, String.Single),
144 combined('fstringescape', 'sqf')),
145 # raw bytes and strings
146 ('(?i)(rb|br|r)(""")',
147 bygroups(String.Affix, String.Double), 'tdqs'),
148 ("(?i)(rb|br|r)(''')",
149 bygroups(String.Affix, String.Single), 'tsqs'),
150 ('(?i)(rb|br|r)(")',
151 bygroups(String.Affix, String.Double), 'dqs'),
152 ("(?i)(rb|br|r)(')",
153 bygroups(String.Affix, String.Single), 'sqs'),
154 # non-raw strings
155 ('([uU]?)(""")', bygroups(String.Affix, String.Double),
156 combined('stringescape', 'tdqs')),
157 ("([uU]?)(''')", bygroups(String.Affix, String.Single),
158 combined('stringescape', 'tsqs')),
159 ('([uU]?)(")', bygroups(String.Affix, String.Double),
160 combined('stringescape', 'dqs')),
161 ("([uU]?)(')", bygroups(String.Affix, String.Single),
162 combined('stringescape', 'sqs')),
163 # non-raw bytes
164 ('([bB])(""")', bygroups(String.Affix, String.Double),
165 combined('bytesescape', 'tdqs')),
166 ("([bB])(''')", bygroups(String.Affix, String.Single),
167 combined('bytesescape', 'tsqs')),
168 ('([bB])(")', bygroups(String.Affix, String.Double),
169 combined('bytesescape', 'dqs')),
170 ("([bB])(')", bygroups(String.Affix, String.Single),
171 combined('bytesescape', 'sqs')),
173 (r'[^\S\n]+', Text),
174 include('numbers'),
175 (r'!=|==|<<|>>|:=|[-~+/*%=<>&^|.]', Operator),
176 (r'[]{}:(),;[]', Punctuation),
177 (r'(in|is|and|or|not)\b', Operator.Word),
178 include('expr-keywords'),
179 include('builtins'),
180 include('magicfuncs'),
181 include('magicvars'),
182 include('name'),
183 ],
184 'expr-inside-fstring': [
185 (r'[{([]', Punctuation, 'expr-inside-fstring-inner'),
186 # without format specifier
187 (r'(=\s*)?' # debug (https://bugs.python.org/issue36817)
188 r'(\![sraf])?' # conversion
189 r'\}', String.Interpol, '#pop'),
190 # with format specifier
191 # we'll catch the remaining '}' in the outer scope
192 (r'(=\s*)?' # debug (https://bugs.python.org/issue36817)
193 r'(\![sraf])?' # conversion
194 r':', String.Interpol, '#pop'),
195 (r'\s+', Whitespace), # allow new lines
196 include('expr'),
197 ],
198 'expr-inside-fstring-inner': [
199 (r'[{([]', Punctuation, 'expr-inside-fstring-inner'),
200 (r'[])}]', Punctuation, '#pop'),
201 (r'\s+', Whitespace), # allow new lines
202 include('expr'),
203 ],
204 'expr-keywords': [
205 # Based on https://docs.python.org/3/reference/expressions.html
206 (words((
207 'async for', 'await', 'else', 'for', 'if', 'lambda',
208 'yield', 'yield from'), suffix=r'\b'),
209 Keyword),
210 (words(('True', 'False', 'None'), suffix=r'\b'), Keyword.Constant),
211 ],
212 'keywords': [
213 (words((
214 'assert', 'async', 'await', 'break', 'continue', 'del', 'elif',
215 'else', 'except', 'finally', 'for', 'global', 'if', 'lambda',
216 'pass', 'raise', 'nonlocal', 'return', 'try', 'while', 'yield',
217 'yield from', 'as', 'with'), suffix=r'\b'),
218 Keyword),
219 (words(('True', 'False', 'None'), suffix=r'\b'), Keyword.Constant),
220 ],
221 'soft-keywords': [
222 # `match`, `case` and `_` soft keywords
223 (r'(^[ \t]*)' # at beginning of line + possible indentation
224 r'(match|case)\b' # a possible keyword
225 r'(?![ \t]*(?:' # not followed by...
226 r'[:,;=^&|@~)\]}]|(?:' + # characters and keywords that mean this isn't
227 r'|'.join(keyword.kwlist) + r')\b))', # pattern matching
228 bygroups(Text, Keyword), 'soft-keywords-inner'),
229 ],
230 'soft-keywords-inner': [
231 # optional `_` keyword
232 (r'(\s+)([^\n_]*)(_\b)', bygroups(Whitespace, using(this), Keyword)),
233 default('#pop')
234 ],
235 'builtins': [
236 (words((
237 '__import__', 'abs', 'all', 'any', 'bin', 'bool', 'bytearray',
238 'breakpoint', 'bytes', 'chr', 'classmethod', 'compile', 'complex',
239 'delattr', 'dict', 'dir', 'divmod', 'enumerate', 'eval', 'filter',
240 'float', 'format', 'frozenset', 'getattr', 'globals', 'hasattr',
241 'hash', 'hex', 'id', 'input', 'int', 'isinstance', 'issubclass',
242 'iter', 'len', 'list', 'locals', 'map', 'max', 'memoryview',
243 'min', 'next', 'object', 'oct', 'open', 'ord', 'pow', 'print',
244 'property', 'range', 'repr', 'reversed', 'round', 'set', 'setattr',
245 'slice', 'sorted', 'staticmethod', 'str', 'sum', 'super', 'tuple',
246 'type', 'vars', 'zip'), prefix=r'(?<!\.)', suffix=r'\b'),
247 Name.Builtin),
248 (r'(?<!\.)(self|Ellipsis|NotImplemented|cls)\b', Name.Builtin.Pseudo),
249 (words((
250 'ArithmeticError', 'AssertionError', 'AttributeError',
251 'BaseException', 'BufferError', 'BytesWarning', 'DeprecationWarning',
252 'EOFError', 'EnvironmentError', 'Exception', 'FloatingPointError',
253 'FutureWarning', 'GeneratorExit', 'IOError', 'ImportError',
254 'ImportWarning', 'IndentationError', 'IndexError', 'KeyError',
255 'KeyboardInterrupt', 'LookupError', 'MemoryError', 'NameError',
256 'NotImplementedError', 'OSError', 'OverflowError',
257 'PendingDeprecationWarning', 'ReferenceError', 'ResourceWarning',
258 'RuntimeError', 'RuntimeWarning', 'StopIteration',
259 'SyntaxError', 'SyntaxWarning', 'SystemError', 'SystemExit',
260 'TabError', 'TypeError', 'UnboundLocalError', 'UnicodeDecodeError',
261 'UnicodeEncodeError', 'UnicodeError', 'UnicodeTranslateError',
262 'UnicodeWarning', 'UserWarning', 'ValueError', 'VMSError',
263 'Warning', 'WindowsError', 'ZeroDivisionError',
264 # new builtin exceptions from PEP 3151
265 'BlockingIOError', 'ChildProcessError', 'ConnectionError',
266 'BrokenPipeError', 'ConnectionAbortedError', 'ConnectionRefusedError',
267 'ConnectionResetError', 'FileExistsError', 'FileNotFoundError',
268 'InterruptedError', 'IsADirectoryError', 'NotADirectoryError',
269 'PermissionError', 'ProcessLookupError', 'TimeoutError',
270 # others new in Python 3
271 'StopAsyncIteration', 'ModuleNotFoundError', 'RecursionError',
272 'EncodingWarning'),
273 prefix=r'(?<!\.)', suffix=r'\b'),
274 Name.Exception),
275 ],
276 'magicfuncs': [
277 (words((
278 '__abs__', '__add__', '__aenter__', '__aexit__', '__aiter__',
279 '__and__', '__anext__', '__await__', '__bool__', '__bytes__',
280 '__call__', '__complex__', '__contains__', '__del__', '__delattr__',
281 '__delete__', '__delitem__', '__dir__', '__divmod__', '__enter__',
282 '__eq__', '__exit__', '__float__', '__floordiv__', '__format__',
283 '__ge__', '__get__', '__getattr__', '__getattribute__',
284 '__getitem__', '__gt__', '__hash__', '__iadd__', '__iand__',
285 '__ifloordiv__', '__ilshift__', '__imatmul__', '__imod__',
286 '__imul__', '__index__', '__init__', '__instancecheck__',
287 '__int__', '__invert__', '__ior__', '__ipow__', '__irshift__',
288 '__isub__', '__iter__', '__itruediv__', '__ixor__', '__le__',
289 '__len__', '__length_hint__', '__lshift__', '__lt__', '__matmul__',
290 '__missing__', '__mod__', '__mul__', '__ne__', '__neg__',
291 '__new__', '__next__', '__or__', '__pos__', '__pow__',
292 '__prepare__', '__radd__', '__rand__', '__rdivmod__', '__repr__',
293 '__reversed__', '__rfloordiv__', '__rlshift__', '__rmatmul__',
294 '__rmod__', '__rmul__', '__ror__', '__round__', '__rpow__',
295 '__rrshift__', '__rshift__', '__rsub__', '__rtruediv__',
296 '__rxor__', '__set__', '__setattr__', '__setitem__', '__str__',
297 '__sub__', '__subclasscheck__', '__truediv__',
298 '__xor__'), suffix=r'\b'),
299 Name.Function.Magic),
300 ],
301 'magicvars': [
302 (words((
303 '__annotations__', '__bases__', '__class__', '__closure__',
304 '__code__', '__defaults__', '__dict__', '__doc__', '__file__',
305 '__func__', '__globals__', '__kwdefaults__', '__module__',
306 '__mro__', '__name__', '__objclass__', '__qualname__',
307 '__self__', '__slots__', '__weakref__'), suffix=r'\b'),
308 Name.Variable.Magic),
309 ],
310 'numbers': [
311 (r'(\d(?:_?\d)*\.(?:\d(?:_?\d)*)?|(?:\d(?:_?\d)*)?\.\d(?:_?\d)*)'
312 r'([eE][+-]?\d(?:_?\d)*)?', Number.Float),
313 (r'\d(?:_?\d)*[eE][+-]?\d(?:_?\d)*j?', Number.Float),
314 (r'0[oO](?:_?[0-7])+', Number.Oct),
315 (r'0[bB](?:_?[01])+', Number.Bin),
316 (r'0[xX](?:_?[a-fA-F0-9])+', Number.Hex),
317 (r'\d(?:_?\d)*', Number.Integer),
318 ],
319 'name': [
320 (r'@' + uni_name, Name.Decorator),
321 (r'@', Operator), # new matrix multiplication operator
322 (uni_name, Name),
323 ],
324 'funcname': [
325 include('magicfuncs'),
326 (uni_name, Name.Function, '#pop'),
327 default('#pop'),
328 ],
329 'classname': [
330 (uni_name, Name.Class, '#pop'),
331 ],
332 'import': [
333 (r'(\s+)(as)(\s+)', bygroups(Text, Keyword, Text)),
334 (r'\.', Name.Namespace),
335 (uni_name, Name.Namespace),
336 (r'(\s*)(,)(\s*)', bygroups(Text, Operator, Text)),
337 default('#pop') # all else: go back
338 ],
339 'fromimport': [
340 (r'(\s+)(import)\b', bygroups(Text, Keyword.Namespace), '#pop'),
341 (r'\.', Name.Namespace),
342 # if None occurs here, it's "raise x from None", since None can
343 # never be a module name
344 (r'None\b', Name.Builtin.Pseudo, '#pop'),
345 (uni_name, Name.Namespace),
346 default('#pop'),
347 ],
348 'rfstringescape': [
349 (r'\{\{', String.Escape),
350 (r'\}\}', String.Escape),
351 ],
352 'fstringescape': [
353 include('rfstringescape'),
354 include('stringescape'),
355 ],
356 'bytesescape': [
357 (r'\\([\\abfnrtv"\']|\n|x[a-fA-F0-9]{2}|[0-7]{1,3})', String.Escape)
358 ],
359 'stringescape': [
360 (r'\\(N\{.*?\}|u[a-fA-F0-9]{4}|U[a-fA-F0-9]{8})', String.Escape),
361 include('bytesescape')
362 ],
363 'fstrings-single': fstring_rules(String.Single),
364 'fstrings-double': fstring_rules(String.Double),
365 'strings-single': innerstring_rules(String.Single),
366 'strings-double': innerstring_rules(String.Double),
367 'dqf': [
368 (r'"', String.Double, '#pop'),
369 (r'\\\\|\\"|\\\n', String.Escape), # included here for raw strings
370 include('fstrings-double')
371 ],
372 'sqf': [
373 (r"'", String.Single, '#pop'),
374 (r"\\\\|\\'|\\\n", String.Escape), # included here for raw strings
375 include('fstrings-single')
376 ],
377 'dqs': [
378 (r'"', String.Double, '#pop'),
379 (r'\\\\|\\"|\\\n', String.Escape), # included here for raw strings
380 include('strings-double')
381 ],
382 'sqs': [
383 (r"'", String.Single, '#pop'),
384 (r"\\\\|\\'|\\\n", String.Escape), # included here for raw strings
385 include('strings-single')
386 ],
387 'tdqf': [
388 (r'"""', String.Double, '#pop'),
389 include('fstrings-double'),
390 (r'\n', String.Double)
391 ],
392 'tsqf': [
393 (r"'''", String.Single, '#pop'),
394 include('fstrings-single'),
395 (r'\n', String.Single)
396 ],
397 'tdqs': [
398 (r'"""', String.Double, '#pop'),
399 include('strings-double'),
400 (r'\n', String.Double)
401 ],
402 'tsqs': [
403 (r"'''", String.Single, '#pop'),
404 include('strings-single'),
405 (r'\n', String.Single)
406 ],
407 }
409 def analyse_text(text):
410 return shebang_matches(text, r'pythonw?(3(\.\d)?)?') or \
411 'import ' in text[:1000]
414Python3Lexer = PythonLexer
417class Python2Lexer(RegexLexer):
418 """
419 For Python 2.x source code.
421 .. versionchanged:: 2.5
422 This class has been renamed from ``PythonLexer``. ``PythonLexer`` now
423 refers to the Python 3 variant. File name patterns like ``*.py`` have
424 been moved to Python 3 as well.
425 """
427 name = 'Python 2.x'
428 url = 'http://www.python.org'
429 aliases = ['python2', 'py2']
430 filenames = [] # now taken over by PythonLexer (3.x)
431 mimetypes = ['text/x-python2', 'application/x-python2']
433 def innerstring_rules(ttype):
434 return [
435 # the old style '%s' % (...) string formatting
436 (r'%(\(\w+\))?[-#0 +]*([0-9]+|[*])?(\.([0-9]+|[*]))?'
437 '[hlL]?[E-GXc-giorsux%]', String.Interpol),
438 # backslashes, quotes and formatting signs must be parsed one at a time
439 (r'[^\\\'"%\n]+', ttype),
440 (r'[\'"\\]', ttype),
441 # unhandled string formatting sign
442 (r'%', ttype),
443 # newlines are an error (use "nl" state)
444 ]
446 tokens = {
447 'root': [
448 (r'\n', Whitespace),
449 (r'^(\s*)([rRuUbB]{,2})("""(?:.|\n)*?""")',
450 bygroups(Whitespace, String.Affix, String.Doc)),
451 (r"^(\s*)([rRuUbB]{,2})('''(?:.|\n)*?''')",
452 bygroups(Whitespace, String.Affix, String.Doc)),
453 (r'[^\S\n]+', Text),
454 (r'\A#!.+$', Comment.Hashbang),
455 (r'#.*$', Comment.Single),
456 (r'[]{}:(),;[]', Punctuation),
457 (r'\\\n', Text),
458 (r'\\', Text),
459 (r'(in|is|and|or|not)\b', Operator.Word),
460 (r'!=|==|<<|>>|[-~+/*%=<>&^|.]', Operator),
461 include('keywords'),
462 (r'(def)((?:\s|\\\s)+)', bygroups(Keyword, Text), 'funcname'),
463 (r'(class)((?:\s|\\\s)+)', bygroups(Keyword, Text), 'classname'),
464 (r'(from)((?:\s|\\\s)+)', bygroups(Keyword.Namespace, Text),
465 'fromimport'),
466 (r'(import)((?:\s|\\\s)+)', bygroups(Keyword.Namespace, Text),
467 'import'),
468 include('builtins'),
469 include('magicfuncs'),
470 include('magicvars'),
471 include('backtick'),
472 ('([rR]|[uUbB][rR]|[rR][uUbB])(""")',
473 bygroups(String.Affix, String.Double), 'tdqs'),
474 ("([rR]|[uUbB][rR]|[rR][uUbB])(''')",
475 bygroups(String.Affix, String.Single), 'tsqs'),
476 ('([rR]|[uUbB][rR]|[rR][uUbB])(")',
477 bygroups(String.Affix, String.Double), 'dqs'),
478 ("([rR]|[uUbB][rR]|[rR][uUbB])(')",
479 bygroups(String.Affix, String.Single), 'sqs'),
480 ('([uUbB]?)(""")', bygroups(String.Affix, String.Double),
481 combined('stringescape', 'tdqs')),
482 ("([uUbB]?)(''')", bygroups(String.Affix, String.Single),
483 combined('stringescape', 'tsqs')),
484 ('([uUbB]?)(")', bygroups(String.Affix, String.Double),
485 combined('stringescape', 'dqs')),
486 ("([uUbB]?)(')", bygroups(String.Affix, String.Single),
487 combined('stringescape', 'sqs')),
488 include('name'),
489 include('numbers'),
490 ],
491 'keywords': [
492 (words((
493 'assert', 'break', 'continue', 'del', 'elif', 'else', 'except',
494 'exec', 'finally', 'for', 'global', 'if', 'lambda', 'pass',
495 'print', 'raise', 'return', 'try', 'while', 'yield',
496 'yield from', 'as', 'with'), suffix=r'\b'),
497 Keyword),
498 ],
499 'builtins': [
500 (words((
501 '__import__', 'abs', 'all', 'any', 'apply', 'basestring', 'bin',
502 'bool', 'buffer', 'bytearray', 'bytes', 'callable', 'chr', 'classmethod',
503 'cmp', 'coerce', 'compile', 'complex', 'delattr', 'dict', 'dir', 'divmod',
504 'enumerate', 'eval', 'execfile', 'exit', 'file', 'filter', 'float',
505 'frozenset', 'getattr', 'globals', 'hasattr', 'hash', 'hex', 'id',
506 'input', 'int', 'intern', 'isinstance', 'issubclass', 'iter', 'len',
507 'list', 'locals', 'long', 'map', 'max', 'min', 'next', 'object',
508 'oct', 'open', 'ord', 'pow', 'property', 'range', 'raw_input', 'reduce',
509 'reload', 'repr', 'reversed', 'round', 'set', 'setattr', 'slice',
510 'sorted', 'staticmethod', 'str', 'sum', 'super', 'tuple', 'type',
511 'unichr', 'unicode', 'vars', 'xrange', 'zip'),
512 prefix=r'(?<!\.)', suffix=r'\b'),
513 Name.Builtin),
514 (r'(?<!\.)(self|None|Ellipsis|NotImplemented|False|True|cls'
515 r')\b', Name.Builtin.Pseudo),
516 (words((
517 'ArithmeticError', 'AssertionError', 'AttributeError',
518 'BaseException', 'DeprecationWarning', 'EOFError', 'EnvironmentError',
519 'Exception', 'FloatingPointError', 'FutureWarning', 'GeneratorExit',
520 'IOError', 'ImportError', 'ImportWarning', 'IndentationError',
521 'IndexError', 'KeyError', 'KeyboardInterrupt', 'LookupError',
522 'MemoryError', 'NameError',
523 'NotImplementedError', 'OSError', 'OverflowError', 'OverflowWarning',
524 'PendingDeprecationWarning', 'ReferenceError',
525 'RuntimeError', 'RuntimeWarning', 'StandardError', 'StopIteration',
526 'SyntaxError', 'SyntaxWarning', 'SystemError', 'SystemExit',
527 'TabError', 'TypeError', 'UnboundLocalError', 'UnicodeDecodeError',
528 'UnicodeEncodeError', 'UnicodeError', 'UnicodeTranslateError',
529 'UnicodeWarning', 'UserWarning', 'ValueError', 'VMSError', 'Warning',
530 'WindowsError', 'ZeroDivisionError'), prefix=r'(?<!\.)', suffix=r'\b'),
531 Name.Exception),
532 ],
533 'magicfuncs': [
534 (words((
535 '__abs__', '__add__', '__and__', '__call__', '__cmp__', '__coerce__',
536 '__complex__', '__contains__', '__del__', '__delattr__', '__delete__',
537 '__delitem__', '__delslice__', '__div__', '__divmod__', '__enter__',
538 '__eq__', '__exit__', '__float__', '__floordiv__', '__ge__', '__get__',
539 '__getattr__', '__getattribute__', '__getitem__', '__getslice__', '__gt__',
540 '__hash__', '__hex__', '__iadd__', '__iand__', '__idiv__', '__ifloordiv__',
541 '__ilshift__', '__imod__', '__imul__', '__index__', '__init__',
542 '__instancecheck__', '__int__', '__invert__', '__iop__', '__ior__',
543 '__ipow__', '__irshift__', '__isub__', '__iter__', '__itruediv__',
544 '__ixor__', '__le__', '__len__', '__long__', '__lshift__', '__lt__',
545 '__missing__', '__mod__', '__mul__', '__ne__', '__neg__', '__new__',
546 '__nonzero__', '__oct__', '__op__', '__or__', '__pos__', '__pow__',
547 '__radd__', '__rand__', '__rcmp__', '__rdiv__', '__rdivmod__', '__repr__',
548 '__reversed__', '__rfloordiv__', '__rlshift__', '__rmod__', '__rmul__',
549 '__rop__', '__ror__', '__rpow__', '__rrshift__', '__rshift__', '__rsub__',
550 '__rtruediv__', '__rxor__', '__set__', '__setattr__', '__setitem__',
551 '__setslice__', '__str__', '__sub__', '__subclasscheck__', '__truediv__',
552 '__unicode__', '__xor__'), suffix=r'\b'),
553 Name.Function.Magic),
554 ],
555 'magicvars': [
556 (words((
557 '__bases__', '__class__', '__closure__', '__code__', '__defaults__',
558 '__dict__', '__doc__', '__file__', '__func__', '__globals__',
559 '__metaclass__', '__module__', '__mro__', '__name__', '__self__',
560 '__slots__', '__weakref__'),
561 suffix=r'\b'),
562 Name.Variable.Magic),
563 ],
564 'numbers': [
565 (r'(\d+\.\d*|\d*\.\d+)([eE][+-]?[0-9]+)?j?', Number.Float),
566 (r'\d+[eE][+-]?[0-9]+j?', Number.Float),
567 (r'0[0-7]+j?', Number.Oct),
568 (r'0[bB][01]+', Number.Bin),
569 (r'0[xX][a-fA-F0-9]+', Number.Hex),
570 (r'\d+L', Number.Integer.Long),
571 (r'\d+j?', Number.Integer)
572 ],
573 'backtick': [
574 ('`.*?`', String.Backtick),
575 ],
576 'name': [
577 (r'@[\w.]+', Name.Decorator),
578 (r'[a-zA-Z_]\w*', Name),
579 ],
580 'funcname': [
581 include('magicfuncs'),
582 (r'[a-zA-Z_]\w*', Name.Function, '#pop'),
583 default('#pop'),
584 ],
585 'classname': [
586 (r'[a-zA-Z_]\w*', Name.Class, '#pop')
587 ],
588 'import': [
589 (r'(?:[ \t]|\\\n)+', Text),
590 (r'as\b', Keyword.Namespace),
591 (r',', Operator),
592 (r'[a-zA-Z_][\w.]*', Name.Namespace),
593 default('#pop') # all else: go back
594 ],
595 'fromimport': [
596 (r'(?:[ \t]|\\\n)+', Text),
597 (r'import\b', Keyword.Namespace, '#pop'),
598 # if None occurs here, it's "raise x from None", since None can
599 # never be a module name
600 (r'None\b', Name.Builtin.Pseudo, '#pop'),
601 # sadly, in "raise x from y" y will be highlighted as namespace too
602 (r'[a-zA-Z_.][\w.]*', Name.Namespace),
603 # anything else here also means "raise x from y" and is therefore
604 # not an error
605 default('#pop'),
606 ],
607 'stringescape': [
608 (r'\\([\\abfnrtv"\']|\n|N\{.*?\}|u[a-fA-F0-9]{4}|'
609 r'U[a-fA-F0-9]{8}|x[a-fA-F0-9]{2}|[0-7]{1,3})', String.Escape)
610 ],
611 'strings-single': innerstring_rules(String.Single),
612 'strings-double': innerstring_rules(String.Double),
613 'dqs': [
614 (r'"', String.Double, '#pop'),
615 (r'\\\\|\\"|\\\n', String.Escape), # included here for raw strings
616 include('strings-double')
617 ],
618 'sqs': [
619 (r"'", String.Single, '#pop'),
620 (r"\\\\|\\'|\\\n", String.Escape), # included here for raw strings
621 include('strings-single')
622 ],
623 'tdqs': [
624 (r'"""', String.Double, '#pop'),
625 include('strings-double'),
626 (r'\n', String.Double)
627 ],
628 'tsqs': [
629 (r"'''", String.Single, '#pop'),
630 include('strings-single'),
631 (r'\n', String.Single)
632 ],
633 }
635 def analyse_text(text):
636 return shebang_matches(text, r'pythonw?2(\.\d)?')
639class PythonConsoleLexer(Lexer):
640 """
641 For Python console output or doctests, such as:
643 .. sourcecode:: pycon
645 >>> a = 'foo'
646 >>> print a
647 foo
648 >>> 1 / 0
649 Traceback (most recent call last):
650 File "<stdin>", line 1, in <module>
651 ZeroDivisionError: integer division or modulo by zero
653 Additional options:
655 `python3`
656 Use Python 3 lexer for code. Default is ``True``.
658 .. versionadded:: 1.0
659 .. versionchanged:: 2.5
660 Now defaults to ``True``.
661 """
662 name = 'Python console session'
663 aliases = ['pycon']
664 mimetypes = ['text/x-python-doctest']
666 def __init__(self, **options):
667 self.python3 = get_bool_opt(options, 'python3', True)
668 Lexer.__init__(self, **options)
670 def get_tokens_unprocessed(self, text):
671 if self.python3:
672 pylexer = PythonLexer(**self.options)
673 tblexer = PythonTracebackLexer(**self.options)
674 else:
675 pylexer = Python2Lexer(**self.options)
676 tblexer = Python2TracebackLexer(**self.options)
678 curcode = ''
679 insertions = []
680 curtb = ''
681 tbindex = 0
682 tb = 0
683 for match in line_re.finditer(text):
684 line = match.group()
685 if line.startswith('>>> ') or line.startswith('... '):
686 tb = 0
687 insertions.append((len(curcode),
688 [(0, Generic.Prompt, line[:4])]))
689 curcode += line[4:]
690 elif line.rstrip() == '...' and not tb:
691 # only a new >>> prompt can end an exception block
692 # otherwise an ellipsis in place of the traceback frames
693 # will be mishandled
694 insertions.append((len(curcode),
695 [(0, Generic.Prompt, '...')]))
696 curcode += line[3:]
697 else:
698 if curcode:
699 yield from do_insertions(
700 insertions, pylexer.get_tokens_unprocessed(curcode))
701 curcode = ''
702 insertions = []
703 if (line.startswith('Traceback (most recent call last):') or
704 re.match(' File "[^"]+", line \\d+\\n$', line)):
705 tb = 1
706 curtb = line
707 tbindex = match.start()
708 elif line == 'KeyboardInterrupt\n':
709 yield match.start(), Name.Class, line
710 elif tb:
711 curtb += line
712 if not (line.startswith(' ') or line.strip() == '...'):
713 tb = 0
714 for i, t, v in tblexer.get_tokens_unprocessed(curtb):
715 yield tbindex+i, t, v
716 curtb = ''
717 else:
718 yield match.start(), Generic.Output, line
719 if curcode:
720 yield from do_insertions(insertions,
721 pylexer.get_tokens_unprocessed(curcode))
722 if curtb:
723 for i, t, v in tblexer.get_tokens_unprocessed(curtb):
724 yield tbindex+i, t, v
727class PythonTracebackLexer(RegexLexer):
728 """
729 For Python 3.x tracebacks, with support for chained exceptions.
731 .. versionadded:: 1.0
733 .. versionchanged:: 2.5
734 This is now the default ``PythonTracebackLexer``. It is still available
735 as the alias ``Python3TracebackLexer``.
736 """
738 name = 'Python Traceback'
739 aliases = ['pytb', 'py3tb']
740 filenames = ['*.pytb', '*.py3tb']
741 mimetypes = ['text/x-python-traceback', 'text/x-python3-traceback']
743 tokens = {
744 'root': [
745 (r'\n', Whitespace),
746 (r'^Traceback \(most recent call last\):\n', Generic.Traceback, 'intb'),
747 (r'^During handling of the above exception, another '
748 r'exception occurred:\n\n', Generic.Traceback),
749 (r'^The above exception was the direct cause of the '
750 r'following exception:\n\n', Generic.Traceback),
751 (r'^(?= File "[^"]+", line \d+)', Generic.Traceback, 'intb'),
752 (r'^.*\n', Other),
753 ],
754 'intb': [
755 (r'^( File )("[^"]+")(, line )(\d+)(, in )(.+)(\n)',
756 bygroups(Text, Name.Builtin, Text, Number, Text, Name, Whitespace)),
757 (r'^( File )("[^"]+")(, line )(\d+)(\n)',
758 bygroups(Text, Name.Builtin, Text, Number, Whitespace)),
759 (r'^( )(.+)(\n)',
760 bygroups(Whitespace, using(PythonLexer), Whitespace), 'markers'),
761 (r'^([ \t]*)(\.\.\.)(\n)',
762 bygroups(Whitespace, Comment, Whitespace)), # for doctests...
763 (r'^([^:]+)(: )(.+)(\n)',
764 bygroups(Generic.Error, Text, Name, Whitespace), '#pop'),
765 (r'^([a-zA-Z_][\w.]*)(:?\n)',
766 bygroups(Generic.Error, Whitespace), '#pop')
767 ],
768 'markers': [
769 # Either `PEP 657 <https://www.python.org/dev/peps/pep-0657/>`
770 # error locations in Python 3.11+, or single-caret markers
771 # for syntax errors before that.
772 (r'^( {4,})([~^]+)(\n)',
773 bygroups(Whitespace, Punctuation.Marker, Whitespace),
774 '#pop'),
775 default('#pop'),
776 ],
777 }
780Python3TracebackLexer = PythonTracebackLexer
783class Python2TracebackLexer(RegexLexer):
784 """
785 For Python tracebacks.
787 .. versionadded:: 0.7
789 .. versionchanged:: 2.5
790 This class has been renamed from ``PythonTracebackLexer``.
791 ``PythonTracebackLexer`` now refers to the Python 3 variant.
792 """
794 name = 'Python 2.x Traceback'
795 aliases = ['py2tb']
796 filenames = ['*.py2tb']
797 mimetypes = ['text/x-python2-traceback']
799 tokens = {
800 'root': [
801 # Cover both (most recent call last) and (innermost last)
802 # The optional ^C allows us to catch keyboard interrupt signals.
803 (r'^(\^C)?(Traceback.*\n)',
804 bygroups(Text, Generic.Traceback), 'intb'),
805 # SyntaxError starts with this.
806 (r'^(?= File "[^"]+", line \d+)', Generic.Traceback, 'intb'),
807 (r'^.*\n', Other),
808 ],
809 'intb': [
810 (r'^( File )("[^"]+")(, line )(\d+)(, in )(.+)(\n)',
811 bygroups(Text, Name.Builtin, Text, Number, Text, Name, Whitespace)),
812 (r'^( File )("[^"]+")(, line )(\d+)(\n)',
813 bygroups(Text, Name.Builtin, Text, Number, Whitespace)),
814 (r'^( )(.+)(\n)',
815 bygroups(Text, using(Python2Lexer), Whitespace), 'marker'),
816 (r'^([ \t]*)(\.\.\.)(\n)',
817 bygroups(Text, Comment, Whitespace)), # for doctests...
818 (r'^([^:]+)(: )(.+)(\n)',
819 bygroups(Generic.Error, Text, Name, Whitespace), '#pop'),
820 (r'^([a-zA-Z_]\w*)(:?\n)',
821 bygroups(Generic.Error, Whitespace), '#pop')
822 ],
823 'marker': [
824 # For syntax errors.
825 (r'( {4,})(\^)', bygroups(Text, Punctuation.Marker), '#pop'),
826 default('#pop'),
827 ],
828 }
831class CythonLexer(RegexLexer):
832 """
833 For Pyrex and Cython source code.
835 .. versionadded:: 1.1
836 """
838 name = 'Cython'
839 url = 'http://cython.org'
840 aliases = ['cython', 'pyx', 'pyrex']
841 filenames = ['*.pyx', '*.pxd', '*.pxi']
842 mimetypes = ['text/x-cython', 'application/x-cython']
844 tokens = {
845 'root': [
846 (r'\n', Whitespace),
847 (r'^(\s*)("""(?:.|\n)*?""")', bygroups(Whitespace, String.Doc)),
848 (r"^(\s*)('''(?:.|\n)*?''')", bygroups(Whitespace, String.Doc)),
849 (r'[^\S\n]+', Text),
850 (r'#.*$', Comment),
851 (r'[]{}:(),;[]', Punctuation),
852 (r'\\\n', Whitespace),
853 (r'\\', Text),
854 (r'(in|is|and|or|not)\b', Operator.Word),
855 (r'(<)([a-zA-Z0-9.?]+)(>)',
856 bygroups(Punctuation, Keyword.Type, Punctuation)),
857 (r'!=|==|<<|>>|[-~+/*%=<>&^|.?]', Operator),
858 (r'(from)(\d+)(<=)(\s+)(<)(\d+)(:)',
859 bygroups(Keyword, Number.Integer, Operator, Name, Operator,
860 Name, Punctuation)),
861 include('keywords'),
862 (r'(def|property)(\s+)', bygroups(Keyword, Text), 'funcname'),
863 (r'(cp?def)(\s+)', bygroups(Keyword, Text), 'cdef'),
864 # (should actually start a block with only cdefs)
865 (r'(cdef)(:)', bygroups(Keyword, Punctuation)),
866 (r'(class|struct)(\s+)', bygroups(Keyword, Text), 'classname'),
867 (r'(from)(\s+)', bygroups(Keyword, Text), 'fromimport'),
868 (r'(c?import)(\s+)', bygroups(Keyword, Text), 'import'),
869 include('builtins'),
870 include('backtick'),
871 ('(?:[rR]|[uU][rR]|[rR][uU])"""', String, 'tdqs'),
872 ("(?:[rR]|[uU][rR]|[rR][uU])'''", String, 'tsqs'),
873 ('(?:[rR]|[uU][rR]|[rR][uU])"', String, 'dqs'),
874 ("(?:[rR]|[uU][rR]|[rR][uU])'", String, 'sqs'),
875 ('[uU]?"""', String, combined('stringescape', 'tdqs')),
876 ("[uU]?'''", String, combined('stringescape', 'tsqs')),
877 ('[uU]?"', String, combined('stringescape', 'dqs')),
878 ("[uU]?'", String, combined('stringescape', 'sqs')),
879 include('name'),
880 include('numbers'),
881 ],
882 'keywords': [
883 (words((
884 'assert', 'async', 'await', 'break', 'by', 'continue', 'ctypedef', 'del', 'elif',
885 'else', 'except', 'except?', 'exec', 'finally', 'for', 'fused', 'gil',
886 'global', 'if', 'include', 'lambda', 'nogil', 'pass', 'print',
887 'raise', 'return', 'try', 'while', 'yield', 'as', 'with'), suffix=r'\b'),
888 Keyword),
889 (r'(DEF|IF|ELIF|ELSE)\b', Comment.Preproc),
890 ],
891 'builtins': [
892 (words((
893 '__import__', 'abs', 'all', 'any', 'apply', 'basestring', 'bin', 'bint',
894 'bool', 'buffer', 'bytearray', 'bytes', 'callable', 'chr',
895 'classmethod', 'cmp', 'coerce', 'compile', 'complex', 'delattr',
896 'dict', 'dir', 'divmod', 'enumerate', 'eval', 'execfile', 'exit',
897 'file', 'filter', 'float', 'frozenset', 'getattr', 'globals',
898 'hasattr', 'hash', 'hex', 'id', 'input', 'int', 'intern', 'isinstance',
899 'issubclass', 'iter', 'len', 'list', 'locals', 'long', 'map', 'max',
900 'min', 'next', 'object', 'oct', 'open', 'ord', 'pow', 'property', 'Py_ssize_t',
901 'range', 'raw_input', 'reduce', 'reload', 'repr', 'reversed',
902 'round', 'set', 'setattr', 'slice', 'sorted', 'staticmethod',
903 'str', 'sum', 'super', 'tuple', 'type', 'unichr', 'unicode', 'unsigned',
904 'vars', 'xrange', 'zip'), prefix=r'(?<!\.)', suffix=r'\b'),
905 Name.Builtin),
906 (r'(?<!\.)(self|None|Ellipsis|NotImplemented|False|True|NULL'
907 r')\b', Name.Builtin.Pseudo),
908 (words((
909 'ArithmeticError', 'AssertionError', 'AttributeError',
910 'BaseException', 'DeprecationWarning', 'EOFError', 'EnvironmentError',
911 'Exception', 'FloatingPointError', 'FutureWarning', 'GeneratorExit',
912 'IOError', 'ImportError', 'ImportWarning', 'IndentationError',
913 'IndexError', 'KeyError', 'KeyboardInterrupt', 'LookupError',
914 'MemoryError', 'NameError', 'NotImplemented', 'NotImplementedError',
915 'OSError', 'OverflowError', 'OverflowWarning',
916 'PendingDeprecationWarning', 'ReferenceError', 'RuntimeError',
917 'RuntimeWarning', 'StandardError', 'StopIteration', 'SyntaxError',
918 'SyntaxWarning', 'SystemError', 'SystemExit', 'TabError',
919 'TypeError', 'UnboundLocalError', 'UnicodeDecodeError',
920 'UnicodeEncodeError', 'UnicodeError', 'UnicodeTranslateError',
921 'UnicodeWarning', 'UserWarning', 'ValueError', 'Warning',
922 'ZeroDivisionError'), prefix=r'(?<!\.)', suffix=r'\b'),
923 Name.Exception),
924 ],
925 'numbers': [
926 (r'(\d+\.?\d*|\d*\.\d+)([eE][+-]?[0-9]+)?', Number.Float),
927 (r'0\d+', Number.Oct),
928 (r'0[xX][a-fA-F0-9]+', Number.Hex),
929 (r'\d+L', Number.Integer.Long),
930 (r'\d+', Number.Integer)
931 ],
932 'backtick': [
933 ('`.*?`', String.Backtick),
934 ],
935 'name': [
936 (r'@\w+', Name.Decorator),
937 (r'[a-zA-Z_]\w*', Name),
938 ],
939 'funcname': [
940 (r'[a-zA-Z_]\w*', Name.Function, '#pop')
941 ],
942 'cdef': [
943 (r'(public|readonly|extern|api|inline)\b', Keyword.Reserved),
944 (r'(struct|enum|union|class)\b', Keyword),
945 (r'([a-zA-Z_]\w*)(\s*)(?=[(:#=]|$)',
946 bygroups(Name.Function, Text), '#pop'),
947 (r'([a-zA-Z_]\w*)(\s*)(,)',
948 bygroups(Name.Function, Text, Punctuation)),
949 (r'from\b', Keyword, '#pop'),
950 (r'as\b', Keyword),
951 (r':', Punctuation, '#pop'),
952 (r'(?=["\'])', Text, '#pop'),
953 (r'[a-zA-Z_]\w*', Keyword.Type),
954 (r'.', Text),
955 ],
956 'classname': [
957 (r'[a-zA-Z_]\w*', Name.Class, '#pop')
958 ],
959 'import': [
960 (r'(\s+)(as)(\s+)', bygroups(Text, Keyword, Text)),
961 (r'[a-zA-Z_][\w.]*', Name.Namespace),
962 (r'(\s*)(,)(\s*)', bygroups(Text, Operator, Text)),
963 default('#pop') # all else: go back
964 ],
965 'fromimport': [
966 (r'(\s+)(c?import)\b', bygroups(Text, Keyword), '#pop'),
967 (r'[a-zA-Z_.][\w.]*', Name.Namespace),
968 # ``cdef foo from "header"``, or ``for foo from 0 < i < 10``
969 default('#pop'),
970 ],
971 'stringescape': [
972 (r'\\([\\abfnrtv"\']|\n|N\{.*?\}|u[a-fA-F0-9]{4}|'
973 r'U[a-fA-F0-9]{8}|x[a-fA-F0-9]{2}|[0-7]{1,3})', String.Escape)
974 ],
975 'strings': [
976 (r'%(\([a-zA-Z0-9]+\))?[-#0 +]*([0-9]+|[*])?(\.([0-9]+|[*]))?'
977 '[hlL]?[E-GXc-giorsux%]', String.Interpol),
978 (r'[^\\\'"%\n]+', String),
979 # quotes, percents and backslashes must be parsed one at a time
980 (r'[\'"\\]', String),
981 # unhandled string formatting sign
982 (r'%', String)
983 # newlines are an error (use "nl" state)
984 ],
985 'nl': [
986 (r'\n', String)
987 ],
988 'dqs': [
989 (r'"', String, '#pop'),
990 (r'\\\\|\\"|\\\n', String.Escape), # included here again for raw strings
991 include('strings')
992 ],
993 'sqs': [
994 (r"'", String, '#pop'),
995 (r"\\\\|\\'|\\\n", String.Escape), # included here again for raw strings
996 include('strings')
997 ],
998 'tdqs': [
999 (r'"""', String, '#pop'),
1000 include('strings'),
1001 include('nl')
1002 ],
1003 'tsqs': [
1004 (r"'''", String, '#pop'),
1005 include('strings'),
1006 include('nl')
1007 ],
1008 }
1011class DgLexer(RegexLexer):
1012 """
1013 Lexer for dg,
1014 a functional and object-oriented programming language
1015 running on the CPython 3 VM.
1017 .. versionadded:: 1.6
1018 """
1019 name = 'dg'
1020 aliases = ['dg']
1021 filenames = ['*.dg']
1022 mimetypes = ['text/x-dg']
1024 tokens = {
1025 'root': [
1026 (r'\s+', Text),
1027 (r'#.*?$', Comment.Single),
1029 (r'(?i)0b[01]+', Number.Bin),
1030 (r'(?i)0o[0-7]+', Number.Oct),
1031 (r'(?i)0x[0-9a-f]+', Number.Hex),
1032 (r'(?i)[+-]?[0-9]+\.[0-9]+(e[+-]?[0-9]+)?j?', Number.Float),
1033 (r'(?i)[+-]?[0-9]+e[+-]?\d+j?', Number.Float),
1034 (r'(?i)[+-]?[0-9]+j?', Number.Integer),
1036 (r"(?i)(br|r?b?)'''", String, combined('stringescape', 'tsqs', 'string')),
1037 (r'(?i)(br|r?b?)"""', String, combined('stringescape', 'tdqs', 'string')),
1038 (r"(?i)(br|r?b?)'", String, combined('stringescape', 'sqs', 'string')),
1039 (r'(?i)(br|r?b?)"', String, combined('stringescape', 'dqs', 'string')),
1041 (r"`\w+'*`", Operator),
1042 (r'\b(and|in|is|or|where)\b', Operator.Word),
1043 (r'[!$%&*+\-./:<-@\\^|~;,]+', Operator),
1045 (words((
1046 'bool', 'bytearray', 'bytes', 'classmethod', 'complex', 'dict', 'dict\'',
1047 'float', 'frozenset', 'int', 'list', 'list\'', 'memoryview', 'object',
1048 'property', 'range', 'set', 'set\'', 'slice', 'staticmethod', 'str',
1049 'super', 'tuple', 'tuple\'', 'type'),
1050 prefix=r'(?<!\.)', suffix=r'(?![\'\w])'),
1051 Name.Builtin),
1052 (words((
1053 '__import__', 'abs', 'all', 'any', 'bin', 'bind', 'chr', 'cmp', 'compile',
1054 'complex', 'delattr', 'dir', 'divmod', 'drop', 'dropwhile', 'enumerate',
1055 'eval', 'exhaust', 'filter', 'flip', 'foldl1?', 'format', 'fst',
1056 'getattr', 'globals', 'hasattr', 'hash', 'head', 'hex', 'id', 'init',
1057 'input', 'isinstance', 'issubclass', 'iter', 'iterate', 'last', 'len',
1058 'locals', 'map', 'max', 'min', 'next', 'oct', 'open', 'ord', 'pow',
1059 'print', 'repr', 'reversed', 'round', 'setattr', 'scanl1?', 'snd',
1060 'sorted', 'sum', 'tail', 'take', 'takewhile', 'vars', 'zip'),
1061 prefix=r'(?<!\.)', suffix=r'(?![\'\w])'),
1062 Name.Builtin),
1063 (r"(?<!\.)(self|Ellipsis|NotImplemented|None|True|False)(?!['\w])",
1064 Name.Builtin.Pseudo),
1066 (r"(?<!\.)[A-Z]\w*(Error|Exception|Warning)'*(?!['\w])",
1067 Name.Exception),
1068 (r"(?<!\.)(Exception|GeneratorExit|KeyboardInterrupt|StopIteration|"
1069 r"SystemExit)(?!['\w])", Name.Exception),
1071 (r"(?<![\w.])(except|finally|for|if|import|not|otherwise|raise|"
1072 r"subclass|while|with|yield)(?!['\w])", Keyword.Reserved),
1074 (r"[A-Z_]+'*(?!['\w])", Name),
1075 (r"[A-Z]\w+'*(?!['\w])", Keyword.Type),
1076 (r"\w+'*", Name),
1078 (r'[()]', Punctuation),
1079 (r'.', Error),
1080 ],
1081 'stringescape': [
1082 (r'\\([\\abfnrtv"\']|\n|N\{.*?\}|u[a-fA-F0-9]{4}|'
1083 r'U[a-fA-F0-9]{8}|x[a-fA-F0-9]{2}|[0-7]{1,3})', String.Escape)
1084 ],
1085 'string': [
1086 (r'%(\(\w+\))?[-#0 +]*([0-9]+|[*])?(\.([0-9]+|[*]))?'
1087 '[hlL]?[E-GXc-giorsux%]', String.Interpol),
1088 (r'[^\\\'"%\n]+', String),
1089 # quotes, percents and backslashes must be parsed one at a time
1090 (r'[\'"\\]', String),
1091 # unhandled string formatting sign
1092 (r'%', String),
1093 (r'\n', String)
1094 ],
1095 'dqs': [
1096 (r'"', String, '#pop')
1097 ],
1098 'sqs': [
1099 (r"'", String, '#pop')
1100 ],
1101 'tdqs': [
1102 (r'"""', String, '#pop')
1103 ],
1104 'tsqs': [
1105 (r"'''", String, '#pop')
1106 ],
1107 }
1110class NumPyLexer(PythonLexer):
1111 """
1112 A Python lexer recognizing Numerical Python builtins.
1114 .. versionadded:: 0.10
1115 """
1117 name = 'NumPy'
1118 url = 'https://numpy.org/'
1119 aliases = ['numpy']
1121 # override the mimetypes to not inherit them from python
1122 mimetypes = []
1123 filenames = []
1125 EXTRA_KEYWORDS = {
1126 'abs', 'absolute', 'accumulate', 'add', 'alen', 'all', 'allclose',
1127 'alltrue', 'alterdot', 'amax', 'amin', 'angle', 'any', 'append',
1128 'apply_along_axis', 'apply_over_axes', 'arange', 'arccos', 'arccosh',
1129 'arcsin', 'arcsinh', 'arctan', 'arctan2', 'arctanh', 'argmax', 'argmin',
1130 'argsort', 'argwhere', 'around', 'array', 'array2string', 'array_equal',
1131 'array_equiv', 'array_repr', 'array_split', 'array_str', 'arrayrange',
1132 'asanyarray', 'asarray', 'asarray_chkfinite', 'ascontiguousarray',
1133 'asfarray', 'asfortranarray', 'asmatrix', 'asscalar', 'astype',
1134 'atleast_1d', 'atleast_2d', 'atleast_3d', 'average', 'bartlett',
1135 'base_repr', 'beta', 'binary_repr', 'bincount', 'binomial',
1136 'bitwise_and', 'bitwise_not', 'bitwise_or', 'bitwise_xor', 'blackman',
1137 'bmat', 'broadcast', 'byte_bounds', 'bytes', 'byteswap', 'c_',
1138 'can_cast', 'ceil', 'choose', 'clip', 'column_stack', 'common_type',
1139 'compare_chararrays', 'compress', 'concatenate', 'conj', 'conjugate',
1140 'convolve', 'copy', 'corrcoef', 'correlate', 'cos', 'cosh', 'cov',
1141 'cross', 'cumprod', 'cumproduct', 'cumsum', 'delete', 'deprecate',
1142 'diag', 'diagflat', 'diagonal', 'diff', 'digitize', 'disp', 'divide',
1143 'dot', 'dsplit', 'dstack', 'dtype', 'dump', 'dumps', 'ediff1d', 'empty',
1144 'empty_like', 'equal', 'exp', 'expand_dims', 'expm1', 'extract', 'eye',
1145 'fabs', 'fastCopyAndTranspose', 'fft', 'fftfreq', 'fftshift', 'fill',
1146 'finfo', 'fix', 'flat', 'flatnonzero', 'flatten', 'fliplr', 'flipud',
1147 'floor', 'floor_divide', 'fmod', 'frexp', 'fromarrays', 'frombuffer',
1148 'fromfile', 'fromfunction', 'fromiter', 'frompyfunc', 'fromstring',
1149 'generic', 'get_array_wrap', 'get_include', 'get_numarray_include',
1150 'get_numpy_include', 'get_printoptions', 'getbuffer', 'getbufsize',
1151 'geterr', 'geterrcall', 'geterrobj', 'getfield', 'gradient', 'greater',
1152 'greater_equal', 'gumbel', 'hamming', 'hanning', 'histogram',
1153 'histogram2d', 'histogramdd', 'hsplit', 'hstack', 'hypot', 'i0',
1154 'identity', 'ifft', 'imag', 'index_exp', 'indices', 'inf', 'info',
1155 'inner', 'insert', 'int_asbuffer', 'interp', 'intersect1d',
1156 'intersect1d_nu', 'inv', 'invert', 'iscomplex', 'iscomplexobj',
1157 'isfinite', 'isfortran', 'isinf', 'isnan', 'isneginf', 'isposinf',
1158 'isreal', 'isrealobj', 'isscalar', 'issctype', 'issubclass_',
1159 'issubdtype', 'issubsctype', 'item', 'itemset', 'iterable', 'ix_',
1160 'kaiser', 'kron', 'ldexp', 'left_shift', 'less', 'less_equal', 'lexsort',
1161 'linspace', 'load', 'loads', 'loadtxt', 'log', 'log10', 'log1p', 'log2',
1162 'logical_and', 'logical_not', 'logical_or', 'logical_xor', 'logspace',
1163 'lstsq', 'mat', 'matrix', 'max', 'maximum', 'maximum_sctype',
1164 'may_share_memory', 'mean', 'median', 'meshgrid', 'mgrid', 'min',
1165 'minimum', 'mintypecode', 'mod', 'modf', 'msort', 'multiply', 'nan',
1166 'nan_to_num', 'nanargmax', 'nanargmin', 'nanmax', 'nanmin', 'nansum',
1167 'ndenumerate', 'ndim', 'ndindex', 'negative', 'newaxis', 'newbuffer',
1168 'newbyteorder', 'nonzero', 'not_equal', 'obj2sctype', 'ogrid', 'ones',
1169 'ones_like', 'outer', 'permutation', 'piecewise', 'pinv', 'pkgload',
1170 'place', 'poisson', 'poly', 'poly1d', 'polyadd', 'polyder', 'polydiv',
1171 'polyfit', 'polyint', 'polymul', 'polysub', 'polyval', 'power', 'prod',
1172 'product', 'ptp', 'put', 'putmask', 'r_', 'randint', 'random_integers',
1173 'random_sample', 'ranf', 'rank', 'ravel', 'real', 'real_if_close',
1174 'recarray', 'reciprocal', 'reduce', 'remainder', 'repeat', 'require',
1175 'reshape', 'resize', 'restoredot', 'right_shift', 'rint', 'roll',
1176 'rollaxis', 'roots', 'rot90', 'round', 'round_', 'row_stack', 's_',
1177 'sample', 'savetxt', 'sctype2char', 'searchsorted', 'seed', 'select',
1178 'set_numeric_ops', 'set_printoptions', 'set_string_function',
1179 'setbufsize', 'setdiff1d', 'seterr', 'seterrcall', 'seterrobj',
1180 'setfield', 'setflags', 'setmember1d', 'setxor1d', 'shape',
1181 'show_config', 'shuffle', 'sign', 'signbit', 'sin', 'sinc', 'sinh',
1182 'size', 'slice', 'solve', 'sometrue', 'sort', 'sort_complex', 'source',
1183 'split', 'sqrt', 'square', 'squeeze', 'standard_normal', 'std',
1184 'subtract', 'sum', 'svd', 'swapaxes', 'take', 'tan', 'tanh', 'tensordot',
1185 'test', 'tile', 'tofile', 'tolist', 'tostring', 'trace', 'transpose',
1186 'trapz', 'tri', 'tril', 'trim_zeros', 'triu', 'true_divide', 'typeDict',
1187 'typename', 'uniform', 'union1d', 'unique', 'unique1d', 'unravel_index',
1188 'unwrap', 'vander', 'var', 'vdot', 'vectorize', 'view', 'vonmises',
1189 'vsplit', 'vstack', 'weibull', 'where', 'who', 'zeros', 'zeros_like'
1190 }
1192 def get_tokens_unprocessed(self, text):
1193 for index, token, value in \
1194 PythonLexer.get_tokens_unprocessed(self, text):
1195 if token is Name and value in self.EXTRA_KEYWORDS:
1196 yield index, Keyword.Pseudo, value
1197 else:
1198 yield index, token, value
1200 def analyse_text(text):
1201 ltext = text[:1000]
1202 return (shebang_matches(text, r'pythonw?(3(\.\d)?)?') or
1203 'import ' in ltext) \
1204 and ('import numpy' in ltext or 'from numpy import' in ltext)