Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/pygments/lexers/python.py: 81%
94 statements
« prev ^ index » next coverage.py v7.4.4, created at 2024-04-20 06:09 +0000
1"""
2 pygments.lexers.python
3 ~~~~~~~~~~~~~~~~~~~~~~
5 Lexers for Python and related languages.
7 :copyright: Copyright 2006-2023 by the Pygments team, see AUTHORS.
8 :license: BSD, see LICENSE for details.
9"""
11import re
12import keyword
14from pygments.lexer import DelegatingLexer, Lexer, RegexLexer, include, \
15 bygroups, using, default, words, combined, do_insertions, this, line_re
16from pygments.util import get_bool_opt, shebang_matches
17from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
18 Number, Punctuation, Generic, Other, Error, Whitespace
19from pygments import unistring as uni
21__all__ = ['PythonLexer', 'PythonConsoleLexer', 'PythonTracebackLexer',
22 'Python2Lexer', 'Python2TracebackLexer',
23 'CythonLexer', 'DgLexer', 'NumPyLexer']
26class PythonLexer(RegexLexer):
27 """
28 For Python source code (version 3.x).
30 .. versionadded:: 0.10
32 .. versionchanged:: 2.5
33 This is now the default ``PythonLexer``. It is still available as the
34 alias ``Python3Lexer``.
35 """
37 name = 'Python'
38 url = 'https://www.python.org'
39 aliases = ['python', 'py', 'sage', 'python3', 'py3', 'bazel', 'starlark']
40 filenames = [
41 '*.py',
42 '*.pyw',
43 # Type stubs
44 '*.pyi',
45 # Jython
46 '*.jy',
47 # Sage
48 '*.sage',
49 # SCons
50 '*.sc',
51 'SConstruct',
52 'SConscript',
53 # Skylark/Starlark (used by Bazel, Buck, and Pants)
54 '*.bzl',
55 'BUCK',
56 'BUILD',
57 'BUILD.bazel',
58 'WORKSPACE',
59 # Twisted Application infrastructure
60 '*.tac',
61 ]
62 mimetypes = ['text/x-python', 'application/x-python',
63 'text/x-python3', 'application/x-python3']
65 uni_name = "[%s][%s]*" % (uni.xid_start, uni.xid_continue)
67 def innerstring_rules(ttype):
68 return [
69 # the old style '%s' % (...) string formatting (still valid in Py3)
70 (r'%(\(\w+\))?[-#0 +]*([0-9]+|[*])?(\.([0-9]+|[*]))?'
71 '[hlL]?[E-GXc-giorsaux%]', String.Interpol),
72 # the new style '{}'.format(...) string formatting
73 (r'\{'
74 r'((\w+)((\.\w+)|(\[[^\]]+\]))*)?' # field name
75 r'(\![sra])?' # conversion
76 r'(\:(.?[<>=\^])?[-+ ]?#?0?(\d+)?,?(\.\d+)?[E-GXb-gnosx%]?)?'
77 r'\}', String.Interpol),
79 # backslashes, quotes and formatting signs must be parsed one at a time
80 (r'[^\\\'"%{\n]+', ttype),
81 (r'[\'"\\]', ttype),
82 # unhandled string formatting sign
83 (r'%|(\{{1,2})', ttype)
84 # newlines are an error (use "nl" state)
85 ]
87 def fstring_rules(ttype):
88 return [
89 # Assuming that a '}' is the closing brace after format specifier.
90 # Sadly, this means that we won't detect syntax error. But it's
91 # more important to parse correct syntax correctly, than to
92 # highlight invalid syntax.
93 (r'\}', String.Interpol),
94 (r'\{', String.Interpol, 'expr-inside-fstring'),
95 # backslashes, quotes and formatting signs must be parsed one at a time
96 (r'[^\\\'"{}\n]+', ttype),
97 (r'[\'"\\]', ttype),
98 # newlines are an error (use "nl" state)
99 ]
101 tokens = {
102 'root': [
103 (r'\n', Whitespace),
104 (r'^(\s*)([rRuUbB]{,2})("""(?:.|\n)*?""")',
105 bygroups(Whitespace, String.Affix, String.Doc)),
106 (r"^(\s*)([rRuUbB]{,2})('''(?:.|\n)*?''')",
107 bygroups(Whitespace, String.Affix, String.Doc)),
108 (r'\A#!.+$', Comment.Hashbang),
109 (r'#.*$', Comment.Single),
110 (r'\\\n', Text),
111 (r'\\', Text),
112 include('keywords'),
113 include('soft-keywords'),
114 (r'(def)((?:\s|\\\s)+)', bygroups(Keyword, Text), 'funcname'),
115 (r'(class)((?:\s|\\\s)+)', bygroups(Keyword, Text), 'classname'),
116 (r'(from)((?:\s|\\\s)+)', bygroups(Keyword.Namespace, Text),
117 'fromimport'),
118 (r'(import)((?:\s|\\\s)+)', bygroups(Keyword.Namespace, Text),
119 'import'),
120 include('expr'),
121 ],
122 'expr': [
123 # raw f-strings
124 ('(?i)(rf|fr)(""")',
125 bygroups(String.Affix, String.Double),
126 combined('rfstringescape', 'tdqf')),
127 ("(?i)(rf|fr)(''')",
128 bygroups(String.Affix, String.Single),
129 combined('rfstringescape', 'tsqf')),
130 ('(?i)(rf|fr)(")',
131 bygroups(String.Affix, String.Double),
132 combined('rfstringescape', 'dqf')),
133 ("(?i)(rf|fr)(')",
134 bygroups(String.Affix, String.Single),
135 combined('rfstringescape', 'sqf')),
136 # non-raw f-strings
137 ('([fF])(""")', bygroups(String.Affix, String.Double),
138 combined('fstringescape', 'tdqf')),
139 ("([fF])(''')", bygroups(String.Affix, String.Single),
140 combined('fstringescape', 'tsqf')),
141 ('([fF])(")', bygroups(String.Affix, String.Double),
142 combined('fstringescape', 'dqf')),
143 ("([fF])(')", bygroups(String.Affix, String.Single),
144 combined('fstringescape', 'sqf')),
145 # raw bytes and strings
146 ('(?i)(rb|br|r)(""")',
147 bygroups(String.Affix, String.Double), 'tdqs'),
148 ("(?i)(rb|br|r)(''')",
149 bygroups(String.Affix, String.Single), 'tsqs'),
150 ('(?i)(rb|br|r)(")',
151 bygroups(String.Affix, String.Double), 'dqs'),
152 ("(?i)(rb|br|r)(')",
153 bygroups(String.Affix, String.Single), 'sqs'),
154 # non-raw strings
155 ('([uU]?)(""")', bygroups(String.Affix, String.Double),
156 combined('stringescape', 'tdqs')),
157 ("([uU]?)(''')", bygroups(String.Affix, String.Single),
158 combined('stringescape', 'tsqs')),
159 ('([uU]?)(")', bygroups(String.Affix, String.Double),
160 combined('stringescape', 'dqs')),
161 ("([uU]?)(')", bygroups(String.Affix, String.Single),
162 combined('stringescape', 'sqs')),
163 # non-raw bytes
164 ('([bB])(""")', bygroups(String.Affix, String.Double),
165 combined('bytesescape', 'tdqs')),
166 ("([bB])(''')", bygroups(String.Affix, String.Single),
167 combined('bytesescape', 'tsqs')),
168 ('([bB])(")', bygroups(String.Affix, String.Double),
169 combined('bytesescape', 'dqs')),
170 ("([bB])(')", bygroups(String.Affix, String.Single),
171 combined('bytesescape', 'sqs')),
173 (r'[^\S\n]+', Text),
174 include('numbers'),
175 (r'!=|==|<<|>>|:=|[-~+/*%=<>&^|.]', Operator),
176 (r'[]{}:(),;[]', Punctuation),
177 (r'(in|is|and|or|not)\b', Operator.Word),
178 include('expr-keywords'),
179 include('builtins'),
180 include('magicfuncs'),
181 include('magicvars'),
182 include('name'),
183 ],
184 'expr-inside-fstring': [
185 (r'[{([]', Punctuation, 'expr-inside-fstring-inner'),
186 # without format specifier
187 (r'(=\s*)?' # debug (https://bugs.python.org/issue36817)
188 r'(\![sraf])?' # conversion
189 r'\}', String.Interpol, '#pop'),
190 # with format specifier
191 # we'll catch the remaining '}' in the outer scope
192 (r'(=\s*)?' # debug (https://bugs.python.org/issue36817)
193 r'(\![sraf])?' # conversion
194 r':', String.Interpol, '#pop'),
195 (r'\s+', Whitespace), # allow new lines
196 include('expr'),
197 ],
198 'expr-inside-fstring-inner': [
199 (r'[{([]', Punctuation, 'expr-inside-fstring-inner'),
200 (r'[])}]', Punctuation, '#pop'),
201 (r'\s+', Whitespace), # allow new lines
202 include('expr'),
203 ],
204 'expr-keywords': [
205 # Based on https://docs.python.org/3/reference/expressions.html
206 (words((
207 'async for', 'await', 'else', 'for', 'if', 'lambda',
208 'yield', 'yield from'), suffix=r'\b'),
209 Keyword),
210 (words(('True', 'False', 'None'), suffix=r'\b'), Keyword.Constant),
211 ],
212 'keywords': [
213 (words((
214 'assert', 'async', 'await', 'break', 'continue', 'del', 'elif',
215 'else', 'except', 'finally', 'for', 'global', 'if', 'lambda',
216 'pass', 'raise', 'nonlocal', 'return', 'try', 'while', 'yield',
217 'yield from', 'as', 'with'), suffix=r'\b'),
218 Keyword),
219 (words(('True', 'False', 'None'), suffix=r'\b'), Keyword.Constant),
220 ],
221 'soft-keywords': [
222 # `match`, `case` and `_` soft keywords
223 (r'(^[ \t]*)' # at beginning of line + possible indentation
224 r'(match|case)\b' # a possible keyword
225 r'(?![ \t]*(?:' # not followed by...
226 r'[:,;=^&|@~)\]}]|(?:' + # characters and keywords that mean this isn't
227 r'|'.join(keyword.kwlist) + r')\b))', # pattern matching
228 bygroups(Text, Keyword), 'soft-keywords-inner'),
229 ],
230 'soft-keywords-inner': [
231 # optional `_` keyword
232 (r'(\s+)([^\n_]*)(_\b)', bygroups(Whitespace, using(this), Keyword)),
233 default('#pop')
234 ],
235 'builtins': [
236 (words((
237 '__import__', 'abs', 'aiter', 'all', 'any', 'bin', 'bool', 'bytearray',
238 'breakpoint', 'bytes', 'callable', 'chr', 'classmethod', 'compile',
239 'complex', 'delattr', 'dict', 'dir', 'divmod', 'enumerate', 'eval',
240 'filter', 'float', 'format', 'frozenset', 'getattr', 'globals',
241 'hasattr', 'hash', 'hex', 'id', 'input', 'int', 'isinstance',
242 'issubclass', 'iter', 'len', 'list', 'locals', 'map', 'max',
243 'memoryview', 'min', 'next', 'object', 'oct', 'open', 'ord', 'pow',
244 'print', 'property', 'range', 'repr', 'reversed', 'round', 'set',
245 'setattr', 'slice', 'sorted', 'staticmethod', 'str', 'sum', 'super',
246 'tuple', 'type', 'vars', 'zip'), prefix=r'(?<!\.)', suffix=r'\b'),
247 Name.Builtin),
248 (r'(?<!\.)(self|Ellipsis|NotImplemented|cls)\b', Name.Builtin.Pseudo),
249 (words((
250 'ArithmeticError', 'AssertionError', 'AttributeError',
251 'BaseException', 'BufferError', 'BytesWarning', 'DeprecationWarning',
252 'EOFError', 'EnvironmentError', 'Exception', 'FloatingPointError',
253 'FutureWarning', 'GeneratorExit', 'IOError', 'ImportError',
254 'ImportWarning', 'IndentationError', 'IndexError', 'KeyError',
255 'KeyboardInterrupt', 'LookupError', 'MemoryError', 'NameError',
256 'NotImplementedError', 'OSError', 'OverflowError',
257 'PendingDeprecationWarning', 'ReferenceError', 'ResourceWarning',
258 'RuntimeError', 'RuntimeWarning', 'StopIteration',
259 'SyntaxError', 'SyntaxWarning', 'SystemError', 'SystemExit',
260 'TabError', 'TypeError', 'UnboundLocalError', 'UnicodeDecodeError',
261 'UnicodeEncodeError', 'UnicodeError', 'UnicodeTranslateError',
262 'UnicodeWarning', 'UserWarning', 'ValueError', 'VMSError',
263 'Warning', 'WindowsError', 'ZeroDivisionError',
264 # new builtin exceptions from PEP 3151
265 'BlockingIOError', 'ChildProcessError', 'ConnectionError',
266 'BrokenPipeError', 'ConnectionAbortedError', 'ConnectionRefusedError',
267 'ConnectionResetError', 'FileExistsError', 'FileNotFoundError',
268 'InterruptedError', 'IsADirectoryError', 'NotADirectoryError',
269 'PermissionError', 'ProcessLookupError', 'TimeoutError',
270 # others new in Python 3
271 'StopAsyncIteration', 'ModuleNotFoundError', 'RecursionError',
272 'EncodingWarning'),
273 prefix=r'(?<!\.)', suffix=r'\b'),
274 Name.Exception),
275 ],
276 'magicfuncs': [
277 (words((
278 '__abs__', '__add__', '__aenter__', '__aexit__', '__aiter__',
279 '__and__', '__anext__', '__await__', '__bool__', '__bytes__',
280 '__call__', '__complex__', '__contains__', '__del__', '__delattr__',
281 '__delete__', '__delitem__', '__dir__', '__divmod__', '__enter__',
282 '__eq__', '__exit__', '__float__', '__floordiv__', '__format__',
283 '__ge__', '__get__', '__getattr__', '__getattribute__',
284 '__getitem__', '__gt__', '__hash__', '__iadd__', '__iand__',
285 '__ifloordiv__', '__ilshift__', '__imatmul__', '__imod__',
286 '__imul__', '__index__', '__init__', '__instancecheck__',
287 '__int__', '__invert__', '__ior__', '__ipow__', '__irshift__',
288 '__isub__', '__iter__', '__itruediv__', '__ixor__', '__le__',
289 '__len__', '__length_hint__', '__lshift__', '__lt__', '__matmul__',
290 '__missing__', '__mod__', '__mul__', '__ne__', '__neg__',
291 '__new__', '__next__', '__or__', '__pos__', '__pow__',
292 '__prepare__', '__radd__', '__rand__', '__rdivmod__', '__repr__',
293 '__reversed__', '__rfloordiv__', '__rlshift__', '__rmatmul__',
294 '__rmod__', '__rmul__', '__ror__', '__round__', '__rpow__',
295 '__rrshift__', '__rshift__', '__rsub__', '__rtruediv__',
296 '__rxor__', '__set__', '__setattr__', '__setitem__', '__str__',
297 '__sub__', '__subclasscheck__', '__truediv__',
298 '__xor__'), suffix=r'\b'),
299 Name.Function.Magic),
300 ],
301 'magicvars': [
302 (words((
303 '__annotations__', '__bases__', '__class__', '__closure__',
304 '__code__', '__defaults__', '__dict__', '__doc__', '__file__',
305 '__func__', '__globals__', '__kwdefaults__', '__module__',
306 '__mro__', '__name__', '__objclass__', '__qualname__',
307 '__self__', '__slots__', '__weakref__'), suffix=r'\b'),
308 Name.Variable.Magic),
309 ],
310 'numbers': [
311 (r'(\d(?:_?\d)*\.(?:\d(?:_?\d)*)?|(?:\d(?:_?\d)*)?\.\d(?:_?\d)*)'
312 r'([eE][+-]?\d(?:_?\d)*)?', Number.Float),
313 (r'\d(?:_?\d)*[eE][+-]?\d(?:_?\d)*j?', Number.Float),
314 (r'0[oO](?:_?[0-7])+', Number.Oct),
315 (r'0[bB](?:_?[01])+', Number.Bin),
316 (r'0[xX](?:_?[a-fA-F0-9])+', Number.Hex),
317 (r'\d(?:_?\d)*', Number.Integer),
318 ],
319 'name': [
320 (r'@' + uni_name, Name.Decorator),
321 (r'@', Operator), # new matrix multiplication operator
322 (uni_name, Name),
323 ],
324 'funcname': [
325 include('magicfuncs'),
326 (uni_name, Name.Function, '#pop'),
327 default('#pop'),
328 ],
329 'classname': [
330 (uni_name, Name.Class, '#pop'),
331 ],
332 'import': [
333 (r'(\s+)(as)(\s+)', bygroups(Text, Keyword, Text)),
334 (r'\.', Name.Namespace),
335 (uni_name, Name.Namespace),
336 (r'(\s*)(,)(\s*)', bygroups(Text, Operator, Text)),
337 default('#pop') # all else: go back
338 ],
339 'fromimport': [
340 (r'(\s+)(import)\b', bygroups(Text, Keyword.Namespace), '#pop'),
341 (r'\.', Name.Namespace),
342 # if None occurs here, it's "raise x from None", since None can
343 # never be a module name
344 (r'None\b', Keyword.Constant, '#pop'),
345 (uni_name, Name.Namespace),
346 default('#pop'),
347 ],
348 'rfstringescape': [
349 (r'\{\{', String.Escape),
350 (r'\}\}', String.Escape),
351 ],
352 'fstringescape': [
353 include('rfstringescape'),
354 include('stringescape'),
355 ],
356 'bytesescape': [
357 (r'\\([\\abfnrtv"\']|\n|x[a-fA-F0-9]{2}|[0-7]{1,3})', String.Escape)
358 ],
359 'stringescape': [
360 (r'\\(N\{.*?\}|u[a-fA-F0-9]{4}|U[a-fA-F0-9]{8})', String.Escape),
361 include('bytesescape')
362 ],
363 'fstrings-single': fstring_rules(String.Single),
364 'fstrings-double': fstring_rules(String.Double),
365 'strings-single': innerstring_rules(String.Single),
366 'strings-double': innerstring_rules(String.Double),
367 'dqf': [
368 (r'"', String.Double, '#pop'),
369 (r'\\\\|\\"|\\\n', String.Escape), # included here for raw strings
370 include('fstrings-double')
371 ],
372 'sqf': [
373 (r"'", String.Single, '#pop'),
374 (r"\\\\|\\'|\\\n", String.Escape), # included here for raw strings
375 include('fstrings-single')
376 ],
377 'dqs': [
378 (r'"', String.Double, '#pop'),
379 (r'\\\\|\\"|\\\n', String.Escape), # included here for raw strings
380 include('strings-double')
381 ],
382 'sqs': [
383 (r"'", String.Single, '#pop'),
384 (r"\\\\|\\'|\\\n", String.Escape), # included here for raw strings
385 include('strings-single')
386 ],
387 'tdqf': [
388 (r'"""', String.Double, '#pop'),
389 include('fstrings-double'),
390 (r'\n', String.Double)
391 ],
392 'tsqf': [
393 (r"'''", String.Single, '#pop'),
394 include('fstrings-single'),
395 (r'\n', String.Single)
396 ],
397 'tdqs': [
398 (r'"""', String.Double, '#pop'),
399 include('strings-double'),
400 (r'\n', String.Double)
401 ],
402 'tsqs': [
403 (r"'''", String.Single, '#pop'),
404 include('strings-single'),
405 (r'\n', String.Single)
406 ],
407 }
409 def analyse_text(text):
410 return shebang_matches(text, r'pythonw?(3(\.\d)?)?') or \
411 'import ' in text[:1000]
414Python3Lexer = PythonLexer
417class Python2Lexer(RegexLexer):
418 """
419 For Python 2.x source code.
421 .. versionchanged:: 2.5
422 This class has been renamed from ``PythonLexer``. ``PythonLexer`` now
423 refers to the Python 3 variant. File name patterns like ``*.py`` have
424 been moved to Python 3 as well.
425 """
427 name = 'Python 2.x'
428 url = 'https://www.python.org'
429 aliases = ['python2', 'py2']
430 filenames = [] # now taken over by PythonLexer (3.x)
431 mimetypes = ['text/x-python2', 'application/x-python2']
433 def innerstring_rules(ttype):
434 return [
435 # the old style '%s' % (...) string formatting
436 (r'%(\(\w+\))?[-#0 +]*([0-9]+|[*])?(\.([0-9]+|[*]))?'
437 '[hlL]?[E-GXc-giorsux%]', String.Interpol),
438 # backslashes, quotes and formatting signs must be parsed one at a time
439 (r'[^\\\'"%\n]+', ttype),
440 (r'[\'"\\]', ttype),
441 # unhandled string formatting sign
442 (r'%', ttype),
443 # newlines are an error (use "nl" state)
444 ]
446 tokens = {
447 'root': [
448 (r'\n', Whitespace),
449 (r'^(\s*)([rRuUbB]{,2})("""(?:.|\n)*?""")',
450 bygroups(Whitespace, String.Affix, String.Doc)),
451 (r"^(\s*)([rRuUbB]{,2})('''(?:.|\n)*?''')",
452 bygroups(Whitespace, String.Affix, String.Doc)),
453 (r'[^\S\n]+', Text),
454 (r'\A#!.+$', Comment.Hashbang),
455 (r'#.*$', Comment.Single),
456 (r'[]{}:(),;[]', Punctuation),
457 (r'\\\n', Text),
458 (r'\\', Text),
459 (r'(in|is|and|or|not)\b', Operator.Word),
460 (r'!=|==|<<|>>|[-~+/*%=<>&^|.]', Operator),
461 include('keywords'),
462 (r'(def)((?:\s|\\\s)+)', bygroups(Keyword, Text), 'funcname'),
463 (r'(class)((?:\s|\\\s)+)', bygroups(Keyword, Text), 'classname'),
464 (r'(from)((?:\s|\\\s)+)', bygroups(Keyword.Namespace, Text),
465 'fromimport'),
466 (r'(import)((?:\s|\\\s)+)', bygroups(Keyword.Namespace, Text),
467 'import'),
468 include('builtins'),
469 include('magicfuncs'),
470 include('magicvars'),
471 include('backtick'),
472 ('([rR]|[uUbB][rR]|[rR][uUbB])(""")',
473 bygroups(String.Affix, String.Double), 'tdqs'),
474 ("([rR]|[uUbB][rR]|[rR][uUbB])(''')",
475 bygroups(String.Affix, String.Single), 'tsqs'),
476 ('([rR]|[uUbB][rR]|[rR][uUbB])(")',
477 bygroups(String.Affix, String.Double), 'dqs'),
478 ("([rR]|[uUbB][rR]|[rR][uUbB])(')",
479 bygroups(String.Affix, String.Single), 'sqs'),
480 ('([uUbB]?)(""")', bygroups(String.Affix, String.Double),
481 combined('stringescape', 'tdqs')),
482 ("([uUbB]?)(''')", bygroups(String.Affix, String.Single),
483 combined('stringescape', 'tsqs')),
484 ('([uUbB]?)(")', bygroups(String.Affix, String.Double),
485 combined('stringescape', 'dqs')),
486 ("([uUbB]?)(')", bygroups(String.Affix, String.Single),
487 combined('stringescape', 'sqs')),
488 include('name'),
489 include('numbers'),
490 ],
491 'keywords': [
492 (words((
493 'assert', 'break', 'continue', 'del', 'elif', 'else', 'except',
494 'exec', 'finally', 'for', 'global', 'if', 'lambda', 'pass',
495 'print', 'raise', 'return', 'try', 'while', 'yield',
496 'yield from', 'as', 'with'), suffix=r'\b'),
497 Keyword),
498 ],
499 'builtins': [
500 (words((
501 '__import__', 'abs', 'all', 'any', 'apply', 'basestring', 'bin',
502 'bool', 'buffer', 'bytearray', 'bytes', 'callable', 'chr', 'classmethod',
503 'cmp', 'coerce', 'compile', 'complex', 'delattr', 'dict', 'dir', 'divmod',
504 'enumerate', 'eval', 'execfile', 'exit', 'file', 'filter', 'float',
505 'frozenset', 'getattr', 'globals', 'hasattr', 'hash', 'hex', 'id',
506 'input', 'int', 'intern', 'isinstance', 'issubclass', 'iter', 'len',
507 'list', 'locals', 'long', 'map', 'max', 'min', 'next', 'object',
508 'oct', 'open', 'ord', 'pow', 'property', 'range', 'raw_input', 'reduce',
509 'reload', 'repr', 'reversed', 'round', 'set', 'setattr', 'slice',
510 'sorted', 'staticmethod', 'str', 'sum', 'super', 'tuple', 'type',
511 'unichr', 'unicode', 'vars', 'xrange', 'zip'),
512 prefix=r'(?<!\.)', suffix=r'\b'),
513 Name.Builtin),
514 (r'(?<!\.)(self|None|Ellipsis|NotImplemented|False|True|cls'
515 r')\b', Name.Builtin.Pseudo),
516 (words((
517 'ArithmeticError', 'AssertionError', 'AttributeError',
518 'BaseException', 'DeprecationWarning', 'EOFError', 'EnvironmentError',
519 'Exception', 'FloatingPointError', 'FutureWarning', 'GeneratorExit',
520 'IOError', 'ImportError', 'ImportWarning', 'IndentationError',
521 'IndexError', 'KeyError', 'KeyboardInterrupt', 'LookupError',
522 'MemoryError', 'NameError',
523 'NotImplementedError', 'OSError', 'OverflowError', 'OverflowWarning',
524 'PendingDeprecationWarning', 'ReferenceError',
525 'RuntimeError', 'RuntimeWarning', 'StandardError', 'StopIteration',
526 'SyntaxError', 'SyntaxWarning', 'SystemError', 'SystemExit',
527 'TabError', 'TypeError', 'UnboundLocalError', 'UnicodeDecodeError',
528 'UnicodeEncodeError', 'UnicodeError', 'UnicodeTranslateError',
529 'UnicodeWarning', 'UserWarning', 'ValueError', 'VMSError', 'Warning',
530 'WindowsError', 'ZeroDivisionError'), prefix=r'(?<!\.)', suffix=r'\b'),
531 Name.Exception),
532 ],
533 'magicfuncs': [
534 (words((
535 '__abs__', '__add__', '__and__', '__call__', '__cmp__', '__coerce__',
536 '__complex__', '__contains__', '__del__', '__delattr__', '__delete__',
537 '__delitem__', '__delslice__', '__div__', '__divmod__', '__enter__',
538 '__eq__', '__exit__', '__float__', '__floordiv__', '__ge__', '__get__',
539 '__getattr__', '__getattribute__', '__getitem__', '__getslice__', '__gt__',
540 '__hash__', '__hex__', '__iadd__', '__iand__', '__idiv__', '__ifloordiv__',
541 '__ilshift__', '__imod__', '__imul__', '__index__', '__init__',
542 '__instancecheck__', '__int__', '__invert__', '__iop__', '__ior__',
543 '__ipow__', '__irshift__', '__isub__', '__iter__', '__itruediv__',
544 '__ixor__', '__le__', '__len__', '__long__', '__lshift__', '__lt__',
545 '__missing__', '__mod__', '__mul__', '__ne__', '__neg__', '__new__',
546 '__nonzero__', '__oct__', '__op__', '__or__', '__pos__', '__pow__',
547 '__radd__', '__rand__', '__rcmp__', '__rdiv__', '__rdivmod__', '__repr__',
548 '__reversed__', '__rfloordiv__', '__rlshift__', '__rmod__', '__rmul__',
549 '__rop__', '__ror__', '__rpow__', '__rrshift__', '__rshift__', '__rsub__',
550 '__rtruediv__', '__rxor__', '__set__', '__setattr__', '__setitem__',
551 '__setslice__', '__str__', '__sub__', '__subclasscheck__', '__truediv__',
552 '__unicode__', '__xor__'), suffix=r'\b'),
553 Name.Function.Magic),
554 ],
555 'magicvars': [
556 (words((
557 '__bases__', '__class__', '__closure__', '__code__', '__defaults__',
558 '__dict__', '__doc__', '__file__', '__func__', '__globals__',
559 '__metaclass__', '__module__', '__mro__', '__name__', '__self__',
560 '__slots__', '__weakref__'),
561 suffix=r'\b'),
562 Name.Variable.Magic),
563 ],
564 'numbers': [
565 (r'(\d+\.\d*|\d*\.\d+)([eE][+-]?[0-9]+)?j?', Number.Float),
566 (r'\d+[eE][+-]?[0-9]+j?', Number.Float),
567 (r'0[0-7]+j?', Number.Oct),
568 (r'0[bB][01]+', Number.Bin),
569 (r'0[xX][a-fA-F0-9]+', Number.Hex),
570 (r'\d+L', Number.Integer.Long),
571 (r'\d+j?', Number.Integer)
572 ],
573 'backtick': [
574 ('`.*?`', String.Backtick),
575 ],
576 'name': [
577 (r'@[\w.]+', Name.Decorator),
578 (r'[a-zA-Z_]\w*', Name),
579 ],
580 'funcname': [
581 include('magicfuncs'),
582 (r'[a-zA-Z_]\w*', Name.Function, '#pop'),
583 default('#pop'),
584 ],
585 'classname': [
586 (r'[a-zA-Z_]\w*', Name.Class, '#pop')
587 ],
588 'import': [
589 (r'(?:[ \t]|\\\n)+', Text),
590 (r'as\b', Keyword.Namespace),
591 (r',', Operator),
592 (r'[a-zA-Z_][\w.]*', Name.Namespace),
593 default('#pop') # all else: go back
594 ],
595 'fromimport': [
596 (r'(?:[ \t]|\\\n)+', Text),
597 (r'import\b', Keyword.Namespace, '#pop'),
598 # if None occurs here, it's "raise x from None", since None can
599 # never be a module name
600 (r'None\b', Name.Builtin.Pseudo, '#pop'),
601 # sadly, in "raise x from y" y will be highlighted as namespace too
602 (r'[a-zA-Z_.][\w.]*', Name.Namespace),
603 # anything else here also means "raise x from y" and is therefore
604 # not an error
605 default('#pop'),
606 ],
607 'stringescape': [
608 (r'\\([\\abfnrtv"\']|\n|N\{.*?\}|u[a-fA-F0-9]{4}|'
609 r'U[a-fA-F0-9]{8}|x[a-fA-F0-9]{2}|[0-7]{1,3})', String.Escape)
610 ],
611 'strings-single': innerstring_rules(String.Single),
612 'strings-double': innerstring_rules(String.Double),
613 'dqs': [
614 (r'"', String.Double, '#pop'),
615 (r'\\\\|\\"|\\\n', String.Escape), # included here for raw strings
616 include('strings-double')
617 ],
618 'sqs': [
619 (r"'", String.Single, '#pop'),
620 (r"\\\\|\\'|\\\n", String.Escape), # included here for raw strings
621 include('strings-single')
622 ],
623 'tdqs': [
624 (r'"""', String.Double, '#pop'),
625 include('strings-double'),
626 (r'\n', String.Double)
627 ],
628 'tsqs': [
629 (r"'''", String.Single, '#pop'),
630 include('strings-single'),
631 (r'\n', String.Single)
632 ],
633 }
635 def analyse_text(text):
636 return shebang_matches(text, r'pythonw?2(\.\d)?')
638class _PythonConsoleLexerBase(RegexLexer):
639 name = 'Python console session'
640 aliases = ['pycon']
641 mimetypes = ['text/x-python-doctest']
643 """Auxiliary lexer for `PythonConsoleLexer`.
645 Code tokens are output as ``Token.Other.Code``, traceback tokens as
646 ``Token.Other.Traceback``.
647 """
648 tokens = {
649 'root': [
650 (r'(>>> )(.*\n)', bygroups(Generic.Prompt, Other.Code), 'continuations'),
651 # This happens, e.g., when tracebacks are embedded in documentation;
652 # trailing whitespaces are often stripped in such contexts.
653 (r'(>>>)(\n)', bygroups(Generic.Prompt, Whitespace)),
654 (r'(\^C)?Traceback \(most recent call last\):\n', Other.Traceback, 'traceback'),
655 # SyntaxError starts with this
656 (r' File "[^"]+", line \d+', Other.Traceback, 'traceback'),
657 (r'.*\n', Generic.Output),
658 ],
659 'continuations': [
660 (r'(\.\.\. )(.*\n)', bygroups(Generic.Prompt, Other.Code)),
661 # See above.
662 (r'(\.\.\.)(\n)', bygroups(Generic.Prompt, Whitespace)),
663 default('#pop'),
664 ],
665 'traceback': [
666 # As soon as we see a traceback, consume everything until the next
667 # >>> prompt.
668 (r'(?=>>>( |$))', Text, '#pop'),
669 (r'(KeyboardInterrupt)(\n)', bygroups(Name.Class, Whitespace)),
670 (r'.*\n', Other.Traceback),
671 ],
672 }
674class PythonConsoleLexer(DelegatingLexer):
675 """
676 For Python console output or doctests, such as:
678 .. sourcecode:: pycon
680 >>> a = 'foo'
681 >>> print(a)
682 foo
683 >>> 1 / 0
684 Traceback (most recent call last):
685 File "<stdin>", line 1, in <module>
686 ZeroDivisionError: integer division or modulo by zero
688 Additional options:
690 `python3`
691 Use Python 3 lexer for code. Default is ``True``.
693 .. versionadded:: 1.0
694 .. versionchanged:: 2.5
695 Now defaults to ``True``.
696 """
698 name = 'Python console session'
699 aliases = ['pycon']
700 mimetypes = ['text/x-python-doctest']
702 def __init__(self, **options):
703 python3 = get_bool_opt(options, 'python3', True)
704 if python3:
705 pylexer = PythonLexer
706 tblexer = PythonTracebackLexer
707 else:
708 pylexer = Python2Lexer
709 tblexer = Python2TracebackLexer
710 # We have two auxiliary lexers. Use DelegatingLexer twice with
711 # different tokens. TODO: DelegatingLexer should support this
712 # directly, by accepting a tuplet of auxiliary lexers and a tuple of
713 # distinguishing tokens. Then we wouldn't need this intermediary
714 # class.
715 class _ReplaceInnerCode(DelegatingLexer):
716 def __init__(self, **options):
717 super().__init__(pylexer, _PythonConsoleLexerBase, Other.Code, **options)
718 super().__init__(tblexer, _ReplaceInnerCode, Other.Traceback, **options)
720class PythonTracebackLexer(RegexLexer):
721 """
722 For Python 3.x tracebacks, with support for chained exceptions.
724 .. versionadded:: 1.0
726 .. versionchanged:: 2.5
727 This is now the default ``PythonTracebackLexer``. It is still available
728 as the alias ``Python3TracebackLexer``.
729 """
731 name = 'Python Traceback'
732 aliases = ['pytb', 'py3tb']
733 filenames = ['*.pytb', '*.py3tb']
734 mimetypes = ['text/x-python-traceback', 'text/x-python3-traceback']
736 tokens = {
737 'root': [
738 (r'\n', Whitespace),
739 (r'^(\^C)?Traceback \(most recent call last\):\n', Generic.Traceback, 'intb'),
740 (r'^During handling of the above exception, another '
741 r'exception occurred:\n\n', Generic.Traceback),
742 (r'^The above exception was the direct cause of the '
743 r'following exception:\n\n', Generic.Traceback),
744 (r'^(?= File "[^"]+", line \d+)', Generic.Traceback, 'intb'),
745 (r'^.*\n', Other),
746 ],
747 'intb': [
748 (r'^( File )("[^"]+")(, line )(\d+)(, in )(.+)(\n)',
749 bygroups(Text, Name.Builtin, Text, Number, Text, Name, Whitespace)),
750 (r'^( File )("[^"]+")(, line )(\d+)(\n)',
751 bygroups(Text, Name.Builtin, Text, Number, Whitespace)),
752 (r'^( )(.+)(\n)',
753 bygroups(Whitespace, using(PythonLexer), Whitespace), 'markers'),
754 (r'^([ \t]*)(\.\.\.)(\n)',
755 bygroups(Whitespace, Comment, Whitespace)), # for doctests...
756 (r'^([^:]+)(: )(.+)(\n)',
757 bygroups(Generic.Error, Text, Name, Whitespace), '#pop'),
758 (r'^([a-zA-Z_][\w.]*)(:?\n)',
759 bygroups(Generic.Error, Whitespace), '#pop'),
760 default('#pop'),
761 ],
762 'markers': [
763 # Either `PEP 657 <https://www.python.org/dev/peps/pep-0657/>`
764 # error locations in Python 3.11+, or single-caret markers
765 # for syntax errors before that.
766 (r'^( {4,})([~^]+)(\n)',
767 bygroups(Whitespace, Punctuation.Marker, Whitespace),
768 '#pop'),
769 default('#pop'),
770 ],
771 }
774Python3TracebackLexer = PythonTracebackLexer
777class Python2TracebackLexer(RegexLexer):
778 """
779 For Python tracebacks.
781 .. versionadded:: 0.7
783 .. versionchanged:: 2.5
784 This class has been renamed from ``PythonTracebackLexer``.
785 ``PythonTracebackLexer`` now refers to the Python 3 variant.
786 """
788 name = 'Python 2.x Traceback'
789 aliases = ['py2tb']
790 filenames = ['*.py2tb']
791 mimetypes = ['text/x-python2-traceback']
793 tokens = {
794 'root': [
795 # Cover both (most recent call last) and (innermost last)
796 # The optional ^C allows us to catch keyboard interrupt signals.
797 (r'^(\^C)?(Traceback.*\n)',
798 bygroups(Text, Generic.Traceback), 'intb'),
799 # SyntaxError starts with this.
800 (r'^(?= File "[^"]+", line \d+)', Generic.Traceback, 'intb'),
801 (r'^.*\n', Other),
802 ],
803 'intb': [
804 (r'^( File )("[^"]+")(, line )(\d+)(, in )(.+)(\n)',
805 bygroups(Text, Name.Builtin, Text, Number, Text, Name, Whitespace)),
806 (r'^( File )("[^"]+")(, line )(\d+)(\n)',
807 bygroups(Text, Name.Builtin, Text, Number, Whitespace)),
808 (r'^( )(.+)(\n)',
809 bygroups(Text, using(Python2Lexer), Whitespace), 'marker'),
810 (r'^([ \t]*)(\.\.\.)(\n)',
811 bygroups(Text, Comment, Whitespace)), # for doctests...
812 (r'^([^:]+)(: )(.+)(\n)',
813 bygroups(Generic.Error, Text, Name, Whitespace), '#pop'),
814 (r'^([a-zA-Z_]\w*)(:?\n)',
815 bygroups(Generic.Error, Whitespace), '#pop')
816 ],
817 'marker': [
818 # For syntax errors.
819 (r'( {4,})(\^)', bygroups(Text, Punctuation.Marker), '#pop'),
820 default('#pop'),
821 ],
822 }
825class CythonLexer(RegexLexer):
826 """
827 For Pyrex and Cython source code.
829 .. versionadded:: 1.1
830 """
832 name = 'Cython'
833 url = 'https://cython.org'
834 aliases = ['cython', 'pyx', 'pyrex']
835 filenames = ['*.pyx', '*.pxd', '*.pxi']
836 mimetypes = ['text/x-cython', 'application/x-cython']
838 tokens = {
839 'root': [
840 (r'\n', Whitespace),
841 (r'^(\s*)("""(?:.|\n)*?""")', bygroups(Whitespace, String.Doc)),
842 (r"^(\s*)('''(?:.|\n)*?''')", bygroups(Whitespace, String.Doc)),
843 (r'[^\S\n]+', Text),
844 (r'#.*$', Comment),
845 (r'[]{}:(),;[]', Punctuation),
846 (r'\\\n', Whitespace),
847 (r'\\', Text),
848 (r'(in|is|and|or|not)\b', Operator.Word),
849 (r'(<)([a-zA-Z0-9.?]+)(>)',
850 bygroups(Punctuation, Keyword.Type, Punctuation)),
851 (r'!=|==|<<|>>|[-~+/*%=<>&^|.?]', Operator),
852 (r'(from)(\d+)(<=)(\s+)(<)(\d+)(:)',
853 bygroups(Keyword, Number.Integer, Operator, Name, Operator,
854 Name, Punctuation)),
855 include('keywords'),
856 (r'(def|property)(\s+)', bygroups(Keyword, Text), 'funcname'),
857 (r'(cp?def)(\s+)', bygroups(Keyword, Text), 'cdef'),
858 # (should actually start a block with only cdefs)
859 (r'(cdef)(:)', bygroups(Keyword, Punctuation)),
860 (r'(class|struct)(\s+)', bygroups(Keyword, Text), 'classname'),
861 (r'(from)(\s+)', bygroups(Keyword, Text), 'fromimport'),
862 (r'(c?import)(\s+)', bygroups(Keyword, Text), 'import'),
863 include('builtins'),
864 include('backtick'),
865 ('(?:[rR]|[uU][rR]|[rR][uU])"""', String, 'tdqs'),
866 ("(?:[rR]|[uU][rR]|[rR][uU])'''", String, 'tsqs'),
867 ('(?:[rR]|[uU][rR]|[rR][uU])"', String, 'dqs'),
868 ("(?:[rR]|[uU][rR]|[rR][uU])'", String, 'sqs'),
869 ('[uU]?"""', String, combined('stringescape', 'tdqs')),
870 ("[uU]?'''", String, combined('stringescape', 'tsqs')),
871 ('[uU]?"', String, combined('stringescape', 'dqs')),
872 ("[uU]?'", String, combined('stringescape', 'sqs')),
873 include('name'),
874 include('numbers'),
875 ],
876 'keywords': [
877 (words((
878 'assert', 'async', 'await', 'break', 'by', 'continue', 'ctypedef', 'del', 'elif',
879 'else', 'except', 'except?', 'exec', 'finally', 'for', 'fused', 'gil',
880 'global', 'if', 'include', 'lambda', 'nogil', 'pass', 'print',
881 'raise', 'return', 'try', 'while', 'yield', 'as', 'with'), suffix=r'\b'),
882 Keyword),
883 (r'(DEF|IF|ELIF|ELSE)\b', Comment.Preproc),
884 ],
885 'builtins': [
886 (words((
887 '__import__', 'abs', 'all', 'any', 'apply', 'basestring', 'bin', 'bint',
888 'bool', 'buffer', 'bytearray', 'bytes', 'callable', 'chr',
889 'classmethod', 'cmp', 'coerce', 'compile', 'complex', 'delattr',
890 'dict', 'dir', 'divmod', 'enumerate', 'eval', 'execfile', 'exit',
891 'file', 'filter', 'float', 'frozenset', 'getattr', 'globals',
892 'hasattr', 'hash', 'hex', 'id', 'input', 'int', 'intern', 'isinstance',
893 'issubclass', 'iter', 'len', 'list', 'locals', 'long', 'map', 'max',
894 'min', 'next', 'object', 'oct', 'open', 'ord', 'pow', 'property', 'Py_ssize_t',
895 'range', 'raw_input', 'reduce', 'reload', 'repr', 'reversed',
896 'round', 'set', 'setattr', 'slice', 'sorted', 'staticmethod',
897 'str', 'sum', 'super', 'tuple', 'type', 'unichr', 'unicode', 'unsigned',
898 'vars', 'xrange', 'zip'), prefix=r'(?<!\.)', suffix=r'\b'),
899 Name.Builtin),
900 (r'(?<!\.)(self|None|Ellipsis|NotImplemented|False|True|NULL'
901 r')\b', Name.Builtin.Pseudo),
902 (words((
903 'ArithmeticError', 'AssertionError', 'AttributeError',
904 'BaseException', 'DeprecationWarning', 'EOFError', 'EnvironmentError',
905 'Exception', 'FloatingPointError', 'FutureWarning', 'GeneratorExit',
906 'IOError', 'ImportError', 'ImportWarning', 'IndentationError',
907 'IndexError', 'KeyError', 'KeyboardInterrupt', 'LookupError',
908 'MemoryError', 'NameError', 'NotImplemented', 'NotImplementedError',
909 'OSError', 'OverflowError', 'OverflowWarning',
910 'PendingDeprecationWarning', 'ReferenceError', 'RuntimeError',
911 'RuntimeWarning', 'StandardError', 'StopIteration', 'SyntaxError',
912 'SyntaxWarning', 'SystemError', 'SystemExit', 'TabError',
913 'TypeError', 'UnboundLocalError', 'UnicodeDecodeError',
914 'UnicodeEncodeError', 'UnicodeError', 'UnicodeTranslateError',
915 'UnicodeWarning', 'UserWarning', 'ValueError', 'Warning',
916 'ZeroDivisionError'), prefix=r'(?<!\.)', suffix=r'\b'),
917 Name.Exception),
918 ],
919 'numbers': [
920 (r'(\d+\.?\d*|\d*\.\d+)([eE][+-]?[0-9]+)?', Number.Float),
921 (r'0\d+', Number.Oct),
922 (r'0[xX][a-fA-F0-9]+', Number.Hex),
923 (r'\d+L', Number.Integer.Long),
924 (r'\d+', Number.Integer)
925 ],
926 'backtick': [
927 ('`.*?`', String.Backtick),
928 ],
929 'name': [
930 (r'@\w+', Name.Decorator),
931 (r'[a-zA-Z_]\w*', Name),
932 ],
933 'funcname': [
934 (r'[a-zA-Z_]\w*', Name.Function, '#pop')
935 ],
936 'cdef': [
937 (r'(public|readonly|extern|api|inline)\b', Keyword.Reserved),
938 (r'(struct|enum|union|class)\b', Keyword),
939 (r'([a-zA-Z_]\w*)(\s*)(?=[(:#=]|$)',
940 bygroups(Name.Function, Text), '#pop'),
941 (r'([a-zA-Z_]\w*)(\s*)(,)',
942 bygroups(Name.Function, Text, Punctuation)),
943 (r'from\b', Keyword, '#pop'),
944 (r'as\b', Keyword),
945 (r':', Punctuation, '#pop'),
946 (r'(?=["\'])', Text, '#pop'),
947 (r'[a-zA-Z_]\w*', Keyword.Type),
948 (r'.', Text),
949 ],
950 'classname': [
951 (r'[a-zA-Z_]\w*', Name.Class, '#pop')
952 ],
953 'import': [
954 (r'(\s+)(as)(\s+)', bygroups(Text, Keyword, Text)),
955 (r'[a-zA-Z_][\w.]*', Name.Namespace),
956 (r'(\s*)(,)(\s*)', bygroups(Text, Operator, Text)),
957 default('#pop') # all else: go back
958 ],
959 'fromimport': [
960 (r'(\s+)(c?import)\b', bygroups(Text, Keyword), '#pop'),
961 (r'[a-zA-Z_.][\w.]*', Name.Namespace),
962 # ``cdef foo from "header"``, or ``for foo from 0 < i < 10``
963 default('#pop'),
964 ],
965 'stringescape': [
966 (r'\\([\\abfnrtv"\']|\n|N\{.*?\}|u[a-fA-F0-9]{4}|'
967 r'U[a-fA-F0-9]{8}|x[a-fA-F0-9]{2}|[0-7]{1,3})', String.Escape)
968 ],
969 'strings': [
970 (r'%(\([a-zA-Z0-9]+\))?[-#0 +]*([0-9]+|[*])?(\.([0-9]+|[*]))?'
971 '[hlL]?[E-GXc-giorsux%]', String.Interpol),
972 (r'[^\\\'"%\n]+', String),
973 # quotes, percents and backslashes must be parsed one at a time
974 (r'[\'"\\]', String),
975 # unhandled string formatting sign
976 (r'%', String)
977 # newlines are an error (use "nl" state)
978 ],
979 'nl': [
980 (r'\n', String)
981 ],
982 'dqs': [
983 (r'"', String, '#pop'),
984 (r'\\\\|\\"|\\\n', String.Escape), # included here again for raw strings
985 include('strings')
986 ],
987 'sqs': [
988 (r"'", String, '#pop'),
989 (r"\\\\|\\'|\\\n", String.Escape), # included here again for raw strings
990 include('strings')
991 ],
992 'tdqs': [
993 (r'"""', String, '#pop'),
994 include('strings'),
995 include('nl')
996 ],
997 'tsqs': [
998 (r"'''", String, '#pop'),
999 include('strings'),
1000 include('nl')
1001 ],
1002 }
1005class DgLexer(RegexLexer):
1006 """
1007 Lexer for dg,
1008 a functional and object-oriented programming language
1009 running on the CPython 3 VM.
1011 .. versionadded:: 1.6
1012 """
1013 name = 'dg'
1014 aliases = ['dg']
1015 filenames = ['*.dg']
1016 mimetypes = ['text/x-dg']
1018 tokens = {
1019 'root': [
1020 (r'\s+', Text),
1021 (r'#.*?$', Comment.Single),
1023 (r'(?i)0b[01]+', Number.Bin),
1024 (r'(?i)0o[0-7]+', Number.Oct),
1025 (r'(?i)0x[0-9a-f]+', Number.Hex),
1026 (r'(?i)[+-]?[0-9]+\.[0-9]+(e[+-]?[0-9]+)?j?', Number.Float),
1027 (r'(?i)[+-]?[0-9]+e[+-]?\d+j?', Number.Float),
1028 (r'(?i)[+-]?[0-9]+j?', Number.Integer),
1030 (r"(?i)(br|r?b?)'''", String, combined('stringescape', 'tsqs', 'string')),
1031 (r'(?i)(br|r?b?)"""', String, combined('stringescape', 'tdqs', 'string')),
1032 (r"(?i)(br|r?b?)'", String, combined('stringescape', 'sqs', 'string')),
1033 (r'(?i)(br|r?b?)"', String, combined('stringescape', 'dqs', 'string')),
1035 (r"`\w+'*`", Operator),
1036 (r'\b(and|in|is|or|where)\b', Operator.Word),
1037 (r'[!$%&*+\-./:<-@\\^|~;,]+', Operator),
1039 (words((
1040 'bool', 'bytearray', 'bytes', 'classmethod', 'complex', 'dict', 'dict\'',
1041 'float', 'frozenset', 'int', 'list', 'list\'', 'memoryview', 'object',
1042 'property', 'range', 'set', 'set\'', 'slice', 'staticmethod', 'str',
1043 'super', 'tuple', 'tuple\'', 'type'),
1044 prefix=r'(?<!\.)', suffix=r'(?![\'\w])'),
1045 Name.Builtin),
1046 (words((
1047 '__import__', 'abs', 'all', 'any', 'bin', 'bind', 'chr', 'cmp', 'compile',
1048 'complex', 'delattr', 'dir', 'divmod', 'drop', 'dropwhile', 'enumerate',
1049 'eval', 'exhaust', 'filter', 'flip', 'foldl1?', 'format', 'fst',
1050 'getattr', 'globals', 'hasattr', 'hash', 'head', 'hex', 'id', 'init',
1051 'input', 'isinstance', 'issubclass', 'iter', 'iterate', 'last', 'len',
1052 'locals', 'map', 'max', 'min', 'next', 'oct', 'open', 'ord', 'pow',
1053 'print', 'repr', 'reversed', 'round', 'setattr', 'scanl1?', 'snd',
1054 'sorted', 'sum', 'tail', 'take', 'takewhile', 'vars', 'zip'),
1055 prefix=r'(?<!\.)', suffix=r'(?![\'\w])'),
1056 Name.Builtin),
1057 (r"(?<!\.)(self|Ellipsis|NotImplemented|None|True|False)(?!['\w])",
1058 Name.Builtin.Pseudo),
1060 (r"(?<!\.)[A-Z]\w*(Error|Exception|Warning)'*(?!['\w])",
1061 Name.Exception),
1062 (r"(?<!\.)(Exception|GeneratorExit|KeyboardInterrupt|StopIteration|"
1063 r"SystemExit)(?!['\w])", Name.Exception),
1065 (r"(?<![\w.])(except|finally|for|if|import|not|otherwise|raise|"
1066 r"subclass|while|with|yield)(?!['\w])", Keyword.Reserved),
1068 (r"[A-Z_]+'*(?!['\w])", Name),
1069 (r"[A-Z]\w+'*(?!['\w])", Keyword.Type),
1070 (r"\w+'*", Name),
1072 (r'[()]', Punctuation),
1073 (r'.', Error),
1074 ],
1075 'stringescape': [
1076 (r'\\([\\abfnrtv"\']|\n|N\{.*?\}|u[a-fA-F0-9]{4}|'
1077 r'U[a-fA-F0-9]{8}|x[a-fA-F0-9]{2}|[0-7]{1,3})', String.Escape)
1078 ],
1079 'string': [
1080 (r'%(\(\w+\))?[-#0 +]*([0-9]+|[*])?(\.([0-9]+|[*]))?'
1081 '[hlL]?[E-GXc-giorsux%]', String.Interpol),
1082 (r'[^\\\'"%\n]+', String),
1083 # quotes, percents and backslashes must be parsed one at a time
1084 (r'[\'"\\]', String),
1085 # unhandled string formatting sign
1086 (r'%', String),
1087 (r'\n', String)
1088 ],
1089 'dqs': [
1090 (r'"', String, '#pop')
1091 ],
1092 'sqs': [
1093 (r"'", String, '#pop')
1094 ],
1095 'tdqs': [
1096 (r'"""', String, '#pop')
1097 ],
1098 'tsqs': [
1099 (r"'''", String, '#pop')
1100 ],
1101 }
1104class NumPyLexer(PythonLexer):
1105 """
1106 A Python lexer recognizing Numerical Python builtins.
1108 .. versionadded:: 0.10
1109 """
1111 name = 'NumPy'
1112 url = 'https://numpy.org/'
1113 aliases = ['numpy']
1115 # override the mimetypes to not inherit them from python
1116 mimetypes = []
1117 filenames = []
1119 EXTRA_KEYWORDS = {
1120 'abs', 'absolute', 'accumulate', 'add', 'alen', 'all', 'allclose',
1121 'alltrue', 'alterdot', 'amax', 'amin', 'angle', 'any', 'append',
1122 'apply_along_axis', 'apply_over_axes', 'arange', 'arccos', 'arccosh',
1123 'arcsin', 'arcsinh', 'arctan', 'arctan2', 'arctanh', 'argmax', 'argmin',
1124 'argsort', 'argwhere', 'around', 'array', 'array2string', 'array_equal',
1125 'array_equiv', 'array_repr', 'array_split', 'array_str', 'arrayrange',
1126 'asanyarray', 'asarray', 'asarray_chkfinite', 'ascontiguousarray',
1127 'asfarray', 'asfortranarray', 'asmatrix', 'asscalar', 'astype',
1128 'atleast_1d', 'atleast_2d', 'atleast_3d', 'average', 'bartlett',
1129 'base_repr', 'beta', 'binary_repr', 'bincount', 'binomial',
1130 'bitwise_and', 'bitwise_not', 'bitwise_or', 'bitwise_xor', 'blackman',
1131 'bmat', 'broadcast', 'byte_bounds', 'bytes', 'byteswap', 'c_',
1132 'can_cast', 'ceil', 'choose', 'clip', 'column_stack', 'common_type',
1133 'compare_chararrays', 'compress', 'concatenate', 'conj', 'conjugate',
1134 'convolve', 'copy', 'corrcoef', 'correlate', 'cos', 'cosh', 'cov',
1135 'cross', 'cumprod', 'cumproduct', 'cumsum', 'delete', 'deprecate',
1136 'diag', 'diagflat', 'diagonal', 'diff', 'digitize', 'disp', 'divide',
1137 'dot', 'dsplit', 'dstack', 'dtype', 'dump', 'dumps', 'ediff1d', 'empty',
1138 'empty_like', 'equal', 'exp', 'expand_dims', 'expm1', 'extract', 'eye',
1139 'fabs', 'fastCopyAndTranspose', 'fft', 'fftfreq', 'fftshift', 'fill',
1140 'finfo', 'fix', 'flat', 'flatnonzero', 'flatten', 'fliplr', 'flipud',
1141 'floor', 'floor_divide', 'fmod', 'frexp', 'fromarrays', 'frombuffer',
1142 'fromfile', 'fromfunction', 'fromiter', 'frompyfunc', 'fromstring',
1143 'generic', 'get_array_wrap', 'get_include', 'get_numarray_include',
1144 'get_numpy_include', 'get_printoptions', 'getbuffer', 'getbufsize',
1145 'geterr', 'geterrcall', 'geterrobj', 'getfield', 'gradient', 'greater',
1146 'greater_equal', 'gumbel', 'hamming', 'hanning', 'histogram',
1147 'histogram2d', 'histogramdd', 'hsplit', 'hstack', 'hypot', 'i0',
1148 'identity', 'ifft', 'imag', 'index_exp', 'indices', 'inf', 'info',
1149 'inner', 'insert', 'int_asbuffer', 'interp', 'intersect1d',
1150 'intersect1d_nu', 'inv', 'invert', 'iscomplex', 'iscomplexobj',
1151 'isfinite', 'isfortran', 'isinf', 'isnan', 'isneginf', 'isposinf',
1152 'isreal', 'isrealobj', 'isscalar', 'issctype', 'issubclass_',
1153 'issubdtype', 'issubsctype', 'item', 'itemset', 'iterable', 'ix_',
1154 'kaiser', 'kron', 'ldexp', 'left_shift', 'less', 'less_equal', 'lexsort',
1155 'linspace', 'load', 'loads', 'loadtxt', 'log', 'log10', 'log1p', 'log2',
1156 'logical_and', 'logical_not', 'logical_or', 'logical_xor', 'logspace',
1157 'lstsq', 'mat', 'matrix', 'max', 'maximum', 'maximum_sctype',
1158 'may_share_memory', 'mean', 'median', 'meshgrid', 'mgrid', 'min',
1159 'minimum', 'mintypecode', 'mod', 'modf', 'msort', 'multiply', 'nan',
1160 'nan_to_num', 'nanargmax', 'nanargmin', 'nanmax', 'nanmin', 'nansum',
1161 'ndenumerate', 'ndim', 'ndindex', 'negative', 'newaxis', 'newbuffer',
1162 'newbyteorder', 'nonzero', 'not_equal', 'obj2sctype', 'ogrid', 'ones',
1163 'ones_like', 'outer', 'permutation', 'piecewise', 'pinv', 'pkgload',
1164 'place', 'poisson', 'poly', 'poly1d', 'polyadd', 'polyder', 'polydiv',
1165 'polyfit', 'polyint', 'polymul', 'polysub', 'polyval', 'power', 'prod',
1166 'product', 'ptp', 'put', 'putmask', 'r_', 'randint', 'random_integers',
1167 'random_sample', 'ranf', 'rank', 'ravel', 'real', 'real_if_close',
1168 'recarray', 'reciprocal', 'reduce', 'remainder', 'repeat', 'require',
1169 'reshape', 'resize', 'restoredot', 'right_shift', 'rint', 'roll',
1170 'rollaxis', 'roots', 'rot90', 'round', 'round_', 'row_stack', 's_',
1171 'sample', 'savetxt', 'sctype2char', 'searchsorted', 'seed', 'select',
1172 'set_numeric_ops', 'set_printoptions', 'set_string_function',
1173 'setbufsize', 'setdiff1d', 'seterr', 'seterrcall', 'seterrobj',
1174 'setfield', 'setflags', 'setmember1d', 'setxor1d', 'shape',
1175 'show_config', 'shuffle', 'sign', 'signbit', 'sin', 'sinc', 'sinh',
1176 'size', 'slice', 'solve', 'sometrue', 'sort', 'sort_complex', 'source',
1177 'split', 'sqrt', 'square', 'squeeze', 'standard_normal', 'std',
1178 'subtract', 'sum', 'svd', 'swapaxes', 'take', 'tan', 'tanh', 'tensordot',
1179 'test', 'tile', 'tofile', 'tolist', 'tostring', 'trace', 'transpose',
1180 'trapz', 'tri', 'tril', 'trim_zeros', 'triu', 'true_divide', 'typeDict',
1181 'typename', 'uniform', 'union1d', 'unique', 'unique1d', 'unravel_index',
1182 'unwrap', 'vander', 'var', 'vdot', 'vectorize', 'view', 'vonmises',
1183 'vsplit', 'vstack', 'weibull', 'where', 'who', 'zeros', 'zeros_like'
1184 }
1186 def get_tokens_unprocessed(self, text):
1187 for index, token, value in \
1188 PythonLexer.get_tokens_unprocessed(self, text):
1189 if token is Name and value in self.EXTRA_KEYWORDS:
1190 yield index, Keyword.Pseudo, value
1191 else:
1192 yield index, token, value
1194 def analyse_text(text):
1195 ltext = text[:1000]
1196 return (shebang_matches(text, r'pythonw?(3(\.\d)?)?') or
1197 'import ' in ltext) \
1198 and ('import numpy' in ltext or 'from numpy import' in ltext)