Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/pygments/lexers/javascript.py: 78%
155 statements
« prev ^ index » next coverage.py v7.2.7, created at 2023-07-01 06:54 +0000
1"""
2 pygments.lexers.javascript
3 ~~~~~~~~~~~~~~~~~~~~~~~~~~
5 Lexers for JavaScript and related languages.
7 :copyright: Copyright 2006-2023 by the Pygments team, see AUTHORS.
8 :license: BSD, see LICENSE for details.
9"""
11import re
13from pygments.lexer import bygroups, combined, default, do_insertions, include, \
14 inherit, Lexer, RegexLexer, this, using, words, line_re
15from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
16 Number, Punctuation, Other, Generic, Whitespace
17from pygments.util import get_bool_opt
18import pygments.unistring as uni
20__all__ = ['JavascriptLexer', 'KalLexer', 'LiveScriptLexer', 'DartLexer',
21 'TypeScriptLexer', 'LassoLexer', 'ObjectiveJLexer',
22 'CoffeeScriptLexer', 'MaskLexer', 'EarlGreyLexer', 'JuttleLexer',
23 'NodeConsoleLexer']
25JS_IDENT_START = ('(?:[$_' + uni.combine('Lu', 'Ll', 'Lt', 'Lm', 'Lo', 'Nl') +
26 ']|\\\\u[a-fA-F0-9]{4})')
27JS_IDENT_PART = ('(?:[$' + uni.combine('Lu', 'Ll', 'Lt', 'Lm', 'Lo', 'Nl',
28 'Mn', 'Mc', 'Nd', 'Pc') +
29 '\u200c\u200d]|\\\\u[a-fA-F0-9]{4})')
30JS_IDENT = JS_IDENT_START + '(?:' + JS_IDENT_PART + ')*'
33class JavascriptLexer(RegexLexer):
34 """
35 For JavaScript source code.
36 """
38 name = 'JavaScript'
39 url = 'https://www.ecma-international.org/publications-and-standards/standards/ecma-262/'
40 aliases = ['javascript', 'js']
41 filenames = ['*.js', '*.jsm', '*.mjs', '*.cjs']
42 mimetypes = ['application/javascript', 'application/x-javascript',
43 'text/x-javascript', 'text/javascript']
45 flags = re.DOTALL | re.MULTILINE
47 tokens = {
48 'commentsandwhitespace': [
49 (r'\s+', Whitespace),
50 (r'<!--', Comment),
51 (r'//.*?$', Comment.Single),
52 (r'/\*.*?\*/', Comment.Multiline)
53 ],
54 'slashstartsregex': [
55 include('commentsandwhitespace'),
56 (r'/(\\.|[^[/\\\n]|\[(\\.|[^\]\\\n])*])+/'
57 r'([gimuysd]+\b|\B)', String.Regex, '#pop'),
58 (r'(?=/)', Text, ('#pop', 'badregex')),
59 default('#pop')
60 ],
61 'badregex': [
62 (r'\n', Whitespace, '#pop')
63 ],
64 'root': [
65 (r'\A#! ?/.*?$', Comment.Hashbang), # recognized by node.js
66 (r'^(?=\s|/|<!--)', Text, 'slashstartsregex'),
67 include('commentsandwhitespace'),
69 # Numeric literals
70 (r'0[bB][01]+n?', Number.Bin),
71 (r'0[oO]?[0-7]+n?', Number.Oct), # Browsers support "0o7" and "07" (< ES5) notations
72 (r'0[xX][0-9a-fA-F]+n?', Number.Hex),
73 (r'[0-9]+n', Number.Integer), # Javascript BigInt requires an "n" postfix
74 # Javascript doesn't have actual integer literals, so every other
75 # numeric literal is handled by the regex below (including "normal")
76 # integers
77 (r'(\.[0-9]+|[0-9]+\.[0-9]*|[0-9]+)([eE][-+]?[0-9]+)?', Number.Float),
79 (r'\.\.\.|=>', Punctuation),
80 (r'\+\+|--|~|\?\?=?|\?|:|\\(?=\n)|'
81 r'(<<|>>>?|==?|!=?|(?:\*\*|\|\||&&|[-<>+*%&|^/]))=?', Operator, 'slashstartsregex'),
82 (r'[{(\[;,]', Punctuation, 'slashstartsregex'),
83 (r'[})\].]', Punctuation),
85 (r'(typeof|instanceof|in|void|delete|new)\b', Operator.Word, 'slashstartsregex'),
87 # Match stuff like: constructor
88 (r'\b(constructor|from|as)\b', Keyword.Reserved),
90 (r'(for|in|while|do|break|return|continue|switch|case|default|if|else|'
91 r'throw|try|catch|finally|yield|await|async|this|of|static|export|'
92 r'import|debugger|extends|super)\b', Keyword, 'slashstartsregex'),
93 (r'(var|let|const|with|function|class)\b', Keyword.Declaration, 'slashstartsregex'),
95 (r'(abstract|boolean|byte|char|double|enum|final|float|goto|'
96 r'implements|int|interface|long|native|package|private|protected|'
97 r'public|short|synchronized|throws|transient|volatile)\b', Keyword.Reserved),
98 (r'(true|false|null|NaN|Infinity|undefined)\b', Keyword.Constant),
100 (r'(Array|Boolean|Date|BigInt|Function|Math|ArrayBuffer|'
101 r'Number|Object|RegExp|String|Promise|Proxy|decodeURI|'
102 r'decodeURIComponent|encodeURI|encodeURIComponent|'
103 r'eval|isFinite|isNaN|parseFloat|parseInt|DataView|'
104 r'document|window|globalThis|global|Symbol|Intl|'
105 r'WeakSet|WeakMap|Set|Map|Reflect|JSON|Atomics|'
106 r'Int(?:8|16|32)Array|BigInt64Array|Float32Array|Float64Array|'
107 r'Uint8ClampedArray|Uint(?:8|16|32)Array|BigUint64Array)\b', Name.Builtin),
109 (r'((?:Eval|Internal|Range|Reference|Syntax|Type|URI)?Error)\b', Name.Exception),
111 # Match stuff like: super(argument, list)
112 (r'(super)(\s*)(\([\w,?.$\s]+\s*\))',
113 bygroups(Keyword, Whitespace), 'slashstartsregex'),
114 # Match stuff like: function() {...}
115 (r'([a-zA-Z_?.$][\w?.$]*)(?=\(\) \{)', Name.Other, 'slashstartsregex'),
117 (JS_IDENT, Name.Other),
118 (r'"(\\\\|\\[^\\]|[^"\\])*"', String.Double),
119 (r"'(\\\\|\\[^\\]|[^'\\])*'", String.Single),
120 (r'`', String.Backtick, 'interp'),
121 ],
122 'interp': [
123 (r'`', String.Backtick, '#pop'),
124 (r'\\.', String.Backtick),
125 (r'\$\{', String.Interpol, 'interp-inside'),
126 (r'\$', String.Backtick),
127 (r'[^`\\$]+', String.Backtick),
128 ],
129 'interp-inside': [
130 # TODO: should this include single-line comments and allow nesting strings?
131 (r'\}', String.Interpol, '#pop'),
132 include('root'),
133 ],
134 }
137class TypeScriptLexer(JavascriptLexer):
138 """
139 For TypeScript source code.
141 .. versionadded:: 1.6
142 """
144 name = 'TypeScript'
145 url = 'https://www.typescriptlang.org/'
146 aliases = ['typescript', 'ts']
147 filenames = ['*.ts']
148 mimetypes = ['application/x-typescript', 'text/x-typescript']
150 # Higher priority than the TypoScriptLexer, as TypeScript is far more
151 # common these days
152 priority = 0.5
154 tokens = {
155 'root': [
156 (r'(abstract|implements|private|protected|public|readonly)\b',
157 Keyword, 'slashstartsregex'),
158 (r'(enum|interface|override)\b', Keyword.Declaration, 'slashstartsregex'),
159 (r'\b(declare|type)\b', Keyword.Reserved),
160 # Match variable type keywords
161 (r'\b(string|boolean|number)\b', Keyword.Type),
162 # Match stuff like: module name {...}
163 (r'\b(module)(\s*)([\w?.$]+)(\s*)',
164 bygroups(Keyword.Reserved, Whitespace, Name.Other, Whitespace), 'slashstartsregex'),
165 # Match stuff like: (function: return type)
166 (r'([\w?.$]+)(\s*)(:)(\s*)([\w?.$]+)',
167 bygroups(Name.Other, Whitespace, Operator, Whitespace, Keyword.Type)),
168 # Match stuff like: Decorators
169 (r'@' + JS_IDENT, Keyword.Declaration),
170 inherit,
171 ],
172 }
175class KalLexer(RegexLexer):
176 """
177 For Kal source code.
179 .. versionadded:: 2.0
180 """
182 name = 'Kal'
183 url = 'http://rzimmerman.github.io/kal'
184 aliases = ['kal']
185 filenames = ['*.kal']
186 mimetypes = ['text/kal', 'application/kal']
188 flags = re.DOTALL
189 tokens = {
190 'commentsandwhitespace': [
191 (r'\s+', Whitespace),
192 (r'###[^#].*?###', Comment.Multiline),
193 (r'(#(?!##[^#]).*?)(\n)', bygroups(Comment.Single, Whitespace)),
194 ],
195 'functiondef': [
196 (r'([$a-zA-Z_][\w$]*)(\s*)', bygroups(Name.Function, Whitespace),
197 '#pop'),
198 include('commentsandwhitespace'),
199 ],
200 'classdef': [
201 (r'\b(inherits)(\s+)(from)\b',
202 bygroups(Keyword, Whitespace, Keyword)),
203 (r'([$a-zA-Z_][\w$]*)(?=\s*\n)', Name.Class, '#pop'),
204 (r'[$a-zA-Z_][\w$]*\b', Name.Class),
205 include('commentsandwhitespace'),
206 ],
207 'listcomprehension': [
208 (r'\]', Punctuation, '#pop'),
209 (r'\b(property|value)\b', Keyword),
210 include('root'),
211 ],
212 'waitfor': [
213 (r'\n', Whitespace, '#pop'),
214 (r'\bfrom\b', Keyword),
215 include('root'),
216 ],
217 'root': [
218 include('commentsandwhitespace'),
219 (r'/(?! )(\\.|[^[/\\\n]|\[(\\.|[^\]\\\n])*])+/'
220 r'([gimuysd]+\b|\B)', String.Regex),
221 (r'\?|:|_(?=\n)|==?|!=|-(?!>)|[<>+*/-]=?',
222 Operator),
223 (r'\b(and|or|isnt|is|not|but|bitwise|mod|\^|xor|exists|'
224 r'doesnt\s+exist)\b', Operator.Word),
225 (r'(\([^()]+\))?(\s*)(>)',
226 bygroups(Name.Function, Whitespace, Punctuation)),
227 (r'[{(]', Punctuation),
228 (r'\[', Punctuation, 'listcomprehension'),
229 (r'[})\].,]', Punctuation),
230 (r'\b(function|method|task)\b', Keyword.Declaration, 'functiondef'),
231 (r'\bclass\b', Keyword.Declaration, 'classdef'),
232 (r'\b(safe(?=\s))?(\s*)(wait(?=\s))(\s+)(for)\b',
233 bygroups(Keyword, Whitespace, Keyword, Whitespace,
234 Keyword), 'waitfor'),
235 (r'\b(me|this)(\.[$a-zA-Z_][\w.$]*)?\b', Name.Variable.Instance),
236 (r'(?<![.$])(run)(\s+)(in)(\s+)(parallel)\b',
237 bygroups(Keyword, Whitespace, Keyword, Whitespace, Keyword)),
238 (r'(?<![.$])(for)(\s+)(parallel|series)?\b',
239 bygroups(Keyword, Whitespace, Keyword)),
240 (r'(?<![.$])(except)(\s+)(when)?\b',
241 bygroups(Keyword, Whitespace, Keyword)),
242 (r'(?<![.$])(fail)(\s+)(with)?\b',
243 bygroups(Keyword, Whitespace, Keyword)),
244 (r'(?<![.$])(inherits)(\s+)(from)?\b',
245 bygroups(Keyword, Whitespace, Keyword)),
246 (r'(?<![.$])(for)(\s+)(parallel|series)?\b',
247 bygroups(Keyword, Whitespace, Keyword)),
248 (words((
249 'in', 'of', 'while', 'until', 'break', 'return', 'continue',
250 'when', 'if', 'unless', 'else', 'otherwise', 'throw', 'raise',
251 'try', 'catch', 'finally', 'new', 'delete', 'typeof',
252 'instanceof', 'super'), prefix=r'(?<![.$])', suffix=r'\b'),
253 Keyword),
254 (words((
255 'true', 'false', 'yes', 'no', 'on', 'off', 'null', 'nothing',
256 'none', 'NaN', 'Infinity', 'undefined'), prefix=r'(?<![.$])',
257 suffix=r'\b'), Keyword.Constant),
258 (words((
259 'Array', 'Boolean', 'Date', 'Error', 'Function', 'Math',
260 'Number', 'Object', 'RegExp', 'String', 'decodeURI',
261 'decodeURIComponent', 'encodeURI', 'encodeURIComponent', 'eval',
262 'isFinite', 'isNaN', 'isSafeInteger', 'parseFloat', 'parseInt',
263 'document', 'window', 'globalThis', 'Symbol', 'print'),
264 suffix=r'\b'), Name.Builtin),
265 (r'([$a-zA-Z_][\w.$]*)(\s*)(:|[+\-*/]?\=)?\b',
266 bygroups(Name.Variable, Whitespace, Operator)),
267 (r'[0-9][0-9]*\.[0-9]+([eE][0-9]+)?[fd]?', Number.Float),
268 (r'0x[0-9a-fA-F]+', Number.Hex),
269 (r'[0-9]+', Number.Integer),
270 ('"""', String, 'tdqs'),
271 ("'''", String, 'tsqs'),
272 ('"', String, 'dqs'),
273 ("'", String, 'sqs'),
274 ],
275 'strings': [
276 (r'[^#\\\'"]+', String),
277 # note that all kal strings are multi-line.
278 # hashmarks, quotes and backslashes must be parsed one at a time
279 ],
280 'interpoling_string': [
281 (r'\}', String.Interpol, "#pop"),
282 include('root')
283 ],
284 'dqs': [
285 (r'"', String, '#pop'),
286 (r'\\.|\'', String), # double-quoted string don't need ' escapes
287 (r'#\{', String.Interpol, "interpoling_string"),
288 include('strings')
289 ],
290 'sqs': [
291 (r"'", String, '#pop'),
292 (r'#|\\.|"', String), # single quoted strings don't need " escapses
293 include('strings')
294 ],
295 'tdqs': [
296 (r'"""', String, '#pop'),
297 (r'\\.|\'|"', String), # no need to escape quotes in triple-string
298 (r'#\{', String.Interpol, "interpoling_string"),
299 include('strings'),
300 ],
301 'tsqs': [
302 (r"'''", String, '#pop'),
303 (r'#|\\.|\'|"', String), # no need to escape quotes in triple-strings
304 include('strings')
305 ],
306 }
309class LiveScriptLexer(RegexLexer):
310 """
311 For LiveScript source code.
313 .. versionadded:: 1.6
314 """
316 name = 'LiveScript'
317 url = 'https://livescript.net/'
318 aliases = ['livescript', 'live-script']
319 filenames = ['*.ls']
320 mimetypes = ['text/livescript']
322 flags = re.DOTALL
323 tokens = {
324 'commentsandwhitespace': [
325 (r'\s+', Whitespace),
326 (r'/\*.*?\*/', Comment.Multiline),
327 (r'(#.*?)(\n)', bygroups(Comment.Single, Whitespace)),
328 ],
329 'multilineregex': [
330 include('commentsandwhitespace'),
331 (r'//([gimuysd]+\b|\B)', String.Regex, '#pop'),
332 (r'/', String.Regex),
333 (r'[^/#]+', String.Regex)
334 ],
335 'slashstartsregex': [
336 include('commentsandwhitespace'),
337 (r'//', String.Regex, ('#pop', 'multilineregex')),
338 (r'/(?! )(\\.|[^[/\\\n]|\[(\\.|[^\]\\\n])*])+/'
339 r'([gimuysd]+\b|\B)', String.Regex, '#pop'),
340 (r'/', Operator, '#pop'),
341 default('#pop'),
342 ],
343 'root': [
344 (r'\A(?=\s|/)', Text, 'slashstartsregex'),
345 include('commentsandwhitespace'),
346 (r'(?:\([^()]+\))?[ ]*[~-]{1,2}>|'
347 r'(?:\(?[^()\n]+\)?)?[ ]*<[~-]{1,2}', Name.Function),
348 (r'\+\+|&&|(?<![.$])\b(?:and|x?or|is|isnt|not)\b|\?|:|=|'
349 r'\|\||\\(?=\n)|(<<|>>>?|==?|!=?|'
350 r'~(?!\~?>)|-(?!\-?>)|<(?!\[)|(?<!\])>|'
351 r'[+*`%&|^/])=?',
352 Operator, 'slashstartsregex'),
353 (r'[{(\[;,]', Punctuation, 'slashstartsregex'),
354 (r'[})\].]', Punctuation),
355 (r'(?<![.$])(for|own|in|of|while|until|loop|break|'
356 r'return|continue|switch|when|then|if|unless|else|'
357 r'throw|try|catch|finally|new|delete|typeof|instanceof|super|'
358 r'extends|this|class|by|const|var|to|til)\b', Keyword,
359 'slashstartsregex'),
360 (r'(?<![.$])(true|false|yes|no|on|off|'
361 r'null|NaN|Infinity|undefined|void)\b',
362 Keyword.Constant),
363 (r'(Array|Boolean|Date|Error|Function|Math|'
364 r'Number|Object|RegExp|String|decodeURI|'
365 r'decodeURIComponent|encodeURI|encodeURIComponent|'
366 r'eval|isFinite|isNaN|parseFloat|parseInt|document|window|'
367 r'globalThis|Symbol|Symbol|BigInt)\b', Name.Builtin),
368 (r'([$a-zA-Z_][\w.\-:$]*)(\s*)([:=])(\s+)',
369 bygroups(Name.Variable, Whitespace, Operator, Whitespace),
370 'slashstartsregex'),
371 (r'(@[$a-zA-Z_][\w.\-:$]*)(\s*)([:=])(\s+)',
372 bygroups(Name.Variable.Instance, Whitespace, Operator,
373 Whitespace),
374 'slashstartsregex'),
375 (r'@', Name.Other, 'slashstartsregex'),
376 (r'@?[$a-zA-Z_][\w-]*', Name.Other, 'slashstartsregex'),
377 (r'[0-9]+\.[0-9]+([eE][0-9]+)?[fd]?(?:[a-zA-Z_]+)?', Number.Float),
378 (r'[0-9]+(~[0-9a-z]+)?(?:[a-zA-Z_]+)?', Number.Integer),
379 ('"""', String, 'tdqs'),
380 ("'''", String, 'tsqs'),
381 ('"', String, 'dqs'),
382 ("'", String, 'sqs'),
383 (r'\\\S+', String),
384 (r'<\[.*?\]>', String),
385 ],
386 'strings': [
387 (r'[^#\\\'"]+', String),
388 # note that all coffee script strings are multi-line.
389 # hashmarks, quotes and backslashes must be parsed one at a time
390 ],
391 'interpoling_string': [
392 (r'\}', String.Interpol, "#pop"),
393 include('root')
394 ],
395 'dqs': [
396 (r'"', String, '#pop'),
397 (r'\\.|\'', String), # double-quoted string don't need ' escapes
398 (r'#\{', String.Interpol, "interpoling_string"),
399 (r'#', String),
400 include('strings')
401 ],
402 'sqs': [
403 (r"'", String, '#pop'),
404 (r'#|\\.|"', String), # single quoted strings don't need " escapses
405 include('strings')
406 ],
407 'tdqs': [
408 (r'"""', String, '#pop'),
409 (r'\\.|\'|"', String), # no need to escape quotes in triple-string
410 (r'#\{', String.Interpol, "interpoling_string"),
411 (r'#', String),
412 include('strings'),
413 ],
414 'tsqs': [
415 (r"'''", String, '#pop'),
416 (r'#|\\.|\'|"', String), # no need to escape quotes in triple-strings
417 include('strings')
418 ],
419 }
422class DartLexer(RegexLexer):
423 """
424 For Dart source code.
426 .. versionadded:: 1.5
427 """
429 name = 'Dart'
430 url = 'http://dart.dev/'
431 aliases = ['dart']
432 filenames = ['*.dart']
433 mimetypes = ['text/x-dart']
435 flags = re.MULTILINE | re.DOTALL
437 tokens = {
438 'root': [
439 include('string_literal'),
440 (r'#!(.*?)$', Comment.Preproc),
441 (r'\b(import|export)\b', Keyword, 'import_decl'),
442 (r'\b(library|source|part of|part)\b', Keyword),
443 (r'[^\S\n]+', Whitespace),
444 (r'(//.*?)(\n)', bygroups(Comment.Single, Whitespace)),
445 (r'/\*.*?\*/', Comment.Multiline),
446 (r'\b(class|extension|mixin)\b(\s+)',
447 bygroups(Keyword.Declaration, Whitespace), 'class'),
448 (r'\b(as|assert|break|case|catch|const|continue|default|do|else|finally|'
449 r'for|if|in|is|new|rethrow|return|super|switch|this|throw|try|while)\b',
450 Keyword),
451 (r'\b(abstract|async|await|const|covariant|extends|external|factory|final|'
452 r'get|implements|late|native|on|operator|required|set|static|sync|typedef|'
453 r'var|with|yield)\b', Keyword.Declaration),
454 (r'\b(bool|double|dynamic|int|num|Function|Never|Null|Object|String|void)\b',
455 Keyword.Type),
456 (r'\b(false|null|true)\b', Keyword.Constant),
457 (r'[~!%^&*+=|?:<>/-]|as\b', Operator),
458 (r'@[a-zA-Z_$]\w*', Name.Decorator),
459 (r'[a-zA-Z_$]\w*:', Name.Label),
460 (r'[a-zA-Z_$]\w*', Name),
461 (r'[(){}\[\],.;]', Punctuation),
462 (r'0[xX][0-9a-fA-F]+', Number.Hex),
463 # DIGIT+ (‘.’ DIGIT*)? EXPONENT?
464 (r'\d+(\.\d*)?([eE][+-]?\d+)?', Number),
465 (r'\.\d+([eE][+-]?\d+)?', Number), # ‘.’ DIGIT+ EXPONENT?
466 (r'\n', Whitespace)
467 # pseudo-keyword negate intentionally left out
468 ],
469 'class': [
470 (r'[a-zA-Z_$]\w*', Name.Class, '#pop')
471 ],
472 'import_decl': [
473 include('string_literal'),
474 (r'\s+', Whitespace),
475 (r'\b(as|deferred|show|hide)\b', Keyword),
476 (r'[a-zA-Z_$]\w*', Name),
477 (r'\,', Punctuation),
478 (r'\;', Punctuation, '#pop')
479 ],
480 'string_literal': [
481 # Raw strings.
482 (r'r"""([\w\W]*?)"""', String.Double),
483 (r"r'''([\w\W]*?)'''", String.Single),
484 (r'r"(.*?)"', String.Double),
485 (r"r'(.*?)'", String.Single),
486 # Normal Strings.
487 (r'"""', String.Double, 'string_double_multiline'),
488 (r"'''", String.Single, 'string_single_multiline'),
489 (r'"', String.Double, 'string_double'),
490 (r"'", String.Single, 'string_single')
491 ],
492 'string_common': [
493 (r"\\(x[0-9A-Fa-f]{2}|u[0-9A-Fa-f]{4}|u\{[0-9A-Fa-f]*\}|[a-z'\"$\\])",
494 String.Escape),
495 (r'(\$)([a-zA-Z_]\w*)', bygroups(String.Interpol, Name)),
496 (r'(\$\{)(.*?)(\})',
497 bygroups(String.Interpol, using(this), String.Interpol))
498 ],
499 'string_double': [
500 (r'"', String.Double, '#pop'),
501 (r'[^"$\\\n]+', String.Double),
502 include('string_common'),
503 (r'\$+', String.Double)
504 ],
505 'string_double_multiline': [
506 (r'"""', String.Double, '#pop'),
507 (r'[^"$\\]+', String.Double),
508 include('string_common'),
509 (r'(\$|\")+', String.Double)
510 ],
511 'string_single': [
512 (r"'", String.Single, '#pop'),
513 (r"[^'$\\\n]+", String.Single),
514 include('string_common'),
515 (r'\$+', String.Single)
516 ],
517 'string_single_multiline': [
518 (r"'''", String.Single, '#pop'),
519 (r'[^\'$\\]+', String.Single),
520 include('string_common'),
521 (r'(\$|\')+', String.Single)
522 ]
523 }
526class LassoLexer(RegexLexer):
527 """
528 For Lasso source code, covering both Lasso 9
529 syntax and LassoScript for Lasso 8.6 and earlier. For Lasso embedded in
530 HTML, use the `LassoHtmlLexer`.
532 Additional options accepted:
534 `builtinshighlighting`
535 If given and ``True``, highlight builtin types, traits, methods, and
536 members (default: ``True``).
537 `requiredelimiters`
538 If given and ``True``, only highlight code between delimiters as Lasso
539 (default: ``False``).
541 .. versionadded:: 1.6
542 """
544 name = 'Lasso'
545 aliases = ['lasso', 'lassoscript']
546 filenames = ['*.lasso', '*.lasso[89]']
547 alias_filenames = ['*.incl', '*.inc', '*.las']
548 mimetypes = ['text/x-lasso']
549 flags = re.IGNORECASE | re.DOTALL | re.MULTILINE
551 tokens = {
552 'root': [
553 (r'^#![ \S]+lasso9\b', Comment.Preproc, 'lasso'),
554 (r'(?=\[|<)', Other, 'delimiters'),
555 (r'\s+', Whitespace),
556 default(('delimiters', 'lassofile')),
557 ],
558 'delimiters': [
559 (r'\[no_square_brackets\]', Comment.Preproc, 'nosquarebrackets'),
560 (r'\[noprocess\]', Comment.Preproc, 'noprocess'),
561 (r'\[', Comment.Preproc, 'squarebrackets'),
562 (r'<\?(lasso(script)?|=)', Comment.Preproc, 'anglebrackets'),
563 (r'<(!--.*?-->)?', Other),
564 (r'[^[<]+', Other),
565 ],
566 'nosquarebrackets': [
567 (r'\[noprocess\]', Comment.Preproc, 'noprocess'),
568 (r'\[', Other),
569 (r'<\?(lasso(script)?|=)', Comment.Preproc, 'anglebrackets'),
570 (r'<(!--.*?-->)?', Other),
571 (r'[^[<]+', Other),
572 ],
573 'noprocess': [
574 (r'\[/noprocess\]', Comment.Preproc, '#pop'),
575 (r'\[', Other),
576 (r'[^[]', Other),
577 ],
578 'squarebrackets': [
579 (r'\]', Comment.Preproc, '#pop'),
580 include('lasso'),
581 ],
582 'anglebrackets': [
583 (r'\?>', Comment.Preproc, '#pop'),
584 include('lasso'),
585 ],
586 'lassofile': [
587 (r'\]|\?>', Comment.Preproc, '#pop'),
588 include('lasso'),
589 ],
590 'whitespacecomments': [
591 (r'\s+', Whitespace),
592 (r'(//.*?)(\s*)$', bygroups(Comment.Single, Whitespace)),
593 (r'/\*\*!.*?\*/', String.Doc),
594 (r'/\*.*?\*/', Comment.Multiline),
595 ],
596 'lasso': [
597 # whitespace/comments
598 include('whitespacecomments'),
600 # literals
601 (r'\d*\.\d+(e[+-]?\d+)?', Number.Float),
602 (r'0x[\da-f]+', Number.Hex),
603 (r'\d+', Number.Integer),
604 (r'(infinity|NaN)\b', Number),
605 (r"'", String.Single, 'singlestring'),
606 (r'"', String.Double, 'doublestring'),
607 (r'`[^`]*`', String.Backtick),
609 # names
610 (r'\$[a-z_][\w.]*', Name.Variable),
611 (r'#([a-z_][\w.]*|\d+\b)', Name.Variable.Instance),
612 (r"(\.)(\s*)('[a-z_][\w.]*')",
613 bygroups(Name.Builtin.Pseudo, Whitespace, Name.Variable.Class)),
614 (r"(self)(\s*)(->)(\s*)('[a-z_][\w.]*')",
615 bygroups(Name.Builtin.Pseudo, Whitespace, Operator, Whitespace,
616 Name.Variable.Class)),
617 (r'(\.\.?)(\s*)([a-z_][\w.]*(=(?!=))?)',
618 bygroups(Name.Builtin.Pseudo, Whitespace, Name.Other.Member)),
619 (r'(->\\?|&)(\s*)([a-z_][\w.]*(=(?!=))?)',
620 bygroups(Operator, Whitespace, Name.Other.Member)),
621 (r'(?<!->)(self|inherited|currentcapture|givenblock)\b',
622 Name.Builtin.Pseudo),
623 (r'-(?!infinity)[a-z_][\w.]*', Name.Attribute),
624 (r'(::)(\s*)([a-z_][\w.]*)',
625 bygroups(Punctuation, Whitespace, Name.Label)),
626 (r'(error_(code|msg)_\w+|Error_AddError|Error_ColumnRestriction|'
627 r'Error_DatabaseConnectionUnavailable|Error_DatabaseTimeout|'
628 r'Error_DeleteError|Error_FieldRestriction|Error_FileNotFound|'
629 r'Error_InvalidDatabase|Error_InvalidPassword|'
630 r'Error_InvalidUsername|Error_ModuleNotFound|'
631 r'Error_NoError|Error_NoPermission|Error_OutOfMemory|'
632 r'Error_ReqColumnMissing|Error_ReqFieldMissing|'
633 r'Error_RequiredColumnMissing|Error_RequiredFieldMissing|'
634 r'Error_UpdateError)\b', Name.Exception),
636 # definitions
637 (r'(define)(\s+)([a-z_][\w.]*)(\s*)(=>)(\s*)(type|trait|thread)\b',
638 bygroups(Keyword.Declaration, Whitespace, Name.Class,
639 Whitespace, Operator, Whitespace, Keyword)),
640 (r'(define)(\s+)([a-z_][\w.]*)(\s*)(->)(\s*)([a-z_][\w.]*=?|[-+*/%])',
641 bygroups(Keyword.Declaration, Whitespace, Name.Class,
642 Whitespace, Operator, Whitespace, Name.Function),
643 'signature'),
644 (r'(define)(\s+)([a-z_][\w.]*)',
645 bygroups(Keyword.Declaration, Whitespace, Name.Function), 'signature'),
646 (r'(public|protected|private|provide)(\s+)(([a-z_][\w.]*=?|[-+*/%])'
647 r'(?=\s*\())', bygroups(Keyword, Whitespace, Name.Function),
648 'signature'),
649 (r'(public|protected|private|provide)(\s+)([a-z_][\w.]*)',
650 bygroups(Keyword, Whitespace, Name.Function)),
652 # keywords
653 (r'(true|false|none|minimal|full|all|void)\b', Keyword.Constant),
654 (r'(local|var|variable|global|data(?=\s))\b', Keyword.Declaration),
655 (r'(array|date|decimal|duration|integer|map|pair|string|tag|xml|'
656 r'null|boolean|bytes|keyword|list|locale|queue|set|stack|'
657 r'staticarray)\b', Keyword.Type),
658 (r'([a-z_][\w.]*)(\s+)(in)\b', bygroups(Name, Whitespace, Keyword)),
659 (r'(let|into)(\s+)([a-z_][\w.]*)', bygroups(Keyword, Whitespace, Name)),
660 (r'require\b', Keyword, 'requiresection'),
661 (r'(/?)(Namespace_Using)\b', bygroups(Punctuation, Keyword.Namespace)),
662 (r'(/?)(Cache|Database_Names|Database_SchemaNames|'
663 r'Database_TableNames|Define_Tag|Define_Type|Email_Batch|'
664 r'Encode_Set|HTML_Comment|Handle|Handle_Error|Header|If|Inline|'
665 r'Iterate|LJAX_Target|Link|Link_CurrentAction|Link_CurrentGroup|'
666 r'Link_CurrentRecord|Link_Detail|Link_FirstGroup|Link_FirstRecord|'
667 r'Link_LastGroup|Link_LastRecord|Link_NextGroup|Link_NextRecord|'
668 r'Link_PrevGroup|Link_PrevRecord|Log|Loop|Output_None|Portal|'
669 r'Private|Protect|Records|Referer|Referrer|Repeating|ResultSet|'
670 r'Rows|Search_Args|Search_Arguments|Select|Sort_Args|'
671 r'Sort_Arguments|Thread_Atomic|Value_List|While|Abort|Case|Else|'
672 r'Fail_If|Fail_IfNot|Fail|If_Empty|If_False|If_Null|If_True|'
673 r'Loop_Abort|Loop_Continue|Loop_Count|Params|Params_Up|Return|'
674 r'Return_Value|Run_Children|SOAP_DefineTag|SOAP_LastRequest|'
675 r'SOAP_LastResponse|Tag_Name|ascending|average|by|define|'
676 r'descending|do|equals|frozen|group|handle_failure|import|in|into|'
677 r'join|let|match|max|min|on|order|parent|protected|provide|public|'
678 r'require|returnhome|skip|split_thread|sum|take|thread|to|trait|'
679 r'type|where|with|yield|yieldhome)\b',
680 bygroups(Punctuation, Keyword)),
682 # other
683 (r',', Punctuation, 'commamember'),
684 (r'(and|or|not)\b', Operator.Word),
685 (r'([a-z_][\w.]*)(\s*)(::)(\s*)([a-z_][\w.]*)?(\s*=(?!=))',
686 bygroups(Name, Whitespace, Punctuation, Whitespace, Name.Label,
687 Operator)),
688 (r'(/?)([\w.]+)', bygroups(Punctuation, Name.Other)),
689 (r'(=)(n?bw|n?ew|n?cn|lte?|gte?|n?eq|n?rx|ft)\b',
690 bygroups(Operator, Operator.Word)),
691 (r':=|[-+*/%=<>&|!?\\]+', Operator),
692 (r'[{}():;,@^]', Punctuation),
693 ],
694 'singlestring': [
695 (r"'", String.Single, '#pop'),
696 (r"[^'\\]+", String.Single),
697 include('escape'),
698 (r"\\", String.Single),
699 ],
700 'doublestring': [
701 (r'"', String.Double, '#pop'),
702 (r'[^"\\]+', String.Double),
703 include('escape'),
704 (r'\\', String.Double),
705 ],
706 'escape': [
707 (r'\\(U[\da-f]{8}|u[\da-f]{4}|x[\da-f]{1,2}|[0-7]{1,3}|:[^:\n\r]+:|'
708 r'[abefnrtv?"\'\\]|$)', String.Escape),
709 ],
710 'signature': [
711 (r'=>', Operator, '#pop'),
712 (r'\)', Punctuation, '#pop'),
713 (r'[(,]', Punctuation, 'parameter'),
714 include('lasso'),
715 ],
716 'parameter': [
717 (r'\)', Punctuation, '#pop'),
718 (r'-?[a-z_][\w.]*', Name.Attribute, '#pop'),
719 (r'\.\.\.', Name.Builtin.Pseudo),
720 include('lasso'),
721 ],
722 'requiresection': [
723 (r'(([a-z_][\w.]*=?|[-+*/%])(?=\s*\())', Name, 'requiresignature'),
724 (r'(([a-z_][\w.]*=?|[-+*/%])(?=(\s*::\s*[\w.]+)?\s*,))', Name),
725 (r'[a-z_][\w.]*=?|[-+*/%]', Name, '#pop'),
726 (r'(::)(\s*)([a-z_][\w.]*)',
727 bygroups(Punctuation, Whitespace, Name.Label)),
728 (r',', Punctuation),
729 include('whitespacecomments'),
730 ],
731 'requiresignature': [
732 (r'(\)(?=(\s*::\s*[\w.]+)?\s*,))', Punctuation, '#pop'),
733 (r'\)', Punctuation, '#pop:2'),
734 (r'-?[a-z_][\w.]*', Name.Attribute),
735 (r'(::)(\s*)([a-z_][\w.]*)',
736 bygroups(Punctuation, Whitespace, Name.Label)),
737 (r'\.\.\.', Name.Builtin.Pseudo),
738 (r'[(,]', Punctuation),
739 include('whitespacecomments'),
740 ],
741 'commamember': [
742 (r'(([a-z_][\w.]*=?|[-+*/%])'
743 r'(?=\s*(\(([^()]*\([^()]*\))*[^)]*\)\s*)?(::[\w.\s]+)?=>))',
744 Name.Function, 'signature'),
745 include('whitespacecomments'),
746 default('#pop'),
747 ],
748 }
750 def __init__(self, **options):
751 self.builtinshighlighting = get_bool_opt(
752 options, 'builtinshighlighting', True)
753 self.requiredelimiters = get_bool_opt(
754 options, 'requiredelimiters', False)
756 self._builtins = set()
757 self._members = set()
758 if self.builtinshighlighting:
759 from pygments.lexers._lasso_builtins import BUILTINS, MEMBERS
760 for key, value in BUILTINS.items():
761 self._builtins.update(value)
762 for key, value in MEMBERS.items():
763 self._members.update(value)
764 RegexLexer.__init__(self, **options)
766 def get_tokens_unprocessed(self, text):
767 stack = ['root']
768 if self.requiredelimiters:
769 stack.append('delimiters')
770 for index, token, value in \
771 RegexLexer.get_tokens_unprocessed(self, text, stack):
772 if (token is Name.Other and value.lower() in self._builtins or
773 token is Name.Other.Member and
774 value.lower().rstrip('=') in self._members):
775 yield index, Name.Builtin, value
776 continue
777 yield index, token, value
779 def analyse_text(text):
780 rv = 0.0
781 if 'bin/lasso9' in text:
782 rv += 0.8
783 if re.search(r'<\?lasso', text, re.I):
784 rv += 0.4
785 if re.search(r'local\(', text, re.I):
786 rv += 0.4
787 return rv
790class ObjectiveJLexer(RegexLexer):
791 """
792 For Objective-J source code with preprocessor directives.
794 .. versionadded:: 1.3
795 """
797 name = 'Objective-J'
798 aliases = ['objective-j', 'objectivej', 'obj-j', 'objj']
799 filenames = ['*.j']
800 mimetypes = ['text/x-objective-j']
802 #: optional Comment or Whitespace
803 _ws = r'(?:\s|//[^\n]*\n|/[*](?:[^*]|[*][^/])*[*]/)*'
805 flags = re.DOTALL | re.MULTILINE
807 tokens = {
808 'root': [
809 include('whitespace'),
811 # function definition
812 (r'^(' + _ws + r'[+-]' + _ws + r')([(a-zA-Z_].*?[^(])(' + _ws + r'\{)',
813 bygroups(using(this), using(this, state='function_signature'),
814 using(this))),
816 # class definition
817 (r'(@interface|@implementation)(\s+)', bygroups(Keyword, Whitespace),
818 'classname'),
819 (r'(@class|@protocol)(\s*)', bygroups(Keyword, Whitespace),
820 'forward_classname'),
821 (r'(\s*)(@end)(\s*)', bygroups(Whitespace, Keyword, Whitespace)),
823 include('statements'),
824 ('[{()}]', Punctuation),
825 (';', Punctuation),
826 ],
827 'whitespace': [
828 (r'(@import)(\s+)("(?:\\\\|\\"|[^"])*")',
829 bygroups(Comment.Preproc, Whitespace, String.Double)),
830 (r'(@import)(\s+)(<(?:\\\\|\\>|[^>])*>)',
831 bygroups(Comment.Preproc, Whitespace, String.Double)),
832 (r'(#(?:include|import))(\s+)("(?:\\\\|\\"|[^"])*")',
833 bygroups(Comment.Preproc, Whitespace, String.Double)),
834 (r'(#(?:include|import))(\s+)(<(?:\\\\|\\>|[^>])*>)',
835 bygroups(Comment.Preproc, Whitespace, String.Double)),
837 (r'#if\s+0', Comment.Preproc, 'if0'),
838 (r'#', Comment.Preproc, 'macro'),
840 (r'\s+', Whitespace),
841 (r'(\\)(\n)',
842 bygroups(String.Escape, Whitespace)), # line continuation
843 (r'//(\n|(.|\n)*?[^\\]\n)', Comment.Single),
844 (r'/(\\\n)?[*](.|\n)*?[*](\\\n)?/', Comment.Multiline),
845 (r'<!--', Comment),
846 ],
847 'slashstartsregex': [
848 include('whitespace'),
849 (r'/(\\.|[^[/\\\n]|\[(\\.|[^\]\\\n])*])+/'
850 r'([gim]+\b|\B)', String.Regex, '#pop'),
851 (r'(?=/)', Text, ('#pop', 'badregex')),
852 default('#pop'),
853 ],
854 'badregex': [
855 (r'\n', Whitespace, '#pop'),
856 ],
857 'statements': [
858 (r'(L|@)?"', String, 'string'),
859 (r"(L|@)?'(\\.|\\[0-7]{1,3}|\\x[a-fA-F0-9]{1,2}|[^\\\'\n])'",
860 String.Char),
861 (r'"(\\\\|\\[^\\]|[^"\\])*"', String.Double),
862 (r"'(\\\\|\\[^\\]|[^'\\])*'", String.Single),
863 (r'(\d+\.\d*|\.\d+|\d+)[eE][+-]?\d+[lL]?', Number.Float),
864 (r'(\d+\.\d*|\.\d+|\d+[fF])[fF]?', Number.Float),
865 (r'0x[0-9a-fA-F]+[Ll]?', Number.Hex),
866 (r'0[0-7]+[Ll]?', Number.Oct),
867 (r'\d+[Ll]?', Number.Integer),
869 (r'^(?=\s|/|<!--)', Text, 'slashstartsregex'),
871 (r'\+\+|--|~|&&|\?|:|\|\||\\(?=\n)|'
872 r'(<<|>>>?|==?|!=?|[-<>+*%&|^/])=?',
873 Operator, 'slashstartsregex'),
874 (r'[{(\[;,]', Punctuation, 'slashstartsregex'),
875 (r'[})\].]', Punctuation),
877 (r'(for|in|while|do|break|return|continue|switch|case|default|if|'
878 r'else|throw|try|catch|finally|new|delete|typeof|instanceof|void|'
879 r'prototype|__proto__)\b', Keyword, 'slashstartsregex'),
881 (r'(var|with|function)\b', Keyword.Declaration, 'slashstartsregex'),
883 (r'(@selector|@private|@protected|@public|@encode|'
884 r'@synchronized|@try|@throw|@catch|@finally|@end|@property|'
885 r'@synthesize|@dynamic|@for|@accessors|new)\b', Keyword),
887 (r'(int|long|float|short|double|char|unsigned|signed|void|'
888 r'id|BOOL|bool|boolean|IBOutlet|IBAction|SEL|@outlet|@action)\b',
889 Keyword.Type),
891 (r'(self|super)\b', Name.Builtin),
893 (r'(TRUE|YES|FALSE|NO|Nil|nil|NULL)\b', Keyword.Constant),
894 (r'(true|false|null|NaN|Infinity|undefined)\b', Keyword.Constant),
895 (r'(ABS|ASIN|ACOS|ATAN|ATAN2|SIN|COS|TAN|EXP|POW|CEIL|FLOOR|ROUND|'
896 r'MIN|MAX|RAND|SQRT|E|LN2|LN10|LOG2E|LOG10E|PI|PI2|PI_2|SQRT1_2|'
897 r'SQRT2)\b', Keyword.Constant),
899 (r'(Array|Boolean|Date|Error|Function|Math|'
900 r'Number|Object|RegExp|String|decodeURI|'
901 r'decodeURIComponent|encodeURI|encodeURIComponent|'
902 r'Error|eval|isFinite|isNaN|parseFloat|parseInt|document|this|'
903 r'window|globalThis|Symbol)\b', Name.Builtin),
905 (r'([$a-zA-Z_]\w*)(' + _ws + r')(?=\()',
906 bygroups(Name.Function, using(this))),
908 (r'[$a-zA-Z_]\w*', Name),
909 ],
910 'classname': [
911 # interface definition that inherits
912 (r'([a-zA-Z_]\w*)(' + _ws + r':' + _ws +
913 r')([a-zA-Z_]\w*)?',
914 bygroups(Name.Class, using(this), Name.Class), '#pop'),
915 # interface definition for a category
916 (r'([a-zA-Z_]\w*)(' + _ws + r'\()([a-zA-Z_]\w*)(\))',
917 bygroups(Name.Class, using(this), Name.Label, Text), '#pop'),
918 # simple interface / implementation
919 (r'([a-zA-Z_]\w*)', Name.Class, '#pop'),
920 ],
921 'forward_classname': [
922 (r'([a-zA-Z_]\w*)(\s*)(,)(\s*)',
923 bygroups(Name.Class, Whitespace, Text, Whitespace), '#push'),
924 (r'([a-zA-Z_]\w*)(\s*)(;?)',
925 bygroups(Name.Class, Whitespace, Text), '#pop'),
926 ],
927 'function_signature': [
928 include('whitespace'),
930 # start of a selector w/ parameters
931 (r'(\(' + _ws + r')' # open paren
932 r'([a-zA-Z_]\w+)' # return type
933 r'(' + _ws + r'\)' + _ws + r')' # close paren
934 r'([$a-zA-Z_]\w+' + _ws + r':)', # function name
935 bygroups(using(this), Keyword.Type, using(this),
936 Name.Function), 'function_parameters'),
938 # no-param function
939 (r'(\(' + _ws + r')' # open paren
940 r'([a-zA-Z_]\w+)' # return type
941 r'(' + _ws + r'\)' + _ws + r')' # close paren
942 r'([$a-zA-Z_]\w+)', # function name
943 bygroups(using(this), Keyword.Type, using(this),
944 Name.Function), "#pop"),
946 # no return type given, start of a selector w/ parameters
947 (r'([$a-zA-Z_]\w+' + _ws + r':)', # function name
948 bygroups(Name.Function), 'function_parameters'),
950 # no return type given, no-param function
951 (r'([$a-zA-Z_]\w+)', # function name
952 bygroups(Name.Function), "#pop"),
954 default('#pop'),
955 ],
956 'function_parameters': [
957 include('whitespace'),
959 # parameters
960 (r'(\(' + _ws + ')' # open paren
961 r'([^)]+)' # type
962 r'(' + _ws + r'\)' + _ws + r')' # close paren
963 r'([$a-zA-Z_]\w+)', # param name
964 bygroups(using(this), Keyword.Type, using(this), Text)),
966 # one piece of a selector name
967 (r'([$a-zA-Z_]\w+' + _ws + r':)', # function name
968 Name.Function),
970 # smallest possible selector piece
971 (r'(:)', Name.Function),
973 # var args
974 (r'(,' + _ws + r'\.\.\.)', using(this)),
976 # param name
977 (r'([$a-zA-Z_]\w+)', Text),
978 ],
979 'expression': [
980 (r'([$a-zA-Z_]\w*)(\()', bygroups(Name.Function,
981 Punctuation)),
982 (r'(\))', Punctuation, "#pop"),
983 ],
984 'string': [
985 (r'"', String, '#pop'),
986 (r'\\([\\abfnrtv"\']|x[a-fA-F0-9]{2,4}|[0-7]{1,3})', String.Escape),
987 (r'[^\\"\n]+', String), # all other characters
988 (r'(\\)(\n)', bygroups(String.Escape, Whitespace)), # line continuation
989 (r'\\', String), # stray backslash
990 ],
991 'macro': [
992 (r'[^/\n]+', Comment.Preproc),
993 (r'/[*](.|\n)*?[*]/', Comment.Multiline),
994 (r'(//.*?)(\n)', bygroups(Comment.Single, Whitespace), '#pop'),
995 (r'/', Comment.Preproc),
996 (r'(?<=\\)\n', Whitespace),
997 (r'\n', Whitespace, '#pop'),
998 ],
999 'if0': [
1000 (r'^\s*#if.*?(?<!\\)\n', Comment.Preproc, '#push'),
1001 (r'^\s*#endif.*?(?<!\\)\n', Comment.Preproc, '#pop'),
1002 (r'(.*?)(\n)', bygroups(Comment, Whitespace)),
1003 ]
1004 }
1006 def analyse_text(text):
1007 if re.search(r'^\s*@import\s+[<"]', text, re.MULTILINE):
1008 # special directive found in most Objective-J files
1009 return True
1010 return False
1013class CoffeeScriptLexer(RegexLexer):
1014 """
1015 For CoffeeScript source code.
1017 .. versionadded:: 1.3
1018 """
1020 name = 'CoffeeScript'
1021 url = 'http://coffeescript.org'
1022 aliases = ['coffeescript', 'coffee-script', 'coffee']
1023 filenames = ['*.coffee']
1024 mimetypes = ['text/coffeescript']
1026 _operator_re = (
1027 r'\+\+|~|&&|\band\b|\bor\b|\bis\b|\bisnt\b|\bnot\b|\?|:|'
1028 r'\|\||\\(?=\n)|'
1029 r'(<<|>>>?|==?(?!>)|!=?|=(?!>)|-(?!>)|[<>+*`%&|\^/])=?')
1031 flags = re.DOTALL
1032 tokens = {
1033 'commentsandwhitespace': [
1034 (r'\s+', Whitespace),
1035 (r'###[^#].*?###', Comment.Multiline),
1036 (r'(#(?!##[^#]).*?)(\n)', bygroups(Comment.Single, Whitespace)),
1037 ],
1038 'multilineregex': [
1039 (r'[^/#]+', String.Regex),
1040 (r'///([gimuysd]+\b|\B)', String.Regex, '#pop'),
1041 (r'#\{', String.Interpol, 'interpoling_string'),
1042 (r'[/#]', String.Regex),
1043 ],
1044 'slashstartsregex': [
1045 include('commentsandwhitespace'),
1046 (r'///', String.Regex, ('#pop', 'multilineregex')),
1047 (r'/(?! )(\\.|[^[/\\\n]|\[(\\.|[^\]\\\n])*])+/'
1048 r'([gimuysd]+\b|\B)', String.Regex, '#pop'),
1049 # This isn't really guarding against mishighlighting well-formed
1050 # code, just the ability to infinite-loop between root and
1051 # slashstartsregex.
1052 (r'/', Operator, '#pop'),
1053 default('#pop'),
1054 ],
1055 'root': [
1056 include('commentsandwhitespace'),
1057 (r'\A(?=\s|/)', Text, 'slashstartsregex'),
1058 (_operator_re, Operator, 'slashstartsregex'),
1059 (r'(?:\([^()]*\))?\s*[=-]>', Name.Function, 'slashstartsregex'),
1060 (r'[{(\[;,]', Punctuation, 'slashstartsregex'),
1061 (r'[})\].]', Punctuation),
1062 (r'(?<![.$])(for|own|in|of|while|until|'
1063 r'loop|break|return|continue|'
1064 r'switch|when|then|if|unless|else|'
1065 r'throw|try|catch|finally|new|delete|typeof|instanceof|super|'
1066 r'extends|this|class|by)\b', Keyword, 'slashstartsregex'),
1067 (r'(?<![.$])(true|false|yes|no|on|off|null|'
1068 r'NaN|Infinity|undefined)\b',
1069 Keyword.Constant),
1070 (r'(Array|Boolean|Date|Error|Function|Math|'
1071 r'Number|Object|RegExp|String|decodeURI|'
1072 r'decodeURIComponent|encodeURI|encodeURIComponent|'
1073 r'eval|isFinite|isNaN|parseFloat|parseInt|document|window|globalThis|Symbol)\b',
1074 Name.Builtin),
1075 (r'([$a-zA-Z_][\w.:$]*)(\s*)([:=])(\s+)',
1076 bygroups(Name.Variable, Whitespace, Operator, Whitespace),
1077 'slashstartsregex'),
1078 (r'(@[$a-zA-Z_][\w.:$]*)(\s*)([:=])(\s+)',
1079 bygroups(Name.Variable.Instance, Whitespace, Operator, Whitespace),
1080 'slashstartsregex'),
1081 (r'@', Name.Other, 'slashstartsregex'),
1082 (r'@?[$a-zA-Z_][\w$]*', Name.Other),
1083 (r'[0-9][0-9]*\.[0-9]+([eE][0-9]+)?[fd]?', Number.Float),
1084 (r'0x[0-9a-fA-F]+', Number.Hex),
1085 (r'[0-9]+', Number.Integer),
1086 ('"""', String, 'tdqs'),
1087 ("'''", String, 'tsqs'),
1088 ('"', String, 'dqs'),
1089 ("'", String, 'sqs'),
1090 ],
1091 'strings': [
1092 (r'[^#\\\'"]+', String),
1093 # note that all coffee script strings are multi-line.
1094 # hashmarks, quotes and backslashes must be parsed one at a time
1095 ],
1096 'interpoling_string': [
1097 (r'\}', String.Interpol, "#pop"),
1098 include('root')
1099 ],
1100 'dqs': [
1101 (r'"', String, '#pop'),
1102 (r'\\.|\'', String), # double-quoted string don't need ' escapes
1103 (r'#\{', String.Interpol, "interpoling_string"),
1104 (r'#', String),
1105 include('strings')
1106 ],
1107 'sqs': [
1108 (r"'", String, '#pop'),
1109 (r'#|\\.|"', String), # single quoted strings don't need " escapses
1110 include('strings')
1111 ],
1112 'tdqs': [
1113 (r'"""', String, '#pop'),
1114 (r'\\.|\'|"', String), # no need to escape quotes in triple-string
1115 (r'#\{', String.Interpol, "interpoling_string"),
1116 (r'#', String),
1117 include('strings'),
1118 ],
1119 'tsqs': [
1120 (r"'''", String, '#pop'),
1121 (r'#|\\.|\'|"', String), # no need to escape quotes in triple-strings
1122 include('strings')
1123 ],
1124 }
1127class MaskLexer(RegexLexer):
1128 """
1129 For Mask markup.
1131 .. versionadded:: 2.0
1132 """
1133 name = 'Mask'
1134 url = 'https://github.com/atmajs/MaskJS'
1135 aliases = ['mask']
1136 filenames = ['*.mask']
1137 mimetypes = ['text/x-mask']
1139 flags = re.MULTILINE | re.IGNORECASE | re.DOTALL
1140 tokens = {
1141 'root': [
1142 (r'\s+', Whitespace),
1143 (r'(//.*?)(\n)', bygroups(Comment.Single, Whitespace)),
1144 (r'/\*.*?\*/', Comment.Multiline),
1145 (r'[{};>]', Punctuation),
1146 (r"'''", String, 'string-trpl-single'),
1147 (r'"""', String, 'string-trpl-double'),
1148 (r"'", String, 'string-single'),
1149 (r'"', String, 'string-double'),
1150 (r'([\w-]+)', Name.Tag, 'node'),
1151 (r'([^.#;{>\s]+)', Name.Class, 'node'),
1152 (r'(#[\w-]+)', Name.Function, 'node'),
1153 (r'(\.[\w-]+)', Name.Variable.Class, 'node')
1154 ],
1155 'string-base': [
1156 (r'\\.', String.Escape),
1157 (r'~\[', String.Interpol, 'interpolation'),
1158 (r'.', String.Single),
1159 ],
1160 'string-single': [
1161 (r"'", String.Single, '#pop'),
1162 include('string-base')
1163 ],
1164 'string-double': [
1165 (r'"', String.Single, '#pop'),
1166 include('string-base')
1167 ],
1168 'string-trpl-single': [
1169 (r"'''", String.Single, '#pop'),
1170 include('string-base')
1171 ],
1172 'string-trpl-double': [
1173 (r'"""', String.Single, '#pop'),
1174 include('string-base')
1175 ],
1176 'interpolation': [
1177 (r'\]', String.Interpol, '#pop'),
1178 (r'(\s*)(:)', bygroups(Whitespace, String.Interpol), 'expression'),
1179 (r'(\s*)(\w+)(:)', bygroups(Whitespace, Name.Other, Punctuation)),
1180 (r'[^\]]+', String.Interpol)
1181 ],
1182 'expression': [
1183 (r'[^\]]+', using(JavascriptLexer), '#pop')
1184 ],
1185 'node': [
1186 (r'\s+', Whitespace),
1187 (r'\.', Name.Variable.Class, 'node-class'),
1188 (r'\#', Name.Function, 'node-id'),
1189 (r'(style)([ \t]*)(=)',
1190 bygroups(Name.Attribute, Whitespace, Operator),
1191 'node-attr-style-value'),
1192 (r'([\w:-]+)([ \t]*)(=)',
1193 bygroups(Name.Attribute, Whitespace, Operator),
1194 'node-attr-value'),
1195 (r'[\w:-]+', Name.Attribute),
1196 (r'[>{;]', Punctuation, '#pop')
1197 ],
1198 'node-class': [
1199 (r'[\w-]+', Name.Variable.Class),
1200 (r'~\[', String.Interpol, 'interpolation'),
1201 default('#pop')
1202 ],
1203 'node-id': [
1204 (r'[\w-]+', Name.Function),
1205 (r'~\[', String.Interpol, 'interpolation'),
1206 default('#pop')
1207 ],
1208 'node-attr-value': [
1209 (r'\s+', Whitespace),
1210 (r'\w+', Name.Variable, '#pop'),
1211 (r"'", String, 'string-single-pop2'),
1212 (r'"', String, 'string-double-pop2'),
1213 default('#pop')
1214 ],
1215 'node-attr-style-value': [
1216 (r'\s+', Whitespace),
1217 (r"'", String.Single, 'css-single-end'),
1218 (r'"', String.Single, 'css-double-end'),
1219 include('node-attr-value')
1220 ],
1221 'css-base': [
1222 (r'\s+', Whitespace),
1223 (r";", Punctuation),
1224 (r"[\w\-]+\s*:", Name.Builtin)
1225 ],
1226 'css-single-end': [
1227 include('css-base'),
1228 (r"'", String.Single, '#pop:2'),
1229 (r"[^;']+", Name.Entity)
1230 ],
1231 'css-double-end': [
1232 include('css-base'),
1233 (r'"', String.Single, '#pop:2'),
1234 (r'[^;"]+', Name.Entity)
1235 ],
1236 'string-single-pop2': [
1237 (r"'", String.Single, '#pop:2'),
1238 include('string-base')
1239 ],
1240 'string-double-pop2': [
1241 (r'"', String.Single, '#pop:2'),
1242 include('string-base')
1243 ],
1244 }
1247class EarlGreyLexer(RegexLexer):
1248 """
1249 For Earl-Grey source code.
1251 .. versionadded: 2.1
1252 """
1254 name = 'Earl Grey'
1255 aliases = ['earl-grey', 'earlgrey', 'eg']
1256 filenames = ['*.eg']
1257 mimetypes = ['text/x-earl-grey']
1259 tokens = {
1260 'root': [
1261 (r'\n', Whitespace),
1262 include('control'),
1263 (r'[^\S\n]+', Text),
1264 (r'(;;.*)(\n)', bygroups(Comment, Whitespace)),
1265 (r'[\[\]{}:(),;]', Punctuation),
1266 (r'(\\)(\n)', bygroups(String.Escape, Whitespace)),
1267 (r'\\', Text),
1268 include('errors'),
1269 (words((
1270 'with', 'where', 'when', 'and', 'not', 'or', 'in',
1271 'as', 'of', 'is'),
1272 prefix=r'(?<=\s|\[)', suffix=r'(?![\w$\-])'),
1273 Operator.Word),
1274 (r'[*@]?->', Name.Function),
1275 (r'[+\-*/~^<>%&|?!@#.]*=', Operator.Word),
1276 (r'\.{2,3}', Operator.Word), # Range Operator
1277 (r'([+*/~^<>&|?!]+)|([#\-](?=\s))|@@+(?=\s)|=+', Operator),
1278 (r'(?<![\w$\-])(var|let)(?:[^\w$])', Keyword.Declaration),
1279 include('keywords'),
1280 include('builtins'),
1281 include('assignment'),
1282 (r'''(?x)
1283 (?:()([a-zA-Z$_](?:[\w$\-]*[\w$])?)|
1284 (?<=[\s{\[(])(\.)([a-zA-Z$_](?:[\w$\-]*[\w$])?))
1285 (?=.*%)''',
1286 bygroups(Punctuation, Name.Tag, Punctuation, Name.Class.Start), 'dbs'),
1287 (r'[rR]?`', String.Backtick, 'bt'),
1288 (r'[rR]?```', String.Backtick, 'tbt'),
1289 (r'(?<=[\s\[{(,;])\.([a-zA-Z$_](?:[\w$\-]*[\w$])?)'
1290 r'(?=[\s\]}),;])', String.Symbol),
1291 include('nested'),
1292 (r'(?:[rR]|[rR]\.[gmi]{1,3})?"', String, combined('stringescape', 'dqs')),
1293 (r'(?:[rR]|[rR]\.[gmi]{1,3})?\'', String, combined('stringescape', 'sqs')),
1294 (r'"""', String, combined('stringescape', 'tdqs')),
1295 include('tuple'),
1296 include('import_paths'),
1297 include('name'),
1298 include('numbers'),
1299 ],
1300 'dbs': [
1301 (r'(\.)([a-zA-Z$_](?:[\w$\-]*[\w$])?)(?=[.\[\s])',
1302 bygroups(Punctuation, Name.Class.DBS)),
1303 (r'(\[)([\^#][a-zA-Z$_](?:[\w$\-]*[\w$])?)(\])',
1304 bygroups(Punctuation, Name.Entity.DBS, Punctuation)),
1305 (r'\s+', Whitespace),
1306 (r'%', Operator.DBS, '#pop'),
1307 ],
1308 'import_paths': [
1309 (r'(?<=[\s:;,])(\.{1,3}(?:[\w\-]*/)*)(\w(?:[\w\-]*\w)*)(?=[\s;,])',
1310 bygroups(Text.Whitespace, Text)),
1311 ],
1312 'assignment': [
1313 (r'(\.)?([a-zA-Z$_](?:[\w$\-]*[\w$])?)'
1314 r'(?=\s+[+\-*/~^<>%&|?!@#.]*\=\s)',
1315 bygroups(Punctuation, Name.Variable))
1316 ],
1317 'errors': [
1318 (words(('Error', 'TypeError', 'ReferenceError'),
1319 prefix=r'(?<![\w\-$.])', suffix=r'(?![\w\-$.])'),
1320 Name.Exception),
1321 (r'''(?x)
1322 (?<![\w$])
1323 E\.[\w$](?:[\w$\-]*[\w$])?
1324 (?:\.[\w$](?:[\w$\-]*[\w$])?)*
1325 (?=[({\[?!\s])''',
1326 Name.Exception),
1327 ],
1328 'control': [
1329 (r'''(?x)
1330 ([a-zA-Z$_](?:[\w$-]*[\w$])?)
1331 (?!\n)\s+
1332 (?!and|as|each\*|each|in|is|mod|of|or|when|where|with)
1333 (?=(?:[+\-*/~^<>%&|?!@#.])?[a-zA-Z$_](?:[\w$-]*[\w$])?)''',
1334 Keyword.Control),
1335 (r'([a-zA-Z$_](?:[\w$-]*[\w$])?)(?!\n)(\s+)(?=[\'"\d{\[(])',
1336 bygroups(Keyword.Control, Whitespace)),
1337 (r'''(?x)
1338 (?:
1339 (?<=[%=])|
1340 (?<=[=\-]>)|
1341 (?<=with|each|with)|
1342 (?<=each\*|where)
1343 )(\s+)
1344 ([a-zA-Z$_](?:[\w$-]*[\w$])?)(:)''',
1345 bygroups(Whitespace, Keyword.Control, Punctuation)),
1346 (r'''(?x)
1347 (?<![+\-*/~^<>%&|?!@#.])(\s+)
1348 ([a-zA-Z$_](?:[\w$-]*[\w$])?)(:)''',
1349 bygroups(Whitespace, Keyword.Control, Punctuation)),
1350 ],
1351 'nested': [
1352 (r'''(?x)
1353 (?<=[\w$\]})])(\.)
1354 ([a-zA-Z$_](?:[\w$-]*[\w$])?)
1355 (?=\s+with(?:\s|\n))''',
1356 bygroups(Punctuation, Name.Function)),
1357 (r'''(?x)
1358 (?<!\s)(\.)
1359 ([a-zA-Z$_](?:[\w$-]*[\w$])?)
1360 (?=[}\]).,;:\s])''',
1361 bygroups(Punctuation, Name.Field)),
1362 (r'''(?x)
1363 (?<=[\w$\]})])(\.)
1364 ([a-zA-Z$_](?:[\w$-]*[\w$])?)
1365 (?=[\[{(:])''',
1366 bygroups(Punctuation, Name.Function)),
1367 ],
1368 'keywords': [
1369 (words((
1370 'each', 'each*', 'mod', 'await', 'break', 'chain',
1371 'continue', 'elif', 'expr-value', 'if', 'match',
1372 'return', 'yield', 'pass', 'else', 'require', 'var',
1373 'let', 'async', 'method', 'gen'),
1374 prefix=r'(?<![\w\-$.])', suffix=r'(?![\w\-$.])'),
1375 Keyword.Pseudo),
1376 (words(('this', 'self', '@'),
1377 prefix=r'(?<![\w\-$.])', suffix=r'(?![\w\-$])'),
1378 Keyword.Constant),
1379 (words((
1380 'Function', 'Object', 'Array', 'String', 'Number',
1381 'Boolean', 'ErrorFactory', 'ENode', 'Promise'),
1382 prefix=r'(?<![\w\-$.])', suffix=r'(?![\w\-$])'),
1383 Keyword.Type),
1384 ],
1385 'builtins': [
1386 (words((
1387 'send', 'object', 'keys', 'items', 'enumerate', 'zip',
1388 'product', 'neighbours', 'predicate', 'equal',
1389 'nequal', 'contains', 'repr', 'clone', 'range',
1390 'getChecker', 'get-checker', 'getProperty', 'get-property',
1391 'getProjector', 'get-projector', 'consume', 'take',
1392 'promisify', 'spawn', 'constructor'),
1393 prefix=r'(?<![\w\-#.])', suffix=r'(?![\w\-.])'),
1394 Name.Builtin),
1395 (words((
1396 'true', 'false', 'null', 'undefined'),
1397 prefix=r'(?<![\w\-$.])', suffix=r'(?![\w\-$.])'),
1398 Name.Constant),
1399 ],
1400 'name': [
1401 (r'@([a-zA-Z$_](?:[\w$-]*[\w$])?)', Name.Variable.Instance),
1402 (r'([a-zA-Z$_](?:[\w$-]*[\w$])?)(\+\+|\-\-)?',
1403 bygroups(Name.Symbol, Operator.Word))
1404 ],
1405 'tuple': [
1406 (r'#[a-zA-Z_][\w\-]*(?=[\s{(,;])', Name.Namespace)
1407 ],
1408 'interpoling_string': [
1409 (r'\}', String.Interpol, '#pop'),
1410 include('root')
1411 ],
1412 'stringescape': [
1413 (r'\\([\\abfnrtv"\']|\n|N\{.*?\}|u[a-fA-F0-9]{4}|'
1414 r'U[a-fA-F0-9]{8}|x[a-fA-F0-9]{2}|[0-7]{1,3})', String.Escape)
1415 ],
1416 'strings': [
1417 (r'[^\\\'"]', String),
1418 (r'[\'"\\]', String),
1419 (r'\n', String) # All strings are multiline in EG
1420 ],
1421 'dqs': [
1422 (r'"', String, '#pop'),
1423 (r'\\\\|\\"|\\\n', String.Escape),
1424 include('strings')
1425 ],
1426 'sqs': [
1427 (r"'", String, '#pop'),
1428 (r"\\\\|\\'|\\\n", String.Escape),
1429 (r'\{', String.Interpol, 'interpoling_string'),
1430 include('strings')
1431 ],
1432 'tdqs': [
1433 (r'"""', String, '#pop'),
1434 include('strings'),
1435 ],
1436 'bt': [
1437 (r'`', String.Backtick, '#pop'),
1438 (r'(?<!`)\n', String.Backtick),
1439 (r'\^=?', String.Escape),
1440 (r'.+', String.Backtick),
1441 ],
1442 'tbt': [
1443 (r'```', String.Backtick, '#pop'),
1444 (r'\n', String.Backtick),
1445 (r'\^=?', String.Escape),
1446 (r'[^`]+', String.Backtick),
1447 ],
1448 'numbers': [
1449 (r'\d+\.(?!\.)\d*([eE][+-]?[0-9]+)?', Number.Float),
1450 (r'\d+[eE][+-]?[0-9]+', Number.Float),
1451 (r'8r[0-7]+', Number.Oct),
1452 (r'2r[01]+', Number.Bin),
1453 (r'16r[a-fA-F0-9]+', Number.Hex),
1454 (r'([3-79]|[12][0-9]|3[0-6])r[a-zA-Z\d]+(\.[a-zA-Z\d]+)?',
1455 Number.Radix),
1456 (r'\d+', Number.Integer)
1457 ],
1458 }
1461class JuttleLexer(RegexLexer):
1462 """
1463 For Juttle source code.
1465 .. versionadded:: 2.2
1466 """
1468 name = 'Juttle'
1469 url = 'http://juttle.github.io/'
1470 aliases = ['juttle']
1471 filenames = ['*.juttle']
1472 mimetypes = ['application/juttle', 'application/x-juttle',
1473 'text/x-juttle', 'text/juttle']
1475 flags = re.DOTALL | re.MULTILINE
1477 tokens = {
1478 'commentsandwhitespace': [
1479 (r'\s+', Whitespace),
1480 (r'(//.*?)(\n)', bygroups(Comment.Single, Whitespace)),
1481 (r'/\*.*?\*/', Comment.Multiline)
1482 ],
1483 'slashstartsregex': [
1484 include('commentsandwhitespace'),
1485 (r'/(\\.|[^[/\\\n]|\[(\\.|[^\]\\\n])*])+/'
1486 r'([gimuysd]+\b|\B)', String.Regex, '#pop'),
1487 (r'(?=/)', Text, ('#pop', 'badregex')),
1488 default('#pop')
1489 ],
1490 'badregex': [
1491 (r'\n', Text, '#pop')
1492 ],
1493 'root': [
1494 (r'^(?=\s|/)', Text, 'slashstartsregex'),
1495 include('commentsandwhitespace'),
1496 (r':\d{2}:\d{2}:\d{2}(\.\d*)?:', String.Moment),
1497 (r':(now|beginning|end|forever|yesterday|today|tomorrow|'
1498 r'(\d+(\.\d*)?|\.\d+)(ms|[smhdwMy])?):', String.Moment),
1499 (r':\d{4}-\d{2}-\d{2}(T\d{2}:\d{2}:\d{2}(\.\d*)?)?'
1500 r'(Z|[+-]\d{2}:\d{2}|[+-]\d{4})?:', String.Moment),
1501 (r':((\d+(\.\d*)?|\.\d+)[ ]+)?(millisecond|second|minute|hour|'
1502 r'day|week|month|year)[s]?'
1503 r'(([ ]+and[ ]+(\d+[ ]+)?(millisecond|second|minute|hour|'
1504 r'day|week|month|year)[s]?)'
1505 r'|[ ]+(ago|from[ ]+now))*:', String.Moment),
1506 (r'\+\+|--|~|&&|\?|:|\|\||\\(?=\n)|'
1507 r'(==?|!=?|[-<>+*%&|^/])=?', Operator, 'slashstartsregex'),
1508 (r'[{(\[;,]', Punctuation, 'slashstartsregex'),
1509 (r'[})\].]', Punctuation),
1510 (r'(import|return|continue|if|else)\b', Keyword, 'slashstartsregex'),
1511 (r'(var|const|function|reducer|sub|input)\b', Keyword.Declaration,
1512 'slashstartsregex'),
1513 (r'(batch|emit|filter|head|join|keep|pace|pass|put|read|reduce|remove|'
1514 r'sequence|skip|sort|split|tail|unbatch|uniq|view|write)\b',
1515 Keyword.Reserved),
1516 (r'(true|false|null|Infinity)\b', Keyword.Constant),
1517 (r'(Array|Date|Juttle|Math|Number|Object|RegExp|String)\b',
1518 Name.Builtin),
1519 (JS_IDENT, Name.Other),
1520 (r'[0-9][0-9]*\.[0-9]+([eE][0-9]+)?[fd]?', Number.Float),
1521 (r'[0-9]+', Number.Integer),
1522 (r'"(\\\\|\\[^\\]|[^"\\])*"', String.Double),
1523 (r"'(\\\\|\\[^\\]|[^'\\])*'", String.Single),
1524 ]
1526 }
1529class NodeConsoleLexer(Lexer):
1530 """
1531 For parsing within an interactive Node.js REPL, such as:
1533 .. sourcecode:: nodejsrepl
1535 > let a = 3
1536 undefined
1537 > a
1538 3
1539 > let b = '4'
1540 undefined
1541 > b
1542 '4'
1543 > b == a
1544 false
1546 .. versionadded: 2.10
1547 """
1548 name = 'Node.js REPL console session'
1549 aliases = ['nodejsrepl', ]
1550 mimetypes = ['text/x-nodejsrepl', ]
1552 def get_tokens_unprocessed(self, text):
1553 jslexer = JavascriptLexer(**self.options)
1555 curcode = ''
1556 insertions = []
1558 for match in line_re.finditer(text):
1559 line = match.group()
1560 if line.startswith('> '):
1561 insertions.append((len(curcode),
1562 [(0, Generic.Prompt, line[:1]),
1563 (1, Whitespace, line[1:2])]))
1565 curcode += line[2:]
1566 elif line.startswith('...'):
1567 # node does a nested ... thing depending on depth
1568 code = line.lstrip('.')
1569 lead = len(line) - len(code)
1571 insertions.append((len(curcode),
1572 [(0, Generic.Prompt, line[:lead])]))
1574 curcode += code
1575 else:
1576 if curcode:
1577 yield from do_insertions(insertions,
1578 jslexer.get_tokens_unprocessed(curcode))
1580 curcode = ''
1581 insertions = []
1583 yield from do_insertions([],
1584 jslexer.get_tokens_unprocessed(line))
1586 if curcode:
1587 yield from do_insertions(insertions,
1588 jslexer.get_tokens_unprocessed(curcode))