Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/pygments/lexers/dsls.py: 77%
149 statements
« prev ^ index » next coverage.py v7.2.7, created at 2023-07-01 06:54 +0000
« prev ^ index » next coverage.py v7.2.7, created at 2023-07-01 06:54 +0000
1"""
2 pygments.lexers.dsls
3 ~~~~~~~~~~~~~~~~~~~~
5 Lexers for various domain-specific languages.
7 :copyright: Copyright 2006-2023 by the Pygments team, see AUTHORS.
8 :license: BSD, see LICENSE for details.
9"""
11import re
13from pygments.lexer import ExtendedRegexLexer, RegexLexer, bygroups, words, \
14 include, default, this, using, combined
15from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
16 Number, Punctuation, Whitespace
18__all__ = ['ProtoBufLexer', 'ZeekLexer', 'PuppetLexer', 'RslLexer',
19 'MscgenLexer', 'VGLLexer', 'AlloyLexer', 'PanLexer',
20 'CrmshLexer', 'ThriftLexer', 'FlatlineLexer', 'SnowballLexer']
23class ProtoBufLexer(RegexLexer):
24 """
25 Lexer for Protocol Buffer definition files.
27 .. versionadded:: 1.4
28 """
30 name = 'Protocol Buffer'
31 url = 'https://developers.google.com/protocol-buffers/'
32 aliases = ['protobuf', 'proto']
33 filenames = ['*.proto']
35 tokens = {
36 'root': [
37 (r'[ \t]+', Whitespace),
38 (r'[,;{}\[\]()<>]', Punctuation),
39 (r'/(\\\n)?/(\n|(.|\n)*?[^\\]\n)', Comment.Single),
40 (r'/(\\\n)?\*(.|\n)*?\*(\\\n)?/', Comment.Multiline),
41 (words((
42 'import', 'option', 'optional', 'required', 'repeated',
43 'reserved', 'default', 'packed', 'ctype', 'extensions', 'to',
44 'max', 'rpc', 'returns', 'oneof', 'syntax'), prefix=r'\b', suffix=r'\b'),
45 Keyword),
46 (words((
47 'int32', 'int64', 'uint32', 'uint64', 'sint32', 'sint64',
48 'fixed32', 'fixed64', 'sfixed32', 'sfixed64',
49 'float', 'double', 'bool', 'string', 'bytes'), suffix=r'\b'),
50 Keyword.Type),
51 (r'(true|false)\b', Keyword.Constant),
52 (r'(package)(\s+)', bygroups(Keyword.Namespace, Whitespace), 'package'),
53 (r'(message|extend)(\s+)',
54 bygroups(Keyword.Declaration, Whitespace), 'message'),
55 (r'(enum|group|service)(\s+)',
56 bygroups(Keyword.Declaration, Whitespace), 'type'),
57 (r'\".*?\"', String),
58 (r'\'.*?\'', String),
59 (r'(\d+\.\d*|\.\d+|\d+)[eE][+-]?\d+[LlUu]*', Number.Float),
60 (r'(\d+\.\d*|\.\d+|\d+[fF])[fF]?', Number.Float),
61 (r'(\-?(inf|nan))\b', Number.Float),
62 (r'0x[0-9a-fA-F]+[LlUu]*', Number.Hex),
63 (r'0[0-7]+[LlUu]*', Number.Oct),
64 (r'\d+[LlUu]*', Number.Integer),
65 (r'[+-=]', Operator),
66 (r'([a-zA-Z_][\w.]*)([ \t]*)(=)',
67 bygroups(Name.Attribute, Whitespace, Operator)),
68 (r'[a-zA-Z_][\w.]*', Name),
69 ],
70 'package': [
71 (r'[a-zA-Z_]\w*', Name.Namespace, '#pop'),
72 default('#pop'),
73 ],
74 'message': [
75 (r'[a-zA-Z_]\w*', Name.Class, '#pop'),
76 default('#pop'),
77 ],
78 'type': [
79 (r'[a-zA-Z_]\w*', Name, '#pop'),
80 default('#pop'),
81 ],
82 }
85class ThriftLexer(RegexLexer):
86 """
87 For Thrift interface definitions.
89 .. versionadded:: 2.1
90 """
91 name = 'Thrift'
92 url = 'https://thrift.apache.org/'
93 aliases = ['thrift']
94 filenames = ['*.thrift']
95 mimetypes = ['application/x-thrift']
97 tokens = {
98 'root': [
99 include('whitespace'),
100 include('comments'),
101 (r'"', String.Double, combined('stringescape', 'dqs')),
102 (r'\'', String.Single, combined('stringescape', 'sqs')),
103 (r'(namespace)(\s+)',
104 bygroups(Keyword.Namespace, Whitespace), 'namespace'),
105 (r'(enum|union|struct|service|exception)(\s+)',
106 bygroups(Keyword.Declaration, Whitespace), 'class'),
107 (r'((?:(?:[^\W\d]|\$)[\w.\[\]$<>]*\s+)+?)' # return arguments
108 r'((?:[^\W\d]|\$)[\w$]*)' # method name
109 r'(\s*)(\()', # signature start
110 bygroups(using(this), Name.Function, Whitespace, Operator)),
111 include('keywords'),
112 include('numbers'),
113 (r'[&=]', Operator),
114 (r'[:;,{}()<>\[\]]', Punctuation),
115 (r'[a-zA-Z_](\.\w|\w)*', Name),
116 ],
117 'whitespace': [
118 (r'\n', Whitespace),
119 (r'\s+', Whitespace),
120 ],
121 'comments': [
122 (r'#.*$', Comment),
123 (r'//.*?\n', Comment),
124 (r'/\*[\w\W]*?\*/', Comment.Multiline),
125 ],
126 'stringescape': [
127 (r'\\([\\nrt"\'])', String.Escape),
128 ],
129 'dqs': [
130 (r'"', String.Double, '#pop'),
131 (r'[^\\"\n]+', String.Double),
132 ],
133 'sqs': [
134 (r"'", String.Single, '#pop'),
135 (r'[^\\\'\n]+', String.Single),
136 ],
137 'namespace': [
138 (r'[a-z*](\.\w|\w)*', Name.Namespace, '#pop'),
139 default('#pop'),
140 ],
141 'class': [
142 (r'[a-zA-Z_]\w*', Name.Class, '#pop'),
143 default('#pop'),
144 ],
145 'keywords': [
146 (r'(async|oneway|extends|throws|required|optional)\b', Keyword),
147 (r'(true|false)\b', Keyword.Constant),
148 (r'(const|typedef)\b', Keyword.Declaration),
149 (words((
150 'cpp_namespace', 'cpp_include', 'cpp_type', 'java_package',
151 'cocoa_prefix', 'csharp_namespace', 'delphi_namespace',
152 'php_namespace', 'py_module', 'perl_package',
153 'ruby_namespace', 'smalltalk_category', 'smalltalk_prefix',
154 'xsd_all', 'xsd_optional', 'xsd_nillable', 'xsd_namespace',
155 'xsd_attrs', 'include'), suffix=r'\b'),
156 Keyword.Namespace),
157 (words((
158 'void', 'bool', 'byte', 'i16', 'i32', 'i64', 'double',
159 'string', 'binary', 'map', 'list', 'set', 'slist',
160 'senum'), suffix=r'\b'),
161 Keyword.Type),
162 (words((
163 'BEGIN', 'END', '__CLASS__', '__DIR__', '__FILE__',
164 '__FUNCTION__', '__LINE__', '__METHOD__', '__NAMESPACE__',
165 'abstract', 'alias', 'and', 'args', 'as', 'assert', 'begin',
166 'break', 'case', 'catch', 'class', 'clone', 'continue',
167 'declare', 'def', 'default', 'del', 'delete', 'do', 'dynamic',
168 'elif', 'else', 'elseif', 'elsif', 'end', 'enddeclare',
169 'endfor', 'endforeach', 'endif', 'endswitch', 'endwhile',
170 'ensure', 'except', 'exec', 'finally', 'float', 'for',
171 'foreach', 'function', 'global', 'goto', 'if', 'implements',
172 'import', 'in', 'inline', 'instanceof', 'interface', 'is',
173 'lambda', 'module', 'native', 'new', 'next', 'nil', 'not',
174 'or', 'pass', 'public', 'print', 'private', 'protected',
175 'raise', 'redo', 'rescue', 'retry', 'register', 'return',
176 'self', 'sizeof', 'static', 'super', 'switch', 'synchronized',
177 'then', 'this', 'throw', 'transient', 'try', 'undef',
178 'unless', 'unsigned', 'until', 'use', 'var', 'virtual',
179 'volatile', 'when', 'while', 'with', 'xor', 'yield'),
180 prefix=r'\b', suffix=r'\b'),
181 Keyword.Reserved),
182 ],
183 'numbers': [
184 (r'[+-]?(\d+\.\d+([eE][+-]?\d+)?|\.?\d+[eE][+-]?\d+)', Number.Float),
185 (r'[+-]?0x[0-9A-Fa-f]+', Number.Hex),
186 (r'[+-]?[0-9]+', Number.Integer),
187 ],
188 }
191class ZeekLexer(RegexLexer):
192 """
193 For Zeek scripts.
195 .. versionadded:: 2.5
196 """
197 name = 'Zeek'
198 url = 'https://www.zeek.org/'
199 aliases = ['zeek', 'bro']
200 filenames = ['*.zeek', '*.bro']
202 _hex = r'[0-9a-fA-F]'
203 _float = r'((\d*\.?\d+)|(\d+\.?\d*))([eE][-+]?\d+)?'
204 _h = r'[A-Za-z0-9][-A-Za-z0-9]*'
206 tokens = {
207 'root': [
208 include('whitespace'),
209 include('comments'),
210 include('directives'),
211 include('attributes'),
212 include('types'),
213 include('keywords'),
214 include('literals'),
215 include('operators'),
216 include('punctuation'),
217 (r'((?:[A-Za-z_]\w*)(?:::(?:[A-Za-z_]\w*))*)(?=\s*\()',
218 Name.Function),
219 include('identifiers'),
220 ],
222 'whitespace': [
223 (r'\n', Whitespace),
224 (r'\s+', Whitespace),
225 (r'(\\)(\n)', bygroups(Text, Whitespace)),
226 ],
228 'comments': [
229 (r'#.*$', Comment),
230 ],
232 'directives': [
233 (r'@(load-plugin|load-sigs|load|unload)\b.*$', Comment.Preproc),
234 (r'@(DEBUG|DIR|FILENAME|deprecated|if|ifdef|ifndef|else|endif)\b', Comment.Preproc),
235 (r'(@prefixes)(\s*)((\+?=).*)$', bygroups(Comment.Preproc,
236 Whitespace, Comment.Preproc)),
237 ],
239 'attributes': [
240 (words(('redef', 'priority', 'log', 'optional', 'default', 'add_func',
241 'delete_func', 'expire_func', 'read_expire', 'write_expire',
242 'create_expire', 'synchronized', 'persistent', 'rotate_interval',
243 'rotate_size', 'encrypt', 'raw_output', 'mergeable', 'error_handler',
244 'type_column', 'deprecated'),
245 prefix=r'&', suffix=r'\b'),
246 Keyword.Pseudo),
247 ],
249 'types': [
250 (words(('any',
251 'enum', 'record', 'set', 'table', 'vector',
252 'function', 'hook', 'event',
253 'addr', 'bool', 'count', 'double', 'file', 'int', 'interval',
254 'pattern', 'port', 'string', 'subnet', 'time'),
255 suffix=r'\b'),
256 Keyword.Type),
258 (r'(opaque)(\s+)(of)(\s+)((?:[A-Za-z_]\w*)(?:::(?:[A-Za-z_]\w*))*)\b',
259 bygroups(Keyword.Type, Whitespace, Operator.Word, Whitespace, Keyword.Type)),
261 (r'(type)(\s+)((?:[A-Za-z_]\w*)(?:::(?:[A-Za-z_]\w*))*)(\s*)(:)(\s*)\b(record|enum)\b',
262 bygroups(Keyword, Whitespace, Name.Class, Whitespace, Operator, Whitespace, Keyword.Type)),
264 (r'(type)(\s+)((?:[A-Za-z_]\w*)(?:::(?:[A-Za-z_]\w*))*)(\s*)(:)',
265 bygroups(Keyword, Whitespace, Name, Whitespace, Operator)),
267 (r'(redef)(\s+)(record|enum)(\s+)((?:[A-Za-z_]\w*)(?:::(?:[A-Za-z_]\w*))*)\b',
268 bygroups(Keyword, Whitespace, Keyword.Type, Whitespace, Name.Class)),
269 ],
271 'keywords': [
272 (words(('redef', 'export', 'if', 'else', 'for', 'while',
273 'return', 'break', 'next', 'continue', 'fallthrough',
274 'switch', 'default', 'case',
275 'add', 'delete',
276 'when', 'timeout', 'schedule'),
277 suffix=r'\b'),
278 Keyword),
279 (r'(print)\b', Keyword),
280 (r'(global|local|const|option)\b', Keyword.Declaration),
281 (r'(module)(\s+)(([A-Za-z_]\w*)(?:::([A-Za-z_]\w*))*)\b',
282 bygroups(Keyword.Namespace, Whitespace, Name.Namespace)),
283 ],
285 'literals': [
286 (r'"', String, 'string'),
288 # Not the greatest match for patterns, but generally helps
289 # disambiguate between start of a pattern and just a division
290 # operator.
291 (r'/(?=.*/)', String.Regex, 'regex'),
293 (r'(T|F)\b', Keyword.Constant),
295 # Port
296 (r'\d{1,5}/(udp|tcp|icmp|unknown)\b', Number),
298 # IPv4 Address
299 (r'(\d{1,3}.){3}(\d{1,3})\b', Number),
301 # IPv6 Address
302 (r'\[([0-9a-fA-F]{0,4}:){2,7}([0-9a-fA-F]{0,4})?((\d{1,3}.){3}(\d{1,3}))?\]', Number),
304 # Numeric
305 (r'0[xX]' + _hex + r'+\b', Number.Hex),
306 (_float + r'\s*(day|hr|min|sec|msec|usec)s?\b', Number.Float),
307 (_float + r'\b', Number.Float),
308 (r'(\d+)\b', Number.Integer),
310 # Hostnames
311 (_h + r'(\.' + _h + r')+', String),
312 ],
314 'operators': [
315 (r'[!%*/+<=>~|&^-]', Operator),
316 (r'([-+=&|]{2}|[+=!><-]=)', Operator),
317 (r'(in|as|is|of)\b', Operator.Word),
318 (r'\??\$', Operator),
319 ],
321 'punctuation': [
322 (r'[{}()\[\],;.]', Punctuation),
323 # The "ternary if", which uses '?' and ':', could instead be
324 # treated as an Operator, but colons are more frequently used to
325 # separate field/identifier names from their types, so the (often)
326 # less-prominent Punctuation is used even with '?' for consistency.
327 (r'[?:]', Punctuation),
328 ],
330 'identifiers': [
331 (r'([a-zA-Z_]\w*)(::)', bygroups(Name, Punctuation)),
332 (r'[a-zA-Z_]\w*', Name)
333 ],
335 'string': [
336 (r'\\.', String.Escape),
337 (r'%-?[0-9]*(\.[0-9]+)?[DTd-gsx]', String.Escape),
338 (r'"', String, '#pop'),
339 (r'.', String),
340 ],
342 'regex': [
343 (r'\\.', String.Escape),
344 (r'/', String.Regex, '#pop'),
345 (r'.', String.Regex),
346 ],
347 }
350BroLexer = ZeekLexer
353class PuppetLexer(RegexLexer):
354 """
355 For Puppet configuration DSL.
357 .. versionadded:: 1.6
358 """
359 name = 'Puppet'
360 url = 'https://puppet.com/'
361 aliases = ['puppet']
362 filenames = ['*.pp']
364 tokens = {
365 'root': [
366 include('comments'),
367 include('keywords'),
368 include('names'),
369 include('numbers'),
370 include('operators'),
371 include('strings'),
373 (r'[]{}:(),;[]', Punctuation),
374 (r'\s+', Whitespace),
375 ],
377 'comments': [
378 (r'(\s*)(#.*)$', bygroups(Whitespace, Comment)),
379 (r'/(\\\n)?[*](.|\n)*?[*](\\\n)?/', Comment.Multiline),
380 ],
382 'operators': [
383 (r'(=>|\?|<|>|=|\+|-|/|\*|~|!|\|)', Operator),
384 (r'(in|and|or|not)\b', Operator.Word),
385 ],
387 'names': [
388 (r'[a-zA-Z_]\w*', Name.Attribute),
389 (r'(\$\S+)(\[)(\S+)(\])', bygroups(Name.Variable, Punctuation,
390 String, Punctuation)),
391 (r'\$\S+', Name.Variable),
392 ],
394 'numbers': [
395 # Copypasta from the Python lexer
396 (r'(\d+\.\d*|\d*\.\d+)([eE][+-]?[0-9]+)?j?', Number.Float),
397 (r'\d+[eE][+-]?[0-9]+j?', Number.Float),
398 (r'0[0-7]+j?', Number.Oct),
399 (r'0[xX][a-fA-F0-9]+', Number.Hex),
400 (r'\d+L', Number.Integer.Long),
401 (r'\d+j?', Number.Integer)
402 ],
404 'keywords': [
405 # Left out 'group' and 'require'
406 # Since they're often used as attributes
407 (words((
408 'absent', 'alert', 'alias', 'audit', 'augeas', 'before', 'case',
409 'check', 'class', 'computer', 'configured', 'contained',
410 'create_resources', 'crit', 'cron', 'debug', 'default',
411 'define', 'defined', 'directory', 'else', 'elsif', 'emerg',
412 'err', 'exec', 'extlookup', 'fail', 'false', 'file',
413 'filebucket', 'fqdn_rand', 'generate', 'host', 'if', 'import',
414 'include', 'info', 'inherits', 'inline_template', 'installed',
415 'interface', 'k5login', 'latest', 'link', 'loglevel',
416 'macauthorization', 'mailalias', 'maillist', 'mcx', 'md5',
417 'mount', 'mounted', 'nagios_command', 'nagios_contact',
418 'nagios_contactgroup', 'nagios_host', 'nagios_hostdependency',
419 'nagios_hostescalation', 'nagios_hostextinfo', 'nagios_hostgroup',
420 'nagios_service', 'nagios_servicedependency', 'nagios_serviceescalation',
421 'nagios_serviceextinfo', 'nagios_servicegroup', 'nagios_timeperiod',
422 'node', 'noop', 'notice', 'notify', 'package', 'present', 'purged',
423 'realize', 'regsubst', 'resources', 'role', 'router', 'running',
424 'schedule', 'scheduled_task', 'search', 'selboolean', 'selmodule',
425 'service', 'sha1', 'shellquote', 'split', 'sprintf',
426 'ssh_authorized_key', 'sshkey', 'stage', 'stopped', 'subscribe',
427 'tag', 'tagged', 'template', 'tidy', 'true', 'undef', 'unmounted',
428 'user', 'versioncmp', 'vlan', 'warning', 'yumrepo', 'zfs', 'zone',
429 'zpool'), prefix='(?i)', suffix=r'\b'),
430 Keyword),
431 ],
433 'strings': [
434 (r'"([^"])*"', String),
435 (r"'(\\'|[^'])*'", String),
436 ],
438 }
441class RslLexer(RegexLexer):
442 """
443 RSL is the formal specification
444 language used in RAISE (Rigorous Approach to Industrial Software Engineering)
445 method.
447 .. versionadded:: 2.0
448 """
449 name = 'RSL'
450 url = 'http://en.wikipedia.org/wiki/RAISE'
451 aliases = ['rsl']
452 filenames = ['*.rsl']
453 mimetypes = ['text/rsl']
455 flags = re.MULTILINE | re.DOTALL
457 tokens = {
458 'root': [
459 (words((
460 'Bool', 'Char', 'Int', 'Nat', 'Real', 'Text', 'Unit', 'abs',
461 'all', 'always', 'any', 'as', 'axiom', 'card', 'case', 'channel',
462 'chaos', 'class', 'devt_relation', 'dom', 'elems', 'else', 'elif',
463 'end', 'exists', 'extend', 'false', 'for', 'hd', 'hide', 'if',
464 'in', 'is', 'inds', 'initialise', 'int', 'inter', 'isin', 'len',
465 'let', 'local', 'ltl_assertion', 'object', 'of', 'out', 'post',
466 'pre', 'read', 'real', 'rng', 'scheme', 'skip', 'stop', 'swap',
467 'then', 'theory', 'test_case', 'tl', 'transition_system', 'true',
468 'type', 'union', 'until', 'use', 'value', 'variable', 'while',
469 'with', 'write', '~isin', '-inflist', '-infset', '-list',
470 '-set'), prefix=r'\b', suffix=r'\b'),
471 Keyword),
472 (r'(variable|value)\b', Keyword.Declaration),
473 (r'--.*?\n', Comment),
474 (r'<:.*?:>', Comment),
475 (r'\{!.*?!\}', Comment),
476 (r'/\*.*?\*/', Comment),
477 (r'^([ \t]*)([\w]+)([ \t]*)(:[^:])', bygroups(Whitespace,
478 Name.Function, Whitespace, Name.Function)),
479 (r'(^[ \t]*)([\w]+)([ \t]*)(\([\w\s,]*\))([ \t]*)(is|as)',
480 bygroups(Whitespace, Name.Function, Whitespace, Text,
481 Whitespace, Keyword)),
482 (r'\b[A-Z]\w*\b', Keyword.Type),
483 (r'(true|false)\b', Keyword.Constant),
484 (r'".*"', String),
485 (r'\'.\'', String.Char),
486 (r'(><|->|-m->|/\\|<=|<<=|<\.|\|\||\|\^\||-~->|-~m->|\\/|>=|>>|'
487 r'\.>|\+\+|-\\|<->|=>|:-|~=|\*\*|<<|>>=|\+>|!!|\|=\||#)',
488 Operator),
489 (r'[0-9]+\.[0-9]+([eE][0-9]+)?[fd]?', Number.Float),
490 (r'0x[0-9a-f]+', Number.Hex),
491 (r'[0-9]+', Number.Integer),
492 (r'\s+', Whitespace),
493 (r'.', Text),
494 ],
495 }
497 def analyse_text(text):
498 """
499 Check for the most common text in the beginning of a RSL file.
500 """
501 if re.search(r'scheme\s*.*?=\s*class\s*type', text, re.I) is not None:
502 return 1.0
505class MscgenLexer(RegexLexer):
506 """
507 For Mscgen files.
509 .. versionadded:: 1.6
510 """
511 name = 'Mscgen'
512 url = 'http://www.mcternan.me.uk/mscgen/'
513 aliases = ['mscgen', 'msc']
514 filenames = ['*.msc']
516 _var = r'(\w+|"(?:\\"|[^"])*")'
518 tokens = {
519 'root': [
520 (r'msc\b', Keyword.Type),
521 # Options
522 (r'(hscale|HSCALE|width|WIDTH|wordwraparcs|WORDWRAPARCS'
523 r'|arcgradient|ARCGRADIENT)\b', Name.Property),
524 # Operators
525 (r'(abox|ABOX|rbox|RBOX|box|BOX|note|NOTE)\b', Operator.Word),
526 (r'(\.|-|\|){3}', Keyword),
527 (r'(?:-|=|\.|:){2}'
528 r'|<<=>>|<->|<=>|<<>>|<:>'
529 r'|->|=>>|>>|=>|:>|-x|-X'
530 r'|<-|<<=|<<|<=|<:|x-|X-|=', Operator),
531 # Names
532 (r'\*', Name.Builtin),
533 (_var, Name.Variable),
534 # Other
535 (r'\[', Punctuation, 'attrs'),
536 (r'\{|\}|,|;', Punctuation),
537 include('comments')
538 ],
539 'attrs': [
540 (r'\]', Punctuation, '#pop'),
541 (_var + r'(\s*)(=)(\s*)' + _var,
542 bygroups(Name.Attribute, Whitespace, Operator, Whitespace,
543 String)),
544 (r',', Punctuation),
545 include('comments')
546 ],
547 'comments': [
548 (r'(?://|#).*?\n', Comment.Single),
549 (r'/\*(?:.|\n)*?\*/', Comment.Multiline),
550 (r'[ \t\r\n]+', Whitespace)
551 ]
552 }
555class VGLLexer(RegexLexer):
556 """
557 For SampleManager VGL source code.
559 .. versionadded:: 1.6
560 """
561 name = 'VGL'
562 url = 'http://www.thermoscientific.com/samplemanager'
563 aliases = ['vgl']
564 filenames = ['*.rpf']
566 flags = re.MULTILINE | re.DOTALL | re.IGNORECASE
568 tokens = {
569 'root': [
570 (r'\{[^}]*\}', Comment.Multiline),
571 (r'declare', Keyword.Constant),
572 (r'(if|then|else|endif|while|do|endwhile|and|or|prompt|object'
573 r'|create|on|line|with|global|routine|value|endroutine|constant'
574 r'|global|set|join|library|compile_option|file|exists|create|copy'
575 r'|delete|enable|windows|name|notprotected)(?! *[=<>.,()])',
576 Keyword),
577 (r'(true|false|null|empty|error|locked)', Keyword.Constant),
578 (r'[~^*#!%&\[\]()<>|+=:;,./?-]', Operator),
579 (r'"[^"]*"', String),
580 (r'(\.)([a-z_$][\w$]*)', bygroups(Operator, Name.Attribute)),
581 (r'[0-9][0-9]*(\.[0-9]+(e[+\-]?[0-9]+)?)?', Number),
582 (r'[a-z_$][\w$]*', Name),
583 (r'[\r\n]+', Whitespace),
584 (r'\s+', Whitespace)
585 ]
586 }
589class AlloyLexer(RegexLexer):
590 """
591 For Alloy source code.
593 .. versionadded:: 2.0
594 """
596 name = 'Alloy'
597 url = 'http://alloy.mit.edu'
598 aliases = ['alloy']
599 filenames = ['*.als']
600 mimetypes = ['text/x-alloy']
602 flags = re.MULTILINE | re.DOTALL
604 iden_rex = r'[a-zA-Z_][\w]*"*'
605 string_rex = r'"\b(\\\\|\\[^\\]|[^"\\])*"'
606 text_tuple = (r'[^\S\n]+', Whitespace)
608 tokens = {
609 'sig': [
610 (r'(extends)\b', Keyword, '#pop'),
611 (iden_rex, Name),
612 text_tuple,
613 (r',', Punctuation),
614 (r'\{', Operator, '#pop'),
615 ],
616 'module': [
617 text_tuple,
618 (iden_rex, Name, '#pop'),
619 ],
620 'fun': [
621 text_tuple,
622 (r'\{', Operator, '#pop'),
623 (iden_rex, Name, '#pop'),
624 ],
625 'fact': [
626 include('fun'),
627 (string_rex, String, '#pop'),
628 ],
629 'root': [
630 (r'--.*?$', Comment.Single),
631 (r'//.*?$', Comment.Single),
632 (r'/\*.*?\*/', Comment.Multiline),
633 text_tuple,
634 (r'(module|open)(\s+)', bygroups(Keyword.Namespace, Whitespace),
635 'module'),
636 (r'(sig|enum)(\s+)', bygroups(Keyword.Declaration, Whitespace), 'sig'),
637 (r'(iden|univ|none)\b', Keyword.Constant),
638 (r'(int|Int)\b', Keyword.Type),
639 (r'(var|this|abstract|extends|set|seq|one|lone|let)\b', Keyword),
640 (r'(all|some|no|sum|disj|when|else)\b', Keyword),
641 (r'(run|check|for|but|exactly|expect|as|steps)\b', Keyword),
642 (r'(always|after|eventually|until|release)\b', Keyword), # future time operators
643 (r'(historically|before|once|since|triggered)\b', Keyword), # past time operators
644 (r'(and|or|implies|iff|in)\b', Operator.Word),
645 (r'(fun|pred|assert)(\s+)', bygroups(Keyword, Whitespace), 'fun'),
646 (r'(fact)(\s+)', bygroups(Keyword, Whitespace), 'fact'),
647 (r'!|#|&&|\+\+|<<|>>|>=|<=>|<=|\.\.|\.|->', Operator),
648 (r'[-+/*%=<>&!^|~{}\[\]().\';]', Operator),
649 (iden_rex, Name),
650 (r'[:,]', Punctuation),
651 (r'[0-9]+', Number.Integer),
652 (string_rex, String),
653 (r'\n', Whitespace),
654 ]
655 }
658class PanLexer(RegexLexer):
659 """
660 Lexer for pan source files.
662 Based on tcsh lexer.
664 .. versionadded:: 2.0
665 """
667 name = 'Pan'
668 url = 'https://github.com/quattor/pan/'
669 aliases = ['pan']
670 filenames = ['*.pan']
672 tokens = {
673 'root': [
674 include('basic'),
675 (r'\(', Keyword, 'paren'),
676 (r'\{', Keyword, 'curly'),
677 include('data'),
678 ],
679 'basic': [
680 (words((
681 'if', 'for', 'with', 'else', 'type', 'bind', 'while', 'valid', 'final',
682 'prefix', 'unique', 'object', 'foreach', 'include', 'template',
683 'function', 'variable', 'structure', 'extensible', 'declaration'),
684 prefix=r'\b', suffix=r'\b'),
685 Keyword),
686 (words((
687 'file_contents', 'format', 'index', 'length', 'match', 'matches',
688 'replace', 'splice', 'split', 'substr', 'to_lowercase', 'to_uppercase',
689 'debug', 'error', 'traceback', 'deprecated', 'base64_decode',
690 'base64_encode', 'digest', 'escape', 'unescape', 'append', 'create',
691 'first', 'nlist', 'key', 'list', 'merge', 'next', 'prepend', 'is_boolean',
692 'is_defined', 'is_double', 'is_list', 'is_long', 'is_nlist', 'is_null',
693 'is_number', 'is_property', 'is_resource', 'is_string', 'to_boolean',
694 'to_double', 'to_long', 'to_string', 'clone', 'delete', 'exists',
695 'path_exists', 'if_exists', 'return', 'value'),
696 prefix=r'\b', suffix=r'\b'),
697 Name.Builtin),
698 (r'#.*', Comment),
699 (r'\\[\w\W]', String.Escape),
700 (r'(\b\w+)(\s*)(=)', bygroups(Name.Variable, Whitespace, Operator)),
701 (r'[\[\]{}()=]+', Operator),
702 (r'<<\s*(\'?)\\?(\w+)[\w\W]+?\2', String),
703 (r';', Punctuation),
704 ],
705 'data': [
706 (r'(?s)"(\\\\|\\[0-7]+|\\.|[^"\\])*"', String.Double),
707 (r"(?s)'(\\\\|\\[0-7]+|\\.|[^'\\])*'", String.Single),
708 (r'\s+', Whitespace),
709 (r'[^=\s\[\]{}()$"\'`\\;#]+', Text),
710 (r'\d+(?= |\Z)', Number),
711 ],
712 'curly': [
713 (r'\}', Keyword, '#pop'),
714 (r':-', Keyword),
715 (r'\w+', Name.Variable),
716 (r'[^}:"\'`$]+', Punctuation),
717 (r':', Punctuation),
718 include('root'),
719 ],
720 'paren': [
721 (r'\)', Keyword, '#pop'),
722 include('root'),
723 ],
724 }
727class CrmshLexer(RegexLexer):
728 """
729 Lexer for crmsh configuration files for Pacemaker clusters.
731 .. versionadded:: 2.1
732 """
733 name = 'Crmsh'
734 url = 'http://crmsh.github.io/'
735 aliases = ['crmsh', 'pcmk']
736 filenames = ['*.crmsh', '*.pcmk']
737 mimetypes = []
739 elem = words((
740 'node', 'primitive', 'group', 'clone', 'ms', 'location',
741 'colocation', 'order', 'fencing_topology', 'rsc_ticket',
742 'rsc_template', 'property', 'rsc_defaults',
743 'op_defaults', 'acl_target', 'acl_group', 'user', 'role',
744 'tag'), suffix=r'(?![\w#$-])')
745 sub = words((
746 'params', 'meta', 'operations', 'op', 'rule',
747 'attributes', 'utilization'), suffix=r'(?![\w#$-])')
748 acl = words(('read', 'write', 'deny'), suffix=r'(?![\w#$-])')
749 bin_rel = words(('and', 'or'), suffix=r'(?![\w#$-])')
750 un_ops = words(('defined', 'not_defined'), suffix=r'(?![\w#$-])')
751 date_exp = words(('in_range', 'date', 'spec', 'in'), suffix=r'(?![\w#$-])')
752 acl_mod = (r'(?:tag|ref|reference|attribute|type|xpath)')
753 bin_ops = (r'(?:lt|gt|lte|gte|eq|ne)')
754 val_qual = (r'(?:string|version|number)')
755 rsc_role_action = (r'(?:Master|Started|Slave|Stopped|'
756 r'start|promote|demote|stop)')
758 tokens = {
759 'root': [
760 (r'^(#.*)(\n)?', bygroups(Comment, Whitespace)),
761 # attr=value (nvpair)
762 (r'([\w#$-]+)(=)("(?:""|[^"])*"|\S+)',
763 bygroups(Name.Attribute, Punctuation, String)),
764 # need this construct, otherwise numeric node ids
765 # are matched as scores
766 # elem id:
767 (r'(node)(\s+)([\w#$-]+)(:)',
768 bygroups(Keyword, Whitespace, Name, Punctuation)),
769 # scores
770 (r'([+-]?([0-9]+|inf)):', Number),
771 # keywords (elements and other)
772 (elem, Keyword),
773 (sub, Keyword),
774 (acl, Keyword),
775 # binary operators
776 (r'(?:%s:)?(%s)(?![\w#$-])' % (val_qual, bin_ops), Operator.Word),
777 # other operators
778 (bin_rel, Operator.Word),
779 (un_ops, Operator.Word),
780 (date_exp, Operator.Word),
781 # builtin attributes (e.g. #uname)
782 (r'#[a-z]+(?![\w#$-])', Name.Builtin),
783 # acl_mod:blah
784 (r'(%s)(:)("(?:""|[^"])*"|\S+)' % acl_mod,
785 bygroups(Keyword, Punctuation, Name)),
786 # rsc_id[:(role|action)]
787 # NB: this matches all other identifiers
788 (r'([\w#$-]+)(?:(:)(%s))?(?![\w#$-])' % rsc_role_action,
789 bygroups(Name, Punctuation, Operator.Word)),
790 # punctuation
791 (r'(\\(?=\n)|[\[\](){}/:@])', Punctuation),
792 (r'\s+|\n', Whitespace),
793 ],
794 }
797class FlatlineLexer(RegexLexer):
798 """
799 Lexer for Flatline expressions.
801 .. versionadded:: 2.2
802 """
803 name = 'Flatline'
804 url = 'https://github.com/bigmlcom/flatline'
805 aliases = ['flatline']
806 filenames = []
807 mimetypes = ['text/x-flatline']
809 special_forms = ('let',)
811 builtins = (
812 "!=", "*", "+", "-", "<", "<=", "=", ">", ">=", "abs", "acos", "all",
813 "all-but", "all-with-defaults", "all-with-numeric-default", "and",
814 "asin", "atan", "avg", "avg-window", "bin-center", "bin-count", "call",
815 "category-count", "ceil", "cond", "cond-window", "cons", "cos", "cosh",
816 "count", "diff-window", "div", "ensure-value", "ensure-weighted-value",
817 "epoch", "epoch-day", "epoch-fields", "epoch-hour", "epoch-millisecond",
818 "epoch-minute", "epoch-month", "epoch-second", "epoch-weekday",
819 "epoch-year", "exp", "f", "field", "field-prop", "fields", "filter",
820 "first", "floor", "head", "if", "in", "integer", "language", "length",
821 "levenshtein", "linear-regression", "list", "ln", "log", "log10", "map",
822 "matches", "matches?", "max", "maximum", "md5", "mean", "median", "min",
823 "minimum", "missing", "missing-count", "missing?", "missing_count",
824 "mod", "mode", "normalize", "not", "nth", "occurrences", "or",
825 "percentile", "percentile-label", "population", "population-fraction",
826 "pow", "preferred", "preferred?", "quantile-label", "rand", "rand-int",
827 "random-value", "re-quote", "real", "replace", "replace-first", "rest",
828 "round", "row-number", "segment-label", "sha1", "sha256", "sin", "sinh",
829 "sqrt", "square", "standard-deviation", "standard_deviation", "str",
830 "subs", "sum", "sum-squares", "sum-window", "sum_squares", "summary",
831 "summary-no", "summary-str", "tail", "tan", "tanh", "to-degrees",
832 "to-radians", "variance", "vectorize", "weighted-random-value", "window",
833 "winnow", "within-percentiles?", "z-score",
834 )
836 valid_name = r'(?!#)[\w!$%*+<=>?/.#-]+'
838 tokens = {
839 'root': [
840 # whitespaces - usually not relevant
841 (r'[,]+', Text),
842 (r'\s+', Whitespace),
844 # numbers
845 (r'-?\d+\.\d+', Number.Float),
846 (r'-?\d+', Number.Integer),
847 (r'0x-?[a-f\d]+', Number.Hex),
849 # strings, symbols and characters
850 (r'"(\\\\|\\[^\\]|[^"\\])*"', String),
851 (r"\\(.|[a-z]+)", String.Char),
853 # expression template placeholder
854 (r'_', String.Symbol),
856 # highlight the special forms
857 (words(special_forms, suffix=' '), Keyword),
859 # highlight the builtins
860 (words(builtins, suffix=' '), Name.Builtin),
862 # the remaining functions
863 (r'(?<=\()' + valid_name, Name.Function),
865 # find the remaining variables
866 (valid_name, Name.Variable),
868 # parentheses
869 (r'(\(|\))', Punctuation),
870 ],
871 }
874class SnowballLexer(ExtendedRegexLexer):
875 """
876 Lexer for Snowball source code.
878 .. versionadded:: 2.2
879 """
881 name = 'Snowball'
882 url = 'http://snowballstem.org/'
883 aliases = ['snowball']
884 filenames = ['*.sbl']
886 _ws = r'\n\r\t '
888 def __init__(self, **options):
889 self._reset_stringescapes()
890 ExtendedRegexLexer.__init__(self, **options)
892 def _reset_stringescapes(self):
893 self._start = "'"
894 self._end = "'"
896 def _string(do_string_first):
897 def callback(lexer, match, ctx):
898 s = match.start()
899 text = match.group()
900 string = re.compile(r'([^%s]*)(.)' % re.escape(lexer._start)).match
901 escape = re.compile(r'([^%s]*)(.)' % re.escape(lexer._end)).match
902 pos = 0
903 do_string = do_string_first
904 while pos < len(text):
905 if do_string:
906 match = string(text, pos)
907 yield s + match.start(1), String.Single, match.group(1)
908 if match.group(2) == "'":
909 yield s + match.start(2), String.Single, match.group(2)
910 ctx.stack.pop()
911 break
912 yield s + match.start(2), String.Escape, match.group(2)
913 pos = match.end()
914 match = escape(text, pos)
915 yield s + match.start(), String.Escape, match.group()
916 if match.group(2) != lexer._end:
917 ctx.stack[-1] = 'escape'
918 break
919 pos = match.end()
920 do_string = True
921 ctx.pos = s + match.end()
922 return callback
924 def _stringescapes(lexer, match, ctx):
925 lexer._start = match.group(3)
926 lexer._end = match.group(5)
927 return bygroups(Keyword.Reserved, Whitespace, String.Escape, Whitespace,
928 String.Escape)(lexer, match, ctx)
930 tokens = {
931 'root': [
932 (words(('len', 'lenof'), suffix=r'\b'), Operator.Word),
933 include('root1'),
934 ],
935 'root1': [
936 (r'[%s]+' % _ws, Whitespace),
937 (r'\d+', Number.Integer),
938 (r"'", String.Single, 'string'),
939 (r'[()]', Punctuation),
940 (r'/\*[\w\W]*?\*/', Comment.Multiline),
941 (r'//.*', Comment.Single),
942 (r'[!*+\-/<=>]=|[-=]>|<[+-]|[$*+\-/<=>?\[\]]', Operator),
943 (words(('as', 'get', 'hex', 'among', 'define', 'decimal',
944 'backwardmode'), suffix=r'\b'),
945 Keyword.Reserved),
946 (words(('strings', 'booleans', 'integers', 'routines', 'externals',
947 'groupings'), suffix=r'\b'),
948 Keyword.Reserved, 'declaration'),
949 (words(('do', 'or', 'and', 'for', 'hop', 'non', 'not', 'set', 'try',
950 'fail', 'goto', 'loop', 'next', 'test', 'true',
951 'false', 'unset', 'atmark', 'attach', 'delete', 'gopast',
952 'insert', 'repeat', 'sizeof', 'tomark', 'atleast',
953 'atlimit', 'reverse', 'setmark', 'tolimit', 'setlimit',
954 'backwards', 'substring'), suffix=r'\b'),
955 Operator.Word),
956 (words(('size', 'limit', 'cursor', 'maxint', 'minint'),
957 suffix=r'\b'),
958 Name.Builtin),
959 (r'(stringdef\b)([%s]*)([^%s]+)' % (_ws, _ws),
960 bygroups(Keyword.Reserved, Whitespace, String.Escape)),
961 (r'(stringescapes\b)([%s]*)(.)([%s]*)(.)' % (_ws, _ws),
962 _stringescapes),
963 (r'[A-Za-z]\w*', Name),
964 ],
965 'declaration': [
966 (r'\)', Punctuation, '#pop'),
967 (words(('len', 'lenof'), suffix=r'\b'), Name,
968 ('root1', 'declaration')),
969 include('root1'),
970 ],
971 'string': [
972 (r"[^']*'", _string(True)),
973 ],
974 'escape': [
975 (r"[^']*'", _string(False)),
976 ],
977 }
979 def get_tokens_unprocessed(self, text=None, context=None):
980 self._reset_stringescapes()
981 return ExtendedRegexLexer.get_tokens_unprocessed(self, text, context)