Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""
2 pygments.lexers.markup
3 ~~~~~~~~~~~~~~~~~~~~~~
5 Lexers for non-HTML markup languages.
7 :copyright: Copyright 2006-present by the Pygments team, see AUTHORS.
8 :license: BSD, see LICENSE for details.
9"""
11import re
13from pygments.lexers.html import XmlLexer
14from pygments.lexers.javascript import JavascriptLexer
15from pygments.lexers.css import CssLexer
16from pygments.lexers.lilypond import LilyPondLexer
17from pygments.lexers.data import JsonLexer
19from pygments.lexer import RegexLexer, DelegatingLexer, include, bygroups, \
20 using, this, do_insertions, default, words
21from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
22 Number, Punctuation, Generic, Other, Whitespace, Literal
23from pygments.util import get_bool_opt, ClassNotFound
25__all__ = ['BBCodeLexer', 'MoinWikiLexer', 'RstLexer', 'TexLexer', 'GroffLexer',
26 'MozPreprocHashLexer', 'MozPreprocPercentLexer',
27 'MozPreprocXulLexer', 'MozPreprocJavascriptLexer',
28 'MozPreprocCssLexer', 'MarkdownLexer', 'OrgLexer', 'TiddlyWiki5Lexer',
29 'WikitextLexer']
32def _shift_indices(tokens, offset):
33 """Re-base token indices yielded by a delegated sub-lexer.
35 A sub-lexer's ``get_tokens_unprocessed`` returns indices relative to the
36 snippet it was given (starting at 0), but the surrounding lexer must yield
37 indices that are absolute within the whole input text.
38 """
39 for index, token, value in tokens:
40 yield index + offset, token, value
43class BBCodeLexer(RegexLexer):
44 """
45 A lexer that highlights BBCode(-like) syntax.
46 """
48 name = 'BBCode'
49 aliases = ['bbcode']
50 mimetypes = ['text/x-bbcode']
51 url = 'https://www.bbcode.org/'
52 version_added = '0.6'
54 tokens = {
55 'root': [
56 (r'[^[]+', Text),
57 # tag/end tag begin
58 (r'\[/?\w+', Keyword, 'tag'),
59 # stray bracket
60 (r'\[', Text),
61 ],
62 'tag': [
63 (r'\s+', Text),
64 # attribute with value
65 (r'(\w+)(=)("?[^\s"\]]+"?)',
66 bygroups(Name.Attribute, Operator, String)),
67 # tag argument (a la [color=green])
68 (r'(=)("?[^\s"\]]+"?)',
69 bygroups(Operator, String)),
70 # tag end
71 (r'\]', Keyword, '#pop'),
72 ],
73 }
76class MoinWikiLexer(RegexLexer):
77 """
78 For MoinMoin (and Trac) Wiki markup.
79 """
81 name = 'MoinMoin/Trac Wiki markup'
82 aliases = ['trac-wiki', 'moin']
83 filenames = []
84 mimetypes = ['text/x-trac-wiki']
85 url = 'https://moinmo.in'
86 version_added = '0.7'
88 flags = re.MULTILINE | re.IGNORECASE
90 tokens = {
91 'root': [
92 (r'^#.*$', Comment),
93 (r'(!)(\S+)', bygroups(Keyword, Text)), # Ignore-next
94 # Titles
95 (r'^(=+)([^=]+)(=+)(\s*#.+)?$',
96 bygroups(Generic.Heading, using(this), Generic.Heading, String)),
97 # Literal code blocks, with optional shebang
98 (r'(\{\{\{)(\n#!.+)?', bygroups(Name.Builtin, Name.Namespace), 'codeblock'),
99 (r'(\'\'\'?|\|\||`|__|~~|\^|,,|::)', Comment), # Formatting
100 # Lists
101 (r'^( +)([.*-])( )', bygroups(Text, Name.Builtin, Text)),
102 (r'^( +)([a-z]{1,5}\.)( )', bygroups(Text, Name.Builtin, Text)),
103 # Other Formatting
104 (r'\[\[\w+.*?\]\]', Keyword), # Macro
105 (r'(\[[^\s\]]+)(\s+[^\]]+?)?(\])',
106 bygroups(Keyword, String, Keyword)), # Link
107 (r'^----+$', Keyword), # Horizontal rules
108 (r'[^\n\'\[{!_~^,|]+', Text),
109 (r'\n', Text),
110 (r'.', Text),
111 ],
112 'codeblock': [
113 (r'\}\}\}', Name.Builtin, '#pop'),
114 # these blocks are allowed to be nested in Trac, but not MoinMoin
115 (r'\{\{\{', Text, '#push'),
116 (r'[^{}]+', Comment.Preproc), # slurp boring text
117 (r'.', Comment.Preproc), # allow loose { or }
118 ],
119 }
122class RstLexer(RegexLexer):
123 """
124 For reStructuredText markup.
126 Additional options accepted:
128 `handlecodeblocks`
129 Highlight the contents of ``.. sourcecode:: language``,
130 ``.. code:: language`` and ``.. code-block:: language``
131 directives with a lexer for the given language (default:
132 ``True``).
134 .. versionadded:: 0.8
135 """
136 name = 'reStructuredText'
137 url = 'https://docutils.sourceforge.io/rst.html'
138 aliases = ['restructuredtext', 'rst', 'rest']
139 filenames = ['*.rst', '*.rest']
140 mimetypes = ["text/x-rst", "text/prs.fallenstein.rst"]
141 version_added = '0.7'
142 flags = re.MULTILINE
144 def _handle_sourcecode(self, match):
145 from pygments.lexers import get_lexer_by_name
147 # section header
148 yield match.start(1), Punctuation, match.group(1)
149 yield match.start(2), Text, match.group(2)
150 yield match.start(3), Operator.Word, match.group(3)
151 yield match.start(4), Punctuation, match.group(4)
152 yield match.start(5), Text, match.group(5)
153 yield match.start(6), Keyword, match.group(6)
154 yield match.start(7), Text, match.group(7)
156 # lookup lexer if wanted and existing
157 lexer = None
158 if self.handlecodeblocks:
159 try:
160 lexer = get_lexer_by_name(match.group(6).strip())
161 except ClassNotFound:
162 pass
163 indention = match.group(8)
164 indention_size = len(indention)
165 code = (indention + match.group(9) + match.group(10) + match.group(11))
167 # no lexer for this language. handle it like it was a code block
168 if lexer is None:
169 yield match.start(8), String, code
170 return
172 # highlight the lines with the lexer.
173 ins = []
174 codelines = code.splitlines(True)
175 code = ''
176 for line in codelines:
177 if len(line) > indention_size:
178 ins.append((len(code), [(0, Text, line[:indention_size])]))
179 code += line[indention_size:]
180 else:
181 code += line
182 yield from _shift_indices(
183 do_insertions(ins, lexer.get_tokens_unprocessed(code)),
184 match.start(8))
186 # from docutils.parsers.rst.states
187 closers = '\'")]}>\u2019\u201d\xbb!?'
188 unicode_delimiters = '\u2010\u2011\u2012\u2013\u2014\u00a0'
189 end_string_suffix = (rf'((?=$)|(?=[-/:.,; \n\x00{re.escape(unicode_delimiters)}{re.escape(closers)}]))')
191 tokens = {
192 'root': [
193 # Heading with overline
194 (r'^(=+|-+|`+|:+|\.+|\'+|"+|~+|\^+|_+|\*+|\++|#+)([ \t]*\n)'
195 r'(.+)(\n)(\1)(\n)',
196 bygroups(Generic.Heading, Text, Generic.Heading,
197 Text, Generic.Heading, Text)),
198 # Plain heading
199 (r'^(\S.*)(\n)(={3,}|-{3,}|`{3,}|:{3,}|\.{3,}|\'{3,}|"{3,}|'
200 r'~{3,}|\^{3,}|_{3,}|\*{3,}|\+{3,}|#{3,})(\n)',
201 bygroups(Generic.Heading, Text, Generic.Heading, Text)),
202 # Bulleted lists
203 (r'^(\s*)([-*+])( .+\n(?:\1 .+\n)*)',
204 bygroups(Text, Number, using(this, state='inline'))),
205 # Numbered lists
206 (r'^(\s*)([0-9#ivxlcmIVXLCM]+\.)( .+\n(?:\1 .+\n)*)',
207 bygroups(Text, Number, using(this, state='inline'))),
208 (r'^(\s*)(\(?[0-9#ivxlcmIVXLCM]+\))( .+\n(?:\1 .+\n)*)',
209 bygroups(Text, Number, using(this, state='inline'))),
210 # Numbered, but keep words at BOL from becoming lists
211 (r'^(\s*)([A-Z]+\.)( .+\n(?:\1 .+\n)+)',
212 bygroups(Text, Number, using(this, state='inline'))),
213 (r'^(\s*)(\(?[A-Za-z]+\))( .+\n(?:\1 .+\n)+)',
214 bygroups(Text, Number, using(this, state='inline'))),
215 # Line blocks
216 (r'^(\s*)(\|)( .+\n(?:\| .+\n)*)',
217 bygroups(Text, Operator, using(this, state='inline'))),
218 # Sourcecode directives
219 (r'^( *\.\.)(\s*)((?:source)?code(?:-block)?)(::)([ \t]*)([^\n]+)'
220 r'(\n[ \t]*\n)([ \t]+)(.*)(\n)((?:(?:\8.*)?\n)+)',
221 _handle_sourcecode),
222 # A directive
223 (r'^( *\.\.)(\s*)([\w:.+-]+?)(::)(?:([ \t]*)(.*))',
224 bygroups(Punctuation, Text, Operator.Word, Punctuation, Text,
225 using(this, state='inline'))),
226 # A reference target
227 (r'^( *\.\.)(\s*)(_(?:[^:\\]|\\.)+:)(.*?)$',
228 bygroups(Punctuation, Text, Name.Tag, using(this, state='inline'))),
229 # A footnote/citation target
230 (r'^( *\.\.)(\s*)(\[.+\])(.*?)$',
231 bygroups(Punctuation, Text, Name.Tag, using(this, state='inline'))),
232 # A substitution def
233 (r'^( *\.\.)(\s*)(\|.+\|)(\s*)([\w:-]+?)(::)(?:([ \t]*)(.*))',
234 bygroups(Punctuation, Text, Name.Tag, Text, Operator.Word,
235 Punctuation, Text, using(this, state='inline'))),
236 # Comments
237 (r'^ *\.\..*(\n( +.*\n|\n)+)?', Comment),
238 # Field list marker
239 (r'^( *)(:(?:\\\\|\\:|[^:\n])+:(?=\s))([ \t]*)',
240 bygroups(Text, Name.Class, Text)),
241 # Definition list
242 (r'^(\S.*(?<!::)\n)((?:(?: +.*)\n)+)',
243 bygroups(using(this, state='inline'), using(this, state='inline'))),
244 # Code blocks
245 (r'(::)(\n[ \t]*\n)([ \t]+)(.*)(\n)((?:(?:\3.*)?\n)+)',
246 bygroups(String.Escape, Text, String, String, Text, String)),
247 include('inline'),
248 ],
249 'inline': [
250 (r'\\.', Text), # escape
251 (r'``', String, 'literal'), # code
252 (r'(`.+?)(<.+?>)(`__?)', # reference with inline target
253 bygroups(String, String.Interpol, String)),
254 (r'`.+?`__?', String), # reference
255 (r'(`.+?`)(:[a-zA-Z0-9:.+-]+?:)?',
256 bygroups(Name.Variable, Name.Attribute)), # role
257 (r'(:[a-zA-Z0-9:.+-]+?:)(`.+?`)',
258 bygroups(Name.Attribute, Name.Variable)), # role (content first)
259 (r'\*\*.+?\*\*', Generic.Strong), # Strong emphasis
260 (r'\*.+?\*', Generic.Emph), # Emphasis
261 (r'\[.*?\]_', String), # Footnote or citation
262 (r'<.+?>', Name.Tag), # Hyperlink
263 (r'[^\\\n\[*`:]+', Text),
264 (r'.', Text),
265 ],
266 'literal': [
267 (r'[^`]+', String),
268 (r'``' + end_string_suffix, String, '#pop'),
269 (r'`', String),
270 ]
271 }
273 def __init__(self, **options):
274 self.handlecodeblocks = get_bool_opt(options, 'handlecodeblocks', True)
275 RegexLexer.__init__(self, **options)
277 def analyse_text(text):
278 if text[:2] == '..' and text[2:3] != '.':
279 return 0.3
280 p1 = text.find("\n")
281 p2 = text.find("\n", p1 + 1)
282 if (p2 > -1 and # has two lines
283 p1 * 2 + 1 == p2 and # they are the same length
284 text[p1+1] in '-=' and # the next line both starts and ends with
285 text[p1+1] == text[p2-1]): # ...a sufficiently high header
286 return 0.5
289class TexLexer(RegexLexer):
290 """
291 Lexer for the TeX and LaTeX typesetting languages.
292 """
294 name = 'TeX'
295 aliases = ['tex', 'latex']
296 filenames = ['*.tex', '*.aux', '*.toc']
297 mimetypes = ['text/x-tex', 'text/x-latex']
298 url = 'https://tug.org'
299 version_added = ''
301 tokens = {
302 'general': [
303 (r'%.*?\n', Comment),
304 (r'[{}]', Name.Builtin),
305 (r'[&_^]', Name.Builtin),
306 ],
307 'root': [
308 (r'(\\begin)(\{)((?:display)?math|equation\*?|align\*?)(\})',
309 bygroups(Keyword, Name.Builtin, Name.Builtin, Name.Builtin),
310 'environmentmath'),
311 (r'\\\[', String.Backtick, 'displaymath'),
312 (r'\\\(', String, 'inlinemath'),
313 (r'\$\$', String.Backtick, 'displaymath'),
314 (r'\$', String, 'inlinemath'),
315 (r'\\([a-zA-Z@_:]+|\S?)', Keyword, 'command'),
316 (r'\\$', Keyword),
317 include('general'),
318 (r'[^\\$%&_^{}]+', Text),
319 ],
320 'math': [
321 (r'\\([a-zA-Z]+|\S?)', Name.Variable),
322 include('general'),
323 (r'[0-9]+', Number),
324 (r'[-=!+*/()\[\]]', Operator),
325 (r'[^=!+*/()\[\]\\$%&_^{}0-9-]+', Name.Builtin),
326 ],
327 'inlinemath': [
328 (r'\\\)', String, '#pop'),
329 (r'\$', String, '#pop'),
330 include('math'),
331 ],
332 'displaymath': [
333 (r'\\\]', String, '#pop'),
334 (r'\$\$', String, '#pop'),
335 (r'\$', Name.Builtin),
336 include('math'),
337 ],
338 'environmentmath': [
339 (r'(\\end)(\{)((?:display)?math|equation\*?|align\*?)(\})',
340 bygroups(Keyword, Name.Builtin, Name.Builtin, Name.Builtin), '#pop'),
341 include('math'),
342 ],
343 'command': [
344 (r'\[.*?\]', Name.Attribute),
345 (r'\*', Keyword),
346 default('#pop'),
347 ],
348 }
350 def analyse_text(text):
351 for start in ("\\documentclass", "\\input", "\\documentstyle",
352 "\\relax"):
353 if text[:len(start)] == start:
354 return True
357class GroffLexer(RegexLexer):
358 """
359 Lexer for the (g)roff typesetting language, supporting groff
360 extensions. Mainly useful for highlighting manpage sources.
361 """
363 name = 'Groff'
364 aliases = ['groff', 'nroff', 'man']
365 filenames = ['*.[1-9]', '*.man', '*.1p', '*.3pm']
366 mimetypes = ['application/x-troff', 'text/troff']
367 url = 'https://www.gnu.org/software/groff'
368 version_added = '0.6'
370 tokens = {
371 'root': [
372 (r'(\.)(\w+)', bygroups(Text, Keyword), 'request'),
373 (r'\.', Punctuation, 'request'),
374 # Regular characters, slurp till we find a backslash or newline
375 (r'[^\\\n]+', Text, 'textline'),
376 default('textline'),
377 ],
378 'textline': [
379 include('escapes'),
380 (r'[^\\\n]+', Text),
381 (r'\n', Text, '#pop'),
382 ],
383 'escapes': [
384 # groff has many ways to write escapes.
385 (r'\\"[^\n]*', Comment),
386 (r'\\[fn]\w', String.Escape),
387 (r'\\\(.{2}', String.Escape),
388 (r'\\.\[.*\]', String.Escape),
389 (r'\\.', String.Escape),
390 (r'\\\n', Text, 'request'),
391 ],
392 'request': [
393 (r'\n', Text, '#pop'),
394 include('escapes'),
395 (r'"[^\n"]+"', String.Double),
396 (r'\d+', Number),
397 (r'\S+', String),
398 (r'\s+', Text),
399 ],
400 }
402 def analyse_text(text):
403 if text[:1] != '.':
404 return False
405 if text[:3] == '.\\"':
406 return True
407 if text[:4] == '.TH ':
408 return True
409 if text[1:3].isalnum() and text[3].isspace():
410 return 0.9
413class MozPreprocHashLexer(RegexLexer):
414 """
415 Lexer for Mozilla Preprocessor files (with '#' as the marker).
417 Other data is left untouched.
418 """
419 name = 'mozhashpreproc'
420 aliases = [name]
421 filenames = []
422 mimetypes = []
423 url = 'https://firefox-source-docs.mozilla.org/build/buildsystem/preprocessor.html'
424 version_added = '2.0'
426 tokens = {
427 'root': [
428 (r'^#', Comment.Preproc, ('expr', 'exprstart')),
429 (r'.+', Other),
430 ],
431 'exprstart': [
432 (r'(literal)(.*)', bygroups(Comment.Preproc, Text), '#pop:2'),
433 (words((
434 'define', 'undef', 'if', 'ifdef', 'ifndef', 'else', 'elif',
435 'elifdef', 'elifndef', 'endif', 'expand', 'filter', 'unfilter',
436 'include', 'includesubst', 'error')),
437 Comment.Preproc, '#pop'),
438 ],
439 'expr': [
440 (words(('!', '!=', '==', '&&', '||')), Operator),
441 (r'(defined)(\()', bygroups(Keyword, Punctuation)),
442 (r'\)', Punctuation),
443 (r'[0-9]+', Number.Decimal),
444 (r'__\w+?__', Name.Variable),
445 (r'@\w+?@', Name.Class),
446 (r'\w+', Name),
447 (r'\n', Text, '#pop'),
448 (r'\s+', Text),
449 (r'\S', Punctuation),
450 ],
451 }
454class MozPreprocPercentLexer(MozPreprocHashLexer):
455 """
456 Lexer for Mozilla Preprocessor files (with '%' as the marker).
458 Other data is left untouched.
459 """
460 name = 'mozpercentpreproc'
461 aliases = [name]
462 filenames = []
463 mimetypes = []
464 url = 'https://firefox-source-docs.mozilla.org/build/buildsystem/preprocessor.html'
465 version_added = '2.0'
467 tokens = {
468 'root': [
469 (r'^%', Comment.Preproc, ('expr', 'exprstart')),
470 (r'.+', Other),
471 ],
472 }
475class MozPreprocXulLexer(DelegatingLexer):
476 """
477 Subclass of the `MozPreprocHashLexer` that highlights unlexed data with the
478 `XmlLexer`.
479 """
480 name = "XUL+mozpreproc"
481 aliases = ['xul+mozpreproc']
482 filenames = ['*.xul.in']
483 mimetypes = []
484 url = 'https://firefox-source-docs.mozilla.org/build/buildsystem/preprocessor.html'
485 version_added = '2.0'
487 def __init__(self, **options):
488 super().__init__(XmlLexer, MozPreprocHashLexer, **options)
491class MozPreprocJavascriptLexer(DelegatingLexer):
492 """
493 Subclass of the `MozPreprocHashLexer` that highlights unlexed data with the
494 `JavascriptLexer`.
495 """
496 name = "Javascript+mozpreproc"
497 aliases = ['javascript+mozpreproc']
498 filenames = ['*.js.in']
499 mimetypes = []
500 url = 'https://firefox-source-docs.mozilla.org/build/buildsystem/preprocessor.html'
501 version_added = '2.0'
503 def __init__(self, **options):
504 super().__init__(JavascriptLexer, MozPreprocHashLexer, **options)
507class MozPreprocCssLexer(DelegatingLexer):
508 """
509 Subclass of the `MozPreprocHashLexer` that highlights unlexed data with the
510 `CssLexer`.
511 """
512 name = "CSS+mozpreproc"
513 aliases = ['css+mozpreproc']
514 filenames = ['*.css.in']
515 mimetypes = []
516 url = 'https://firefox-source-docs.mozilla.org/build/buildsystem/preprocessor.html'
517 version_added = '2.0'
519 def __init__(self, **options):
520 super().__init__(CssLexer, MozPreprocPercentLexer, **options)
523class MarkdownLexer(RegexLexer):
524 """
525 For Markdown markup.
526 """
527 name = 'Markdown'
528 url = 'https://daringfireball.net/projects/markdown/'
529 aliases = ['markdown', 'md']
530 filenames = ['*.md', '*.markdown']
531 mimetypes = ["text/x-markdown"]
532 version_added = '2.2'
533 flags = re.MULTILINE
535 def _handle_codeblock(self, match):
536 from pygments.lexers import get_lexer_by_name
538 yield match.start('initial'), String.Backtick, match.group('initial')
539 yield match.start('lang'), String.Backtick, match.group('lang')
540 if match.group('afterlang') is not None:
541 yield match.start('whitespace'), Whitespace, match.group('whitespace')
542 yield match.start('extra'), Text, match.group('extra')
543 yield match.start('newline'), Whitespace, match.group('newline')
545 # lookup lexer if wanted and existing
546 lexer = None
547 if self.handlecodeblocks:
548 try:
549 lexer = get_lexer_by_name(match.group('lang').strip())
550 except ClassNotFound:
551 pass
552 code = match.group('code')
553 # no lexer for this language. handle it like it was a code block
554 if lexer is None:
555 yield match.start('code'), String, code
556 else:
557 yield from _shift_indices(
558 do_insertions([], lexer.get_tokens_unprocessed(code)),
559 match.start('code'))
561 yield match.start('terminator'), String.Backtick, match.group('terminator')
563 tokens = {
564 'root': [
565 # heading with '#' prefix (atx-style)
566 (r'(^#[^#].+)(\n)', bygroups(Generic.Heading, Text)),
567 # subheading with '#' prefix (atx-style)
568 (r'(^#{2,6}[^#].+)(\n)', bygroups(Generic.Subheading, Text)),
569 # heading with '=' underlines (Setext-style)
570 (r'^(.+)(\n)(=+)(\n)', bygroups(Generic.Heading, Text, Generic.Heading, Text)),
571 # subheading with '-' underlines (Setext-style)
572 (r'^(.+)(\n)(-+)(\n)', bygroups(Generic.Subheading, Text, Generic.Subheading, Text)),
573 # task list
574 (r'^(\s*)([*-] )(\[[ xX]\])( .+\n)',
575 bygroups(Whitespace, Keyword, Keyword, using(this, state='inline'))),
576 # bulleted list
577 (r'^(\s*)([*-])(\s)(.+\n)',
578 bygroups(Whitespace, Keyword, Whitespace, using(this, state='inline'))),
579 # numbered list
580 (r'^(\s*)([0-9]+\.)( .+\n)',
581 bygroups(Whitespace, Keyword, using(this, state='inline'))),
582 # quote
583 (r'^(\s*>\s)(.+\n)', bygroups(Keyword, Generic.Emph)),
584 # code block fenced by 3 backticks
585 (r'^(\s*```\n[\w\W]*?^\s*```$\n)', String.Backtick),
586 # code block with language
587 # Some tools include extra stuff after the language name, just
588 # highlight that as text. For example: https://docs.enola.dev/use/execmd
589 (r'''(?x)
590 ^(?P<initial>\s*```)
591 (?P<lang>[\w\-]+)
592 (?P<afterlang>
593 (?P<whitespace>[^\S\n]+)
594 (?P<extra>.*))?
595 (?P<newline>\n)
596 (?P<code>(.|\n)*?)
597 (?P<terminator>^\s*```$\n)
598 ''',
599 _handle_codeblock),
601 include('inline'),
602 ],
603 'inline': [
604 # escape
605 (r'\\.', Text),
606 # inline code
607 (r'([^`]?)(`[^`\n]+`)', bygroups(Text, String.Backtick)),
608 # warning: the following rules eat outer tags.
609 # eg. **foo _bar_ baz** => foo and baz are not recognized as bold
610 # bold-italics fenced by '***'
611 (r'([^\*]?)(\*\*\*[^* \n][^*\n]*\*\*\*)', bygroups(Text, Generic.EmphStrong)),
612 # bold-italics fenced by '___'
613 (r'([^_]?)(___[^_ \n][^_\n]*___)', bygroups(Text, Generic.EmphStrong)),
614 # bold fenced by '**'
615 (r'([^\*]?)(\*\*[^* \n][^*\n]*\*\*)', bygroups(Text, Generic.Strong)),
616 # bold fenced by '__'
617 (r'([^_]?)(__[^_ \n][^_\n]*__)', bygroups(Text, Generic.Strong)),
618 # italics fenced by '*'
619 (r'([^\*]?)(\*[^* \n][^*\n]*\*)', bygroups(Text, Generic.Emph)),
620 # italics fenced by '_'
621 (r'([^_]?)(_[^_ \n][^_\n]*_)', bygroups(Text, Generic.Emph)),
622 # strikethrough
623 (r'([^~]?)(~~[^~ \n][^~\n]*~~)', bygroups(Text, Generic.Deleted)),
624 # mentions and topics (twitter and github stuff)
625 (r'[@#][\w/:-]+', Name.Entity),
626 # (image?) links eg: 
627 (r'(!?\[)([^]]+)(\])(\()([^)]+)(\))',
628 bygroups(Text, Name.Tag, Text, Text, Name.Attribute, Text)),
629 # reference-style links, e.g.:
630 # [an example][id]
631 # [id]: http://example.com/
632 (r'(\[)([^]]+)(\])(\[)([^]]*)(\])',
633 bygroups(Text, Name.Tag, Text, Text, Name.Label, Text)),
634 (r'^(\s*\[)([^]]*)(\]:\s*)(.+)',
635 bygroups(Text, Name.Label, Text, Name.Attribute)),
637 # general text, must come last!
638 (r'[^\\\s]+', Text),
639 (r'.', Text),
640 ],
641 }
643 def __init__(self, **options):
644 self.handlecodeblocks = get_bool_opt(options, 'handlecodeblocks', True)
645 RegexLexer.__init__(self, **options)
647class OrgLexer(RegexLexer):
648 """
649 For Org Mode markup.
650 """
651 name = 'Org Mode'
652 url = 'https://orgmode.org'
653 aliases = ['org', 'orgmode', 'org-mode']
654 filenames = ['*.org']
655 mimetypes = ["text/org"]
656 version_added = '2.18'
658 def _inline(start, end):
659 return rf'(?<!\w){start}(.|\n(?!\n))+?{end}(?!\w)'
661 tokens = {
662 'root': [
663 (r'^# .*', Comment.Single),
665 # Headings
666 (r'^(\* )(COMMENT)( .*)',
667 bygroups(Generic.Heading, Comment.Preproc, Generic.Heading)),
668 (r'^(\*\*+ )(COMMENT)( .*)',
669 bygroups(Generic.Subheading, Comment.Preproc, Generic.Subheading)),
670 (r'^(\* )(DONE)( .*)',
671 bygroups(Generic.Heading, Generic.Deleted, Generic.Heading)),
672 (r'^(\*\*+ )(DONE)( .*)',
673 bygroups(Generic.Subheading, Generic.Deleted, Generic.Subheading)),
674 (r'^(\* )(TODO)( .*)',
675 bygroups(Generic.Heading, Generic.Error, Generic.Heading)),
676 (r'^(\*\*+ )(TODO)( .*)',
677 bygroups(Generic.Subheading, Generic.Error, Generic.Subheading)),
679 (r'^(\* .+?)( :[a-zA-Z0-9_@:]+:)?$', bygroups(Generic.Heading, Generic.Emph)),
680 (r'^(\*\*+ .+?)( :[a-zA-Z0-9_@:]+:)?$', bygroups(Generic.Subheading, Generic.Emph)),
682 # Unordered lists items, including TODO items and description items
683 (r'^(?:( *)([+-] )|( +)(\* ))(\[[ X-]\])?(.+ ::)?',
684 bygroups(Whitespace, Keyword, Whitespace, Keyword, Generic.Prompt, Name.Label)),
686 # Ordered list items
687 (r'^( *)([0-9]+[.)])( \[@[0-9]+\])?', bygroups(Whitespace, Keyword, Generic.Emph)),
689 # Dynamic blocks
690 (r'(?i)^( *#\+begin: *)((?:.|\n)*?)(^ *#\+end: *$)',
691 bygroups(Operator.Word, using(this), Operator.Word)),
693 # Comment blocks
694 (r'(?i)^( *#\+begin_comment *\n)((?:.|\n)*?)(^ *#\+end_comment *$)',
695 bygroups(Operator.Word, Comment.Multiline, Operator.Word)),
697 # Source code blocks
698 # TODO: language-dependent syntax highlighting (see Markdown lexer)
699 (r'(?i)^( *#\+begin_src .*)((?:.|\n)*?)(^ *#\+end_src *$)',
700 bygroups(Operator.Word, Text, Operator.Word)),
702 # Other blocks
703 (r'(?i)^( *#\+begin_\w+)( *\n)((?:.|\n)*?)(^ *#\+end_\w+)( *$)',
704 bygroups(Operator.Word, Whitespace, Text, Operator.Word, Whitespace)),
706 # Keywords
707 (r'^(#\+\w+:)(.*)$', bygroups(Name.Namespace, Text)),
709 # Properties and drawers
710 (r'(?i)^( *:\w+: *\n)((?:.|\n)*?)(^ *:end: *$)',
711 bygroups(Name.Decorator, Comment.Special, Name.Decorator)),
713 # Line break operator
714 (r'\\\\$', Operator),
716 (r'^\s*CLOSED:\s+', Generic.Deleted, 'dateline'),
717 (r'^\s*(?:DEADLINE:|SCHEDULED:)\s+', Generic.Error, 'dateline'),
719 # Bold
720 (_inline(r'\*', r'\*+'), Generic.Strong),
721 # Italic
722 (_inline(r'/', r'/'), Generic.Emph),
723 # Verbatim
724 (_inline(r'=', r'='), String), # TODO token
725 # Code
726 (_inline(r'~', r'~'), String),
727 # Strikethrough
728 (_inline(r'\+', r'\+'), Generic.Deleted),
729 # Underline
730 (_inline(r'_', r'_+'), Generic.EmphStrong),
732 # Dates
733 (r'<.+?>', Literal.Date),
734 # Macros
735 (r'\{\{\{.+?\}\}\}', Comment.Preproc),
736 # Footnotes
737 (r'(?<!\[)\[fn:.+?\]', Name.Tag),
738 # Links
739 (r'(?s)(\[\[)(.*?)(\]\[)(.*?)(\]\])',
740 bygroups(Punctuation, Name.Attribute, Punctuation, Name.Tag, Punctuation)),
741 (r'(?s)(\[\[)(.+?)(\]\])', bygroups(Punctuation, Name.Attribute, Punctuation)),
742 (r'(<<)(.+?)(>>)', bygroups(Punctuation, Name.Attribute, Punctuation)),
744 # Tables
745 (r'^( *)(\|[ -].*?[ -]\|)$', bygroups(Whitespace, String)),
747 # Any other text
748 (r'[^#*+\-0-9:\\/=~_<{\[|\n]+', Text),
749 (r'[#*+\-0-9:\\/=~_<{\[|\n]', Text),
750 ],
751 'dateline': [
752 (r'\s*CLOSED:\s+', Generic.Deleted),
753 (r'\s*(?:DEADLINE:|SCHEDULED:)\s+', Generic.Error),
754 (r'\[.+?\]', Literal.Date),
755 (r'<[^>]+?>', Literal.Date),
756 (r'(\s*)$', Text, '#pop'),
757 (r'.', Text),
758 ],
759 }
761class TiddlyWiki5Lexer(RegexLexer):
762 """
763 For TiddlyWiki5 markup.
764 """
765 name = 'tiddler'
766 url = 'https://tiddlywiki.com/#TiddlerFiles'
767 aliases = ['tid']
768 filenames = ['*.tid']
769 mimetypes = ["text/vnd.tiddlywiki"]
770 version_added = '2.7'
771 flags = re.MULTILINE
773 def _handle_codeblock(self, match):
774 """
775 match args: 1:backticks, 2:lang_name, 3:newline, 4:code, 5:backticks
776 """
777 from pygments.lexers import get_lexer_by_name
779 # section header
780 yield match.start(1), String, match.group(1)
781 yield match.start(2), String, match.group(2)
782 yield match.start(3), Text, match.group(3)
784 # lookup lexer if wanted and existing
785 lexer = None
786 if self.handlecodeblocks:
787 try:
788 lexer = get_lexer_by_name(match.group(2).strip())
789 except ClassNotFound:
790 pass
791 code = match.group(4)
793 # no lexer for this language. handle it like it was a code block
794 if lexer is None:
795 yield match.start(4), String, code
796 return
798 yield from _shift_indices(
799 do_insertions([], lexer.get_tokens_unprocessed(code)),
800 match.start(4))
802 yield match.start(5), String, match.group(5)
804 def _handle_cssblock(self, match):
805 """
806 match args: 1:style tag 2:newline, 3:code, 4:closing style tag
807 """
808 from pygments.lexers import get_lexer_by_name
810 # section header
811 yield match.start(1), String, match.group(1)
812 yield match.start(2), String, match.group(2)
814 lexer = None
815 if self.handlecodeblocks:
816 try:
817 lexer = get_lexer_by_name('css')
818 except ClassNotFound:
819 pass
820 code = match.group(3)
822 # no lexer for this language. handle it like it was a code block
823 if lexer is None:
824 yield match.start(3), String, code
825 return
827 yield from _shift_indices(
828 do_insertions([], lexer.get_tokens_unprocessed(code)),
829 match.start(3))
831 yield match.start(4), String, match.group(4)
833 tokens = {
834 'root': [
835 # title in metadata section
836 (r'^(title)(:\s)(.+\n)', bygroups(Keyword, Text, Generic.Heading)),
837 # headings
838 (r'^(!)([^!].+\n)', bygroups(Generic.Heading, Text)),
839 (r'^(!{2,6})(.+\n)', bygroups(Generic.Subheading, Text)),
840 # bulleted or numbered lists or single-line block quotes
841 # (can be mixed)
842 (r'^(\s*)([*#>]+)(\s*)(.+\n)',
843 bygroups(Text, Keyword, Text, using(this, state='inline'))),
844 # multi-line block quotes
845 (r'^(<<<.*\n)([\w\W]*?)(^<<<.*$)', bygroups(String, Text, String)),
846 # table header
847 (r'^(\|.*?\|h)$', bygroups(Generic.Strong)),
848 # table footer or caption
849 (r'^(\|.*?\|[cf])$', bygroups(Generic.Emph)),
850 # table class
851 (r'^(\|.*?\|k)$', bygroups(Name.Tag)),
852 # definitions
853 (r'^(;.*)$', bygroups(Generic.Strong)),
854 # text block
855 (r'^(```\n)([\w\W]*?)(^```$)', bygroups(String, Text, String)),
856 # code block with language
857 (r'^(```)(\w+)(\n)([\w\W]*?)(^```$)', _handle_codeblock),
858 # CSS style block
859 (r'^(<style>)(\n)([\w\W]*?)(^</style>$)', _handle_cssblock),
861 include('keywords'),
862 include('inline'),
863 ],
864 'keywords': [
865 (words((
866 '\\define', '\\end', 'caption', 'created', 'modified', 'tags',
867 'title', 'type'), prefix=r'^', suffix=r'\b'),
868 Keyword),
869 ],
870 'inline': [
871 # escape
872 (r'\\.', Text),
873 # created or modified date
874 (r'\d{17}', Number.Integer),
875 # italics
876 (r'(\s)(//[^/]+//)((?=\W|\n))',
877 bygroups(Text, Generic.Emph, Text)),
878 # superscript
879 (r'(\s)(\^\^[^\^]+\^\^)', bygroups(Text, Generic.Emph)),
880 # subscript
881 (r'(\s)(,,[^,]+,,)', bygroups(Text, Generic.Emph)),
882 # underscore
883 (r'(\s)(__[^_]+__)', bygroups(Text, Generic.Strong)),
884 # bold
885 (r"(\s)(''[^']+'')((?=\W|\n))",
886 bygroups(Text, Generic.Strong, Text)),
887 # strikethrough
888 (r'(\s)(~~[^~]+~~)((?=\W|\n))',
889 bygroups(Text, Generic.Deleted, Text)),
890 # TiddlyWiki variables
891 (r'<<[^>]+>>', Name.Tag),
892 (r'\$\$[^$]+\$\$', Name.Tag),
893 (r'\$\([^)]+\)\$', Name.Tag),
894 # TiddlyWiki style or class
895 (r'^@@.*$', Name.Tag),
896 # HTML tags
897 (r'</?[^>]+>', Name.Tag),
898 # inline code
899 (r'`[^`]+`', String.Backtick),
900 # HTML escaped symbols
901 (r'&\S*?;', String.Regex),
902 # Wiki links
903 (r'(\[{2})([^]\|]+)(\]{2})', bygroups(Text, Name.Tag, Text)),
904 # External links
905 (r'(\[{2})([^]\|]+)(\|)([^]\|]+)(\]{2})',
906 bygroups(Text, Name.Tag, Text, Name.Attribute, Text)),
907 # Transclusion
908 (r'(\{{2})([^}]+)(\}{2})', bygroups(Text, Name.Tag, Text)),
909 # URLs
910 (r'(\b.?.?tps?://[^\s"]+)', bygroups(Name.Attribute)),
912 # general text, must come last!
913 (r'[\w]+', Text),
914 (r'.', Text)
915 ],
916 }
918 def __init__(self, **options):
919 self.handlecodeblocks = get_bool_opt(options, 'handlecodeblocks', True)
920 RegexLexer.__init__(self, **options)
923class WikitextLexer(RegexLexer):
924 """
925 For MediaWiki Wikitext.
927 Parsing Wikitext is tricky, and results vary between different MediaWiki
928 installations, so we only highlight common syntaxes (built-in or from
929 popular extensions), and also assume templates produce no unbalanced
930 syntaxes.
931 """
932 name = 'Wikitext'
933 url = 'https://www.mediawiki.org/wiki/Wikitext'
934 aliases = ['wikitext', 'mediawiki']
935 filenames = []
936 mimetypes = ['text/x-wiki']
937 version_added = '2.15'
938 flags = re.MULTILINE
940 def nowiki_tag_rules(tag_name):
941 return [
942 (rf'(?i)(</)({tag_name})(\s*)(>)', bygroups(Punctuation,
943 Name.Tag, Whitespace, Punctuation), '#pop'),
944 include('entity'),
945 include('text'),
946 ]
948 def plaintext_tag_rules(tag_name):
949 return [
950 (rf'(?si)(.*?)(</)({tag_name})(\s*)(>)', bygroups(Text,
951 Punctuation, Name.Tag, Whitespace, Punctuation), '#pop'),
952 ]
954 def delegate_tag_rules(tag_name, lexer, **lexer_kwargs):
955 return [
956 (rf'(?i)(</)({tag_name})(\s*)(>)', bygroups(Punctuation,
957 Name.Tag, Whitespace, Punctuation), '#pop'),
958 (rf'(?si).+?(?=</{tag_name}\s*>)', using(lexer, **lexer_kwargs)),
959 ]
961 def text_rules(token):
962 return [
963 (r'\w+', token),
964 (r'[^\S\n]+', token),
965 (r'(?s).', token),
966 ]
968 def handle_syntaxhighlight(self, match, ctx):
969 from pygments.lexers import get_lexer_by_name
971 attr_content = match.group()
972 start = 0
973 index = 0
974 while True:
975 index = attr_content.find('>', start)
976 # Exclude comment end (-->)
977 if attr_content[index-2:index] != '--':
978 break
979 start = index + 1
981 if index == -1:
982 # No tag end
983 yield from self.get_tokens_unprocessed(attr_content, stack=['root', 'attr'])
984 return
985 attr = attr_content[:index]
986 yield from self.get_tokens_unprocessed(attr, stack=['root', 'attr'])
987 yield match.start(3) + index, Punctuation, '>'
989 lexer = None
990 content = attr_content[index+1:]
991 lang_match = re.findall(r'\blang=("|\'|)(\w+)(\1)', attr)
993 if len(lang_match) >= 1:
994 # Pick the last match in case of multiple matches
995 lang = lang_match[-1][1]
996 try:
997 lexer = get_lexer_by_name(lang)
998 except ClassNotFound:
999 pass
1001 if lexer is None:
1002 yield match.start() + index + 1, Text, content
1003 else:
1004 yield from lexer.get_tokens_unprocessed(content)
1006 def handle_score(self, match, ctx):
1007 attr_content = match.group()
1008 start = 0
1009 index = 0
1010 while True:
1011 index = attr_content.find('>', start)
1012 # Exclude comment end (-->)
1013 if attr_content[index-2:index] != '--':
1014 break
1015 start = index + 1
1017 if index == -1:
1018 # No tag end
1019 yield from self.get_tokens_unprocessed(attr_content, stack=['root', 'attr'])
1020 return
1021 attr = attr_content[:index]
1022 content = attr_content[index+1:]
1023 yield from self.get_tokens_unprocessed(attr, stack=['root', 'attr'])
1024 yield match.start(3) + index, Punctuation, '>'
1026 lang_match = re.findall(r'\blang=("|\'|)(\w+)(\1)', attr)
1027 # Pick the last match in case of multiple matches
1028 lang = lang_match[-1][1] if len(lang_match) >= 1 else 'lilypond'
1030 if lang == 'lilypond': # Case sensitive
1031 yield from LilyPondLexer().get_tokens_unprocessed(content)
1032 else: # ABC
1033 # FIXME: Use ABC lexer in the future
1034 yield match.start() + index + 1, Text, content
1036 # a-z removed to prevent linter from complaining, REMEMBER to use (?i)
1037 title_char = r' %!"$&\'()*,\-./0-9:;=?@A-Z\\\^_`~+\u0080-\uFFFF'
1038 nbsp_char = r'(?:\t| |&\#0*160;|&\#[Xx]0*[Aa]0;|[ \xA0\u1680\u2000-\u200A\u202F\u205F\u3000])'
1039 link_address = r'(?:[0-9.]+|\[[0-9a-f:.]+\]|[^\x00-\x20"<>\[\]\x7F\xA0\u1680\u2000-\u200A\u202F\u205F\u3000\uFFFD])'
1040 link_char_class = r'[^\x00-\x20"<>\[\]\x7F\xA0\u1680\u2000-\u200A\u202F\u205F\u3000\uFFFD]'
1041 double_slashes_i = {
1042 '__FORCETOC__', '__NOCONTENTCONVERT__', '__NOCC__', '__NOEDITSECTION__', '__NOGALLERY__',
1043 '__NOTITLECONVERT__', '__NOTC__', '__NOTOC__', '__TOC__',
1044 }
1045 double_slashes = {
1046 '__EXPECTUNUSEDCATEGORY__', '__HIDDENCAT__', '__INDEX__', '__NEWSECTIONLINK__',
1047 '__NOINDEX__', '__NONEWSECTIONLINK__', '__STATICREDIRECT__', '__NOGLOBAL__',
1048 '__DISAMBIG__', '__EXPECTED_UNCONNECTED_PAGE__',
1049 }
1050 protocols = {
1051 'bitcoin:', 'ftp://', 'ftps://', 'geo:', 'git://', 'gopher://', 'http://', 'https://',
1052 'irc://', 'ircs://', 'magnet:', 'mailto:', 'mms://', 'news:', 'nntp://', 'redis://',
1053 'sftp://', 'sip:', 'sips:', 'sms:', 'ssh://', 'svn://', 'tel:', 'telnet://', 'urn:',
1054 'worldwind://', 'xmpp:', '//',
1055 }
1056 non_relative_protocols = protocols - {'//'}
1057 html_tags = {
1058 'abbr', 'b', 'bdi', 'bdo', 'big', 'blockquote', 'br', 'caption', 'center', 'cite', 'code',
1059 'data', 'dd', 'del', 'dfn', 'div', 'dl', 'dt', 'em', 'font', 'h1', 'h2', 'h3', 'h4', 'h5',
1060 'h6', 'hr', 'i', 'ins', 'kbd', 'li', 'link', 'mark', 'meta', 'ol', 'p', 'q', 'rb', 'rp',
1061 'rt', 'rtc', 'ruby', 's', 'samp', 'small', 'span', 'strike', 'strong', 'sub', 'sup',
1062 'table', 'td', 'th', 'time', 'tr', 'tt', 'u', 'ul', 'var', 'wbr',
1063 }
1064 parser_tags = {
1065 'graph', 'charinsert', 'rss', 'chem', 'categorytree', 'nowiki', 'inputbox', 'math',
1066 'hiero', 'score', 'pre', 'ref', 'translate', 'imagemap', 'templatestyles', 'languages',
1067 'noinclude', 'mapframe', 'section', 'poem', 'syntaxhighlight', 'includeonly', 'tvar',
1068 'onlyinclude', 'templatedata', 'langconvert', 'timeline', 'dynamicpagelist', 'gallery',
1069 'maplink', 'ce', 'references',
1070 }
1071 variant_langs = {
1072 # ZhConverter.php
1073 'zh', 'zh-hans', 'zh-hant', 'zh-cn', 'zh-hk', 'zh-mo', 'zh-my', 'zh-sg', 'zh-tw',
1074 # WuuConverter.php
1075 'wuu', 'wuu-hans', 'wuu-hant',
1076 # UzConverter.php
1077 'uz', 'uz-latn', 'uz-cyrl',
1078 # TlyConverter.php
1079 'tly', 'tly-cyrl',
1080 # TgConverter.php
1081 'tg', 'tg-latn',
1082 # SrConverter.php
1083 'sr', 'sr-ec', 'sr-el',
1084 # ShiConverter.php
1085 'shi', 'shi-tfng', 'shi-latn',
1086 # ShConverter.php
1087 'sh-latn', 'sh-cyrl',
1088 # KuConverter.php
1089 'ku', 'ku-arab', 'ku-latn',
1090 # IuConverter.php
1091 'iu', 'ike-cans', 'ike-latn',
1092 # GanConverter.php
1093 'gan', 'gan-hans', 'gan-hant',
1094 # EnConverter.php
1095 'en', 'en-x-piglatin',
1096 # CrhConverter.php
1097 'crh', 'crh-cyrl', 'crh-latn',
1098 # BanConverter.php
1099 'ban', 'ban-bali', 'ban-x-dharma', 'ban-x-palmleaf', 'ban-x-pku',
1100 }
1101 magic_vars_i = {
1102 'ARTICLEPATH', 'INT', 'PAGEID', 'SCRIPTPATH', 'SERVER', 'SERVERNAME', 'STYLEPATH',
1103 }
1104 magic_vars = {
1105 '!', '=', 'BASEPAGENAME', 'BASEPAGENAMEE', 'CASCADINGSOURCES', 'CONTENTLANGUAGE',
1106 'CONTENTLANG', 'CURRENTDAY', 'CURRENTDAY2', 'CURRENTDAYNAME', 'CURRENTDOW', 'CURRENTHOUR',
1107 'CURRENTMONTH', 'CURRENTMONTH2', 'CURRENTMONTH1', 'CURRENTMONTHABBREV', 'CURRENTMONTHNAME',
1108 'CURRENTMONTHNAMEGEN', 'CURRENTTIME', 'CURRENTTIMESTAMP', 'CURRENTVERSION', 'CURRENTWEEK',
1109 'CURRENTYEAR', 'DIRECTIONMARK', 'DIRMARK', 'FULLPAGENAME', 'FULLPAGENAMEE', 'LOCALDAY',
1110 'LOCALDAY2', 'LOCALDAYNAME', 'LOCALDOW', 'LOCALHOUR', 'LOCALMONTH', 'LOCALMONTH2',
1111 'LOCALMONTH1', 'LOCALMONTHABBREV', 'LOCALMONTHNAME', 'LOCALMONTHNAMEGEN', 'LOCALTIME',
1112 'LOCALTIMESTAMP', 'LOCALWEEK', 'LOCALYEAR', 'NAMESPACE', 'NAMESPACEE', 'NAMESPACENUMBER',
1113 'NUMBEROFACTIVEUSERS', 'NUMBEROFADMINS', 'NUMBEROFARTICLES', 'NUMBEROFEDITS',
1114 'NUMBEROFFILES', 'NUMBEROFPAGES', 'NUMBEROFUSERS', 'PAGELANGUAGE', 'PAGENAME', 'PAGENAMEE',
1115 'REVISIONDAY', 'REVISIONDAY2', 'REVISIONID', 'REVISIONMONTH', 'REVISIONMONTH1',
1116 'REVISIONSIZE', 'REVISIONTIMESTAMP', 'REVISIONUSER', 'REVISIONYEAR', 'ROOTPAGENAME',
1117 'ROOTPAGENAMEE', 'SITENAME', 'SUBJECTPAGENAME', 'ARTICLEPAGENAME', 'SUBJECTPAGENAMEE',
1118 'ARTICLEPAGENAMEE', 'SUBJECTSPACE', 'ARTICLESPACE', 'SUBJECTSPACEE', 'ARTICLESPACEE',
1119 'SUBPAGENAME', 'SUBPAGENAMEE', 'TALKPAGENAME', 'TALKPAGENAMEE', 'TALKSPACE', 'TALKSPACEE',
1120 }
1121 parser_functions_i = {
1122 'ANCHORENCODE', 'BIDI', 'CANONICALURL', 'CANONICALURLE', 'FILEPATH', 'FORMATNUM',
1123 'FULLURL', 'FULLURLE', 'GENDER', 'GRAMMAR', 'INT', r'\#LANGUAGE', 'LC', 'LCFIRST', 'LOCALURL',
1124 'LOCALURLE', 'NS', 'NSE', 'PADLEFT', 'PADRIGHT', 'PAGEID', 'PLURAL', 'UC', 'UCFIRST',
1125 'URLENCODE',
1126 }
1127 parser_functions = {
1128 'BASEPAGENAME', 'BASEPAGENAMEE', 'CASCADINGSOURCES', 'DEFAULTSORT', 'DEFAULTSORTKEY',
1129 'DEFAULTCATEGORYSORT', 'FULLPAGENAME', 'FULLPAGENAMEE', 'NAMESPACE', 'NAMESPACEE',
1130 'NAMESPACENUMBER', 'NUMBERINGROUP', 'NUMINGROUP', 'NUMBEROFACTIVEUSERS', 'NUMBEROFADMINS',
1131 'NUMBEROFARTICLES', 'NUMBEROFEDITS', 'NUMBEROFFILES', 'NUMBEROFPAGES', 'NUMBEROFUSERS',
1132 'PAGENAME', 'PAGENAMEE', 'PAGESINCATEGORY', 'PAGESINCAT', 'PAGESIZE', 'PROTECTIONEXPIRY',
1133 'PROTECTIONLEVEL', 'REVISIONDAY', 'REVISIONDAY2', 'REVISIONID', 'REVISIONMONTH',
1134 'REVISIONMONTH1', 'REVISIONTIMESTAMP', 'REVISIONUSER', 'REVISIONYEAR', 'ROOTPAGENAME',
1135 'ROOTPAGENAMEE', 'SUBJECTPAGENAME', 'ARTICLEPAGENAME', 'SUBJECTPAGENAMEE',
1136 'ARTICLEPAGENAMEE', 'SUBJECTSPACE', 'ARTICLESPACE', 'SUBJECTSPACEE', 'ARTICLESPACEE',
1137 'SUBPAGENAME', 'SUBPAGENAMEE', 'TALKPAGENAME', 'TALKPAGENAMEE', 'TALKSPACE', 'TALKSPACEE',
1138 'INT', 'DISPLAYTITLE', 'PAGESINNAMESPACE', 'PAGESINNS',
1139 }
1141 tokens = {
1142 'root': [
1143 # Redirects
1144 (r"""(?xi)
1145 (\A\s*?)(\#REDIRECT:?) # may contain a colon
1146 (\s+)(\[\[) (?=[^\]\n]* \]\]$)
1147 """,
1148 bygroups(Whitespace, Keyword, Whitespace, Punctuation), 'redirect-inner'),
1149 # Subheadings
1150 (r'^(={2,6})(.+?)(\1)(\s*$\n)',
1151 bygroups(Generic.Subheading, Generic.Subheading, Generic.Subheading, Whitespace)),
1152 # Headings
1153 (r'^(=.+?=)(\s*$\n)',
1154 bygroups(Generic.Heading, Whitespace)),
1155 # Double-slashed magic words
1156 (words(double_slashes_i, prefix=r'(?i)'), Name.Function.Magic),
1157 (words(double_slashes), Name.Function.Magic),
1158 # Raw URLs
1159 (r'(?i)\b(?:{}){}{}*'.format('|'.join(protocols),
1160 link_address, link_char_class), Name.Label),
1161 # Magic links
1162 (rf'\b(?:RFC|PMID){nbsp_char}+[0-9]+\b',
1163 Name.Function.Magic),
1164 (r"""(?x)
1165 \bISBN {nbsp_char}
1166 (?: 97[89] {nbsp_dash}? )?
1167 (?: [0-9] {nbsp_dash}? ){{9}} # escape format()
1168 [0-9Xx]\b
1169 """.format(nbsp_char=nbsp_char, nbsp_dash=f'(?:-|{nbsp_char})'), Name.Function.Magic),
1170 include('list'),
1171 include('inline'),
1172 include('text'),
1173 ],
1174 'redirect-inner': [
1175 (r'(\]\])(\s*?\n)', bygroups(Punctuation, Whitespace), '#pop'),
1176 (r'(\#)([^#]*?)', bygroups(Punctuation, Name.Label)),
1177 (rf'(?i)[{title_char}]+', Name.Tag),
1178 ],
1179 'list': [
1180 # Description lists
1181 (r'^;', Keyword, 'dt'),
1182 # Ordered lists, unordered lists and indents
1183 (r'^[#:*]+', Keyword),
1184 # Horizontal rules
1185 (r'^-{4,}', Keyword),
1186 ],
1187 'inline': [
1188 # Signatures
1189 (r'~{3,5}', Keyword),
1190 # Entities
1191 include('entity'),
1192 # Bold & italic
1193 (r"('')(''')(?!')", bygroups(Generic.Emph,
1194 Generic.EmphStrong), 'inline-italic-bold'),
1195 (r"'''(?!')", Generic.Strong, 'inline-bold'),
1196 (r"''(?!')", Generic.Emph, 'inline-italic'),
1197 # Comments & parameters & templates
1198 include('replaceable'),
1199 # Media links
1200 (
1201 r"""(?xi)
1202 (\[\[)
1203 (File|Image) (:)
1204 ((?: [{}] | \{{{{2,3}}[^{{}}]*?\}}{{2,3}} | <!--[\s\S]*?--> )*)
1205 (?: (\#) ([{}]*?) )?
1206 """.format(title_char, f'{title_char}#'),
1207 bygroups(Punctuation, Name.Namespace, Punctuation,
1208 using(this, state=['wikilink-name']), Punctuation, Name.Label),
1209 'medialink-inner'
1210 ),
1211 # Wikilinks
1212 (
1213 r"""(?xi)
1214 (\[\[)(?!{}) # Should not contain URLs
1215 (?: ([{}]*) (:))?
1216 ((?: [{}] | \{{{{2,3}}[^{{}}]*?\}}{{2,3}} | <!--[\s\S]*?--> )*?)
1217 (?: (\#) ([{}]*?) )?
1218 (\]\])
1219 """.format('|'.join(protocols), title_char.replace('/', ''),
1220 title_char, f'{title_char}#'),
1221 bygroups(Punctuation, Name.Namespace, Punctuation,
1222 using(this, state=['wikilink-name']), Punctuation, Name.Label, Punctuation)
1223 ),
1224 (
1225 r"""(?xi)
1226 (\[\[)(?!{})
1227 (?: ([{}]*) (:))?
1228 ((?: [{}] | \{{{{2,3}}[^{{}}]*?\}}{{2,3}} | <!--[\s\S]*?--> )*?)
1229 (?: (\#) ([{}]*?) )?
1230 (\|)
1231 """.format('|'.join(protocols), title_char.replace('/', ''),
1232 title_char, f'{title_char}#'),
1233 bygroups(Punctuation, Name.Namespace, Punctuation,
1234 using(this, state=['wikilink-name']), Punctuation, Name.Label, Punctuation),
1235 'wikilink-inner'
1236 ),
1237 # External links
1238 (
1239 r"""(?xi)
1240 (\[)
1241 ((?:{}) {} {}*)
1242 (\s*)
1243 """.format('|'.join(protocols), link_address, link_char_class),
1244 bygroups(Punctuation, Name.Label, Whitespace),
1245 'extlink-inner'
1246 ),
1247 # Tables
1248 (r'^(:*)(\s*?)(\{\|)([^\n]*)$', bygroups(Keyword,
1249 Whitespace, Punctuation, using(this, state=['root', 'attr'])), 'table'),
1250 # HTML tags
1251 (r'(?i)(<)({})\b'.format('|'.join(html_tags)),
1252 bygroups(Punctuation, Name.Tag), 'tag-inner-ordinary'),
1253 (r'(?i)(</)({})\b(\s*)(>)'.format('|'.join(html_tags)),
1254 bygroups(Punctuation, Name.Tag, Whitespace, Punctuation)),
1255 # <nowiki>
1256 (r'(?i)(<)(nowiki)\b', bygroups(Punctuation,
1257 Name.Tag), ('tag-nowiki', 'tag-inner')),
1258 # <pre>
1259 (r'(?i)(<)(pre)\b', bygroups(Punctuation,
1260 Name.Tag), ('tag-pre', 'tag-inner')),
1261 # <categorytree>
1262 (r'(?i)(<)(categorytree)\b', bygroups(
1263 Punctuation, Name.Tag), ('tag-categorytree', 'tag-inner')),
1264 # <hiero>
1265 (r'(?i)(<)(hiero)\b', bygroups(Punctuation,
1266 Name.Tag), ('tag-hiero', 'tag-inner')),
1267 # <math>
1268 (r'(?i)(<)(math)\b', bygroups(Punctuation,
1269 Name.Tag), ('tag-math', 'tag-inner')),
1270 # <chem>
1271 (r'(?i)(<)(chem)\b', bygroups(Punctuation,
1272 Name.Tag), ('tag-chem', 'tag-inner')),
1273 # <ce>
1274 (r'(?i)(<)(ce)\b', bygroups(Punctuation,
1275 Name.Tag), ('tag-ce', 'tag-inner')),
1276 # <charinsert>
1277 (r'(?i)(<)(charinsert)\b', bygroups(
1278 Punctuation, Name.Tag), ('tag-charinsert', 'tag-inner')),
1279 # <templatedata>
1280 (r'(?i)(<)(templatedata)\b', bygroups(
1281 Punctuation, Name.Tag), ('tag-templatedata', 'tag-inner')),
1282 # <gallery>
1283 (r'(?i)(<)(gallery)\b', bygroups(
1284 Punctuation, Name.Tag), ('tag-gallery', 'tag-inner')),
1285 # <graph>
1286 (r'(?i)(<)(gallery)\b', bygroups(
1287 Punctuation, Name.Tag), ('tag-graph', 'tag-inner')),
1288 # <dynamicpagelist>
1289 (r'(?i)(<)(dynamicpagelist)\b', bygroups(
1290 Punctuation, Name.Tag), ('tag-dynamicpagelist', 'tag-inner')),
1291 # <inputbox>
1292 (r'(?i)(<)(inputbox)\b', bygroups(
1293 Punctuation, Name.Tag), ('tag-inputbox', 'tag-inner')),
1294 # <rss>
1295 (r'(?i)(<)(rss)\b', bygroups(
1296 Punctuation, Name.Tag), ('tag-rss', 'tag-inner')),
1297 # <imagemap>
1298 (r'(?i)(<)(imagemap)\b', bygroups(
1299 Punctuation, Name.Tag), ('tag-imagemap', 'tag-inner')),
1300 # <syntaxhighlight>
1301 (r'(?i)(</)(syntaxhighlight)\b(\s*)(>)',
1302 bygroups(Punctuation, Name.Tag, Whitespace, Punctuation)),
1303 (r'(?si)(<)(syntaxhighlight)\b([^>]*?(?<!/)>.*?)(?=</\2\s*>)',
1304 bygroups(Punctuation, Name.Tag, handle_syntaxhighlight)),
1305 # <syntaxhighlight>: Fallback case for self-closing tags
1306 (r'(?i)(<)(syntaxhighlight)\b(\s*?)((?:[^>]|-->)*?)(/\s*?(?<!--)>)', bygroups(
1307 Punctuation, Name.Tag, Whitespace, using(this, state=['root', 'attr']), Punctuation)),
1308 # <source>
1309 (r'(?i)(</)(source)\b(\s*)(>)',
1310 bygroups(Punctuation, Name.Tag, Whitespace, Punctuation)),
1311 (r'(?si)(<)(source)\b([^>]*?(?<!/)>.*?)(?=</\2\s*>)',
1312 bygroups(Punctuation, Name.Tag, handle_syntaxhighlight)),
1313 # <source>: Fallback case for self-closing tags
1314 (r'(?i)(<)(source)\b(\s*?)((?:[^>]|-->)*?)(/\s*?(?<!--)>)', bygroups(
1315 Punctuation, Name.Tag, Whitespace, using(this, state=['root', 'attr']), Punctuation)),
1316 # <score>
1317 (r'(?i)(</)(score)\b(\s*)(>)',
1318 bygroups(Punctuation, Name.Tag, Whitespace, Punctuation)),
1319 (r'(?si)(<)(score)\b([^>]*?(?<!/)>.*?)(?=</\2\s*>)',
1320 bygroups(Punctuation, Name.Tag, handle_score)),
1321 # <score>: Fallback case for self-closing tags
1322 (r'(?i)(<)(score)\b(\s*?)((?:[^>]|-->)*?)(/\s*?(?<!--)>)', bygroups(
1323 Punctuation, Name.Tag, Whitespace, using(this, state=['root', 'attr']), Punctuation)),
1324 # Other parser tags
1325 (r'(?i)(<)({})\b'.format('|'.join(parser_tags)),
1326 bygroups(Punctuation, Name.Tag), 'tag-inner-ordinary'),
1327 (r'(?i)(</)({})\b(\s*)(>)'.format('|'.join(parser_tags)),
1328 bygroups(Punctuation, Name.Tag, Whitespace, Punctuation)),
1329 # LanguageConverter markups
1330 (
1331 r"""(?xi)
1332 (-\{{) # Use {{ to escape format()
1333 ([^|]) (\|)
1334 (?:
1335 (?: ([^;]*?) (=>))?
1336 (\s* (?:{variants}) \s*) (:)
1337 )?
1338 """.format(variants='|'.join(variant_langs)),
1339 bygroups(Punctuation, Keyword, Punctuation,
1340 using(this, state=['root', 'lc-raw']),
1341 Operator, Name.Label, Punctuation),
1342 'lc-inner'
1343 ),
1344 # LanguageConverter markups: composite conversion grammar
1345 (
1346 r"""(?xi)
1347 (-\{)
1348 ([a-z\s;-]*?) (\|)
1349 """,
1350 bygroups(Punctuation,
1351 using(this, state=['root', 'lc-flag']),
1352 Punctuation),
1353 'lc-raw'
1354 ),
1355 # LanguageConverter markups: fallbacks
1356 (
1357 r"""(?xi)
1358 (-\{{) (?!\{{) # Use {{ to escape format()
1359 (?: (\s* (?:{variants}) \s*) (:))?
1360 """.format(variants='|'.join(variant_langs)),
1361 bygroups(Punctuation, Name.Label, Punctuation),
1362 'lc-inner'
1363 ),
1364 ],
1365 'wikilink-name': [
1366 include('replaceable'),
1367 (r'[^{<]+', Name.Tag),
1368 (r'(?s).', Name.Tag),
1369 ],
1370 'wikilink-inner': [
1371 # Quit in case of another wikilink
1372 (r'(?=\[\[)', Punctuation, '#pop'),
1373 (r'\]\]', Punctuation, '#pop'),
1374 include('inline'),
1375 include('text'),
1376 ],
1377 'medialink-inner': [
1378 (r'\]\]', Punctuation, '#pop'),
1379 (r'(\|)([^\n=|]*)(=)',
1380 bygroups(Punctuation, Name.Attribute, Operator)),
1381 (r'\|', Punctuation),
1382 include('inline'),
1383 include('text'),
1384 ],
1385 'quote-common': [
1386 # Quit in case of link/template endings
1387 (r'(?=\]\]|\{\{|\}\})', Punctuation, '#pop'),
1388 (r'\n', Text, '#pop'),
1389 ],
1390 'inline-italic': [
1391 include('quote-common'),
1392 (r"('')(''')(?!')", bygroups(Generic.Emph,
1393 Generic.Strong), ('#pop', 'inline-bold')),
1394 (r"'''(?!')", Generic.EmphStrong, ('#pop', 'inline-italic-bold')),
1395 (r"''(?!')", Generic.Emph, '#pop'),
1396 include('inline'),
1397 include('text-italic'),
1398 ],
1399 'inline-bold': [
1400 include('quote-common'),
1401 (r"(''')('')(?!')", bygroups(
1402 Generic.Strong, Generic.Emph), ('#pop', 'inline-italic')),
1403 (r"'''(?!')", Generic.Strong, '#pop'),
1404 (r"''(?!')", Generic.EmphStrong, ('#pop', 'inline-bold-italic')),
1405 include('inline'),
1406 include('text-bold'),
1407 ],
1408 'inline-bold-italic': [
1409 include('quote-common'),
1410 (r"('')(''')(?!')", bygroups(Generic.EmphStrong,
1411 Generic.Strong), '#pop'),
1412 (r"'''(?!')", Generic.EmphStrong, ('#pop', 'inline-italic')),
1413 (r"''(?!')", Generic.EmphStrong, ('#pop', 'inline-bold')),
1414 include('inline'),
1415 include('text-bold-italic'),
1416 ],
1417 'inline-italic-bold': [
1418 include('quote-common'),
1419 (r"(''')('')(?!')", bygroups(
1420 Generic.EmphStrong, Generic.Emph), '#pop'),
1421 (r"'''(?!')", Generic.EmphStrong, ('#pop', 'inline-italic')),
1422 (r"''(?!')", Generic.EmphStrong, ('#pop', 'inline-bold')),
1423 include('inline'),
1424 include('text-bold-italic'),
1425 ],
1426 'lc-flag': [
1427 (r'\s+', Whitespace),
1428 (r';', Punctuation),
1429 *text_rules(Keyword),
1430 ],
1431 'lc-inner': [
1432 (
1433 r"""(?xi)
1434 (;)
1435 (?: ([^;]*?) (=>))?
1436 (\s* (?:{variants}) \s*) (:)
1437 """.format(variants='|'.join(variant_langs)),
1438 bygroups(Punctuation, using(this, state=['root', 'lc-raw']),
1439 Operator, Name.Label, Punctuation)
1440 ),
1441 (r';?\s*?\}-', Punctuation, '#pop'),
1442 include('inline'),
1443 include('text'),
1444 ],
1445 'lc-raw': [
1446 (r'\}-', Punctuation, '#pop'),
1447 include('inline'),
1448 include('text'),
1449 ],
1450 'replaceable': [
1451 # Comments
1452 (r'<!--[\s\S]*?(?:-->|\Z)', Comment.Multiline),
1453 # Parameters
1454 (
1455 r"""(?x)
1456 (\{{3})
1457 ([^|]*?)
1458 (?=\}{3}|\|)
1459 """,
1460 bygroups(Punctuation, Name.Variable),
1461 'parameter-inner',
1462 ),
1463 # Magic variables
1464 (r'(?i)(\{{\{{)(\s*)({})(\s*)(\}}\}})'.format('|'.join(magic_vars_i)),
1465 bygroups(Punctuation, Whitespace, Name.Function, Whitespace, Punctuation)),
1466 (r'(\{{\{{)(\s*)({})(\s*)(\}}\}})'.format('|'.join(magic_vars)),
1467 bygroups(Punctuation, Whitespace, Name.Function, Whitespace, Punctuation)),
1468 # Parser functions & templates
1469 (r'\{\{', Punctuation, 'template-begin-space'),
1470 # <tvar> legacy syntax
1471 (r'(?i)(<)(tvar)\b(\|)([^>]*?)(>)', bygroups(Punctuation,
1472 Name.Tag, Punctuation, String, Punctuation)),
1473 (r'</>', Punctuation, '#pop'),
1474 # <tvar>
1475 (r'(?i)(<)(tvar)\b', bygroups(Punctuation, Name.Tag), 'tag-inner-ordinary'),
1476 (r'(?i)(</)(tvar)\b(\s*)(>)',
1477 bygroups(Punctuation, Name.Tag, Whitespace, Punctuation)),
1478 ],
1479 'parameter-inner': [
1480 (r'\}{3}', Punctuation, '#pop'),
1481 (r'\|', Punctuation),
1482 include('inline'),
1483 include('text'),
1484 ],
1485 'template-begin-space': [
1486 # Templates allow line breaks at the beginning, and due to how MediaWiki handles
1487 # comments, an extra state is required to handle things like {{\n<!---->\n name}}
1488 (r'<!--[\s\S]*?(?:-->|\Z)', Comment.Multiline),
1489 (r'\s+', Whitespace),
1490 # Parser functions
1491 (
1492 r'(?i)(\#[{}]*?|{})(:)'.format(title_char,
1493 '|'.join(parser_functions_i)),
1494 bygroups(Name.Function, Punctuation), ('#pop', 'template-inner')
1495 ),
1496 (
1497 r'({})(:)'.format('|'.join(parser_functions)),
1498 bygroups(Name.Function, Punctuation), ('#pop', 'template-inner')
1499 ),
1500 # Templates
1501 (
1502 rf'(?i)([{title_char}]*?)(:)',
1503 bygroups(Name.Namespace, Punctuation), ('#pop', 'template-name')
1504 ),
1505 default(('#pop', 'template-name'),),
1506 ],
1507 'template-name': [
1508 (r'(\s*?)(\|)', bygroups(Text, Punctuation), ('#pop', 'template-inner')),
1509 (r'\}\}', Punctuation, '#pop'),
1510 (r'\n', Text, '#pop'),
1511 include('replaceable'),
1512 *text_rules(Name.Tag),
1513 ],
1514 'template-inner': [
1515 (r'\}\}', Punctuation, '#pop'),
1516 (r'\|', Punctuation),
1517 (
1518 r"""(?x)
1519 (?<=\|)
1520 ( (?: (?! \{\{ | \}\} )[^=\|<])*? ) # Exclude templates and tags
1521 (=)
1522 """,
1523 bygroups(Name.Label, Operator)
1524 ),
1525 include('inline'),
1526 include('text'),
1527 ],
1528 'table': [
1529 # Use [ \t\n\r\0\x0B] instead of \s to follow PHP trim() behavior
1530 # Endings
1531 (r'^([ \t\n\r\0\x0B]*?)(\|\})',
1532 bygroups(Whitespace, Punctuation), '#pop'),
1533 # Table rows
1534 (r'^([ \t\n\r\0\x0B]*?)(\|-+)(.*)$', bygroups(Whitespace, Punctuation,
1535 using(this, state=['root', 'attr']))),
1536 # Captions
1537 (
1538 r"""(?x)
1539 ^([ \t\n\r\0\x0B]*?)(\|\+)
1540 # Exclude links, template and tags
1541 (?: ( (?: (?! \[\[ | \{\{ )[^|\n<] )*? )(\|) )?
1542 (.*?)$
1543 """,
1544 bygroups(Whitespace, Punctuation, using(this, state=[
1545 'root', 'attr']), Punctuation, Generic.Heading),
1546 ),
1547 # Table data
1548 (
1549 r"""(?x)
1550 ( ^(?:[ \t\n\r\0\x0B]*?)\| | \|\| )
1551 (?: ( (?: (?! \[\[ | \{\{ )[^|\n<] )*? )(\|)(?!\|) )?
1552 """,
1553 bygroups(Punctuation, using(this, state=[
1554 'root', 'attr']), Punctuation),
1555 ),
1556 # Table headers
1557 (
1558 r"""(?x)
1559 ( ^(?:[ \t\n\r\0\x0B]*?)! )
1560 (?: ( (?: (?! \[\[ | \{\{ )[^|\n<] )*? )(\|)(?!\|) )?
1561 """,
1562 bygroups(Punctuation, using(this, state=[
1563 'root', 'attr']), Punctuation),
1564 'table-header',
1565 ),
1566 include('list'),
1567 include('inline'),
1568 include('text'),
1569 ],
1570 'table-header': [
1571 # Requires another state for || handling inside headers
1572 (r'\n', Text, '#pop'),
1573 (
1574 r"""(?x)
1575 (!!|\|\|)
1576 (?:
1577 ( (?: (?! \[\[ | \{\{ )[^|\n<] )*? )
1578 (\|)(?!\|)
1579 )?
1580 """,
1581 bygroups(Punctuation, using(this, state=[
1582 'root', 'attr']), Punctuation)
1583 ),
1584 *text_rules(Generic.Subheading),
1585 ],
1586 'entity': [
1587 (r'&\S*?;', Name.Entity),
1588 ],
1589 'dt': [
1590 (r'\n', Text, '#pop'),
1591 include('inline'),
1592 (r':', Keyword, '#pop'),
1593 include('text'),
1594 ],
1595 'extlink-inner': [
1596 (r'\]', Punctuation, '#pop'),
1597 include('inline'),
1598 include('text'),
1599 ],
1600 'nowiki-ish': [
1601 include('entity'),
1602 include('text'),
1603 ],
1604 'attr': [
1605 include('replaceable'),
1606 (r'\s+', Whitespace),
1607 (r'(=)(\s*)(")', bygroups(Operator, Whitespace, String.Double), 'attr-val-2'),
1608 (r"(=)(\s*)(')", bygroups(Operator, Whitespace, String.Single), 'attr-val-1'),
1609 (r'(=)(\s*)', bygroups(Operator, Whitespace), 'attr-val-0'),
1610 (r'[\w:-]+', Name.Attribute),
1612 ],
1613 'attr-val-0': [
1614 (r'\s', Whitespace, '#pop'),
1615 include('replaceable'),
1616 *text_rules(String),
1617 ],
1618 'attr-val-1': [
1619 (r"'", String.Single, '#pop'),
1620 include('replaceable'),
1621 *text_rules(String.Single),
1622 ],
1623 'attr-val-2': [
1624 (r'"', String.Double, '#pop'),
1625 include('replaceable'),
1626 *text_rules(String.Double),
1627 ],
1628 'tag-inner-ordinary': [
1629 (r'/?\s*>', Punctuation, '#pop'),
1630 include('tag-attr'),
1631 ],
1632 'tag-inner': [
1633 # Return to root state for self-closing tags
1634 (r'/\s*>', Punctuation, '#pop:2'),
1635 (r'\s*>', Punctuation, '#pop'),
1636 include('tag-attr'),
1637 ],
1638 # There states below are just like their non-tag variants, the key difference is
1639 # they forcibly quit when encountering tag closing markup
1640 'tag-attr': [
1641 include('replaceable'),
1642 (r'\s+', Whitespace),
1643 (r'(=)(\s*)(")', bygroups(Operator,
1644 Whitespace, String.Double), 'tag-attr-val-2'),
1645 (r"(=)(\s*)(')", bygroups(Operator,
1646 Whitespace, String.Single), 'tag-attr-val-1'),
1647 (r'(=)(\s*)', bygroups(Operator, Whitespace), 'tag-attr-val-0'),
1648 (r'[\w:-]+', Name.Attribute),
1650 ],
1651 'tag-attr-val-0': [
1652 (r'\s', Whitespace, '#pop'),
1653 (r'/?>', Punctuation, '#pop:2'),
1654 include('replaceable'),
1655 *text_rules(String),
1656 ],
1657 'tag-attr-val-1': [
1658 (r"'", String.Single, '#pop'),
1659 (r'/?>', Punctuation, '#pop:2'),
1660 include('replaceable'),
1661 *text_rules(String.Single),
1662 ],
1663 'tag-attr-val-2': [
1664 (r'"', String.Double, '#pop'),
1665 (r'/?>', Punctuation, '#pop:2'),
1666 include('replaceable'),
1667 *text_rules(String.Double),
1668 ],
1669 'tag-nowiki': nowiki_tag_rules('nowiki'),
1670 'tag-pre': nowiki_tag_rules('pre'),
1671 'tag-categorytree': plaintext_tag_rules('categorytree'),
1672 'tag-dynamicpagelist': plaintext_tag_rules('dynamicpagelist'),
1673 'tag-hiero': plaintext_tag_rules('hiero'),
1674 'tag-inputbox': plaintext_tag_rules('inputbox'),
1675 'tag-imagemap': plaintext_tag_rules('imagemap'),
1676 'tag-charinsert': plaintext_tag_rules('charinsert'),
1677 'tag-timeline': plaintext_tag_rules('timeline'),
1678 'tag-gallery': plaintext_tag_rules('gallery'),
1679 'tag-graph': plaintext_tag_rules('graph'),
1680 'tag-rss': plaintext_tag_rules('rss'),
1681 'tag-math': delegate_tag_rules('math', TexLexer, state='math'),
1682 'tag-chem': delegate_tag_rules('chem', TexLexer, state='math'),
1683 'tag-ce': delegate_tag_rules('ce', TexLexer, state='math'),
1684 'tag-templatedata': delegate_tag_rules('templatedata', JsonLexer),
1685 'text-italic': text_rules(Generic.Emph),
1686 'text-bold': text_rules(Generic.Strong),
1687 'text-bold-italic': text_rules(Generic.EmphStrong),
1688 'text': text_rules(Text),
1689 }