Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""
2 pygments.lexer
3 ~~~~~~~~~~~~~~
5 Base lexer classes.
7 :copyright: Copyright 2006-present by the Pygments team, see AUTHORS.
8 :license: BSD, see LICENSE for details.
9"""
11import re
12import sys
13import time
15from pip._vendor.pygments.filter import apply_filters, Filter
16from pip._vendor.pygments.filters import get_filter_by_name
17from pip._vendor.pygments.token import Error, Text, Other, Whitespace, _TokenType
18from pip._vendor.pygments.util import get_bool_opt, get_int_opt, get_list_opt, \
19 make_analysator, Future, guess_decode
20from pip._vendor.pygments.regexopt import regex_opt
22__all__ = ['Lexer', 'RegexLexer', 'ExtendedRegexLexer', 'DelegatingLexer',
23 'LexerContext', 'include', 'inherit', 'bygroups', 'using', 'this',
24 'default', 'words', 'line_re']
26line_re = re.compile('.*?\n')
28_encoding_map = [(b'\xef\xbb\xbf', 'utf-8'),
29 (b'\xff\xfe\0\0', 'utf-32'),
30 (b'\0\0\xfe\xff', 'utf-32be'),
31 (b'\xff\xfe', 'utf-16'),
32 (b'\xfe\xff', 'utf-16be')]
34_default_analyse = staticmethod(lambda x: 0.0)
37class LexerMeta(type):
38 """
39 This metaclass automagically converts ``analyse_text`` methods into
40 static methods which always return float values.
41 """
43 def __new__(mcs, name, bases, d):
44 if 'analyse_text' in d:
45 d['analyse_text'] = make_analysator(d['analyse_text'])
46 return type.__new__(mcs, name, bases, d)
49class Lexer(metaclass=LexerMeta):
50 """
51 Lexer for a specific language.
53 See also :doc:`lexerdevelopment`, a high-level guide to writing
54 lexers.
56 Lexer classes have attributes used for choosing the most appropriate
57 lexer based on various criteria.
59 .. autoattribute:: name
60 :no-value:
61 .. autoattribute:: aliases
62 :no-value:
63 .. autoattribute:: filenames
64 :no-value:
65 .. autoattribute:: alias_filenames
66 .. autoattribute:: mimetypes
67 :no-value:
68 .. autoattribute:: priority
70 Lexers included in Pygments should have two additional attributes:
72 .. autoattribute:: url
73 :no-value:
74 .. autoattribute:: version_added
75 :no-value:
77 Lexers included in Pygments may have additional attributes:
79 .. autoattribute:: _example
80 :no-value:
82 You can pass options to the constructor. The basic options recognized
83 by all lexers and processed by the base `Lexer` class are:
85 ``stripnl``
86 Strip leading and trailing newlines from the input (default: True).
87 ``stripall``
88 Strip all leading and trailing whitespace from the input
89 (default: False).
90 ``ensurenl``
91 Make sure that the input ends with a newline (default: True). This
92 is required for some lexers that consume input linewise.
94 .. versionadded:: 1.3
96 ``tabsize``
97 If given and greater than 0, expand tabs in the input (default: 0).
98 ``encoding``
99 If given, must be an encoding name. This encoding will be used to
100 convert the input string to Unicode, if it is not already a Unicode
101 string (default: ``'guess'``, which uses a simple UTF-8 / Locale /
102 Latin1 detection. Can also be ``'chardet'`` to use the chardet
103 library, if it is installed.
104 ``inencoding``
105 Overrides the ``encoding`` if given.
106 """
108 #: Full name of the lexer, in human-readable form
109 name = None
111 #: A list of short, unique identifiers that can be used to look
112 #: up the lexer from a list, e.g., using `get_lexer_by_name()`.
113 aliases = []
115 #: A list of `fnmatch` patterns that match filenames which contain
116 #: content for this lexer. The patterns in this list should be unique among
117 #: all lexers.
118 filenames = []
120 #: A list of `fnmatch` patterns that match filenames which may or may not
121 #: contain content for this lexer. This list is used by the
122 #: :func:`.guess_lexer_for_filename()` function, to determine which lexers
123 #: are then included in guessing the correct one. That means that
124 #: e.g. every lexer for HTML and a template language should include
125 #: ``\*.html`` in this list.
126 alias_filenames = []
128 #: A list of MIME types for content that can be lexed with this lexer.
129 mimetypes = []
131 #: Priority, should multiple lexers match and no content is provided
132 priority = 0
134 #: URL of the language specification/definition. Used in the Pygments
135 #: documentation. Set to an empty string to disable.
136 url = None
138 #: Version of Pygments in which the lexer was added.
139 version_added = None
141 #: Example file name. Relative to the ``tests/examplefiles`` directory.
142 #: This is used by the documentation generator to show an example.
143 _example = None
145 def __init__(self, **options):
146 """
147 This constructor takes arbitrary options as keyword arguments.
148 Every subclass must first process its own options and then call
149 the `Lexer` constructor, since it processes the basic
150 options like `stripnl`.
152 An example looks like this:
154 .. sourcecode:: python
156 def __init__(self, **options):
157 self.compress = options.get('compress', '')
158 Lexer.__init__(self, **options)
160 As these options must all be specifiable as strings (due to the
161 command line usage), there are various utility functions
162 available to help with that, see `Utilities`_.
163 """
164 self.options = options
165 self.stripnl = get_bool_opt(options, 'stripnl', True)
166 self.stripall = get_bool_opt(options, 'stripall', False)
167 self.ensurenl = get_bool_opt(options, 'ensurenl', True)
168 self.tabsize = get_int_opt(options, 'tabsize', 0)
169 self.encoding = options.get('encoding', 'guess')
170 self.encoding = options.get('inencoding') or self.encoding
171 self.filters = []
172 for filter_ in get_list_opt(options, 'filters', ()):
173 self.add_filter(filter_)
175 def __repr__(self):
176 if self.options:
177 return f'<pygments.lexers.{self.__class__.__name__} with {self.options!r}>'
178 else:
179 return f'<pygments.lexers.{self.__class__.__name__}>'
181 def add_filter(self, filter_, **options):
182 """
183 Add a new stream filter to this lexer.
184 """
185 if not isinstance(filter_, Filter):
186 filter_ = get_filter_by_name(filter_, **options)
187 self.filters.append(filter_)
189 def analyse_text(text):
190 """
191 A static method which is called for lexer guessing.
193 It should analyse the text and return a float in the range
194 from ``0.0`` to ``1.0``. If it returns ``0.0``, the lexer
195 will not be selected as the most probable one, if it returns
196 ``1.0``, it will be selected immediately. This is used by
197 `guess_lexer`.
199 The `LexerMeta` metaclass automatically wraps this function so
200 that it works like a static method (no ``self`` or ``cls``
201 parameter) and the return value is automatically converted to
202 `float`. If the return value is an object that is boolean `False`
203 it's the same as if the return values was ``0.0``.
204 """
206 def _preprocess_lexer_input(self, text):
207 """Apply preprocessing such as decoding the input, removing BOM and normalizing newlines."""
209 if not isinstance(text, str):
210 if self.encoding == 'guess':
211 text, _ = guess_decode(text)
212 elif self.encoding == 'chardet':
213 try:
214 # pip vendoring note: this code is not reachable by pip,
215 # removed import of chardet to make it clear.
216 raise ImportError('chardet is not vendored by pip')
217 except ImportError as e:
218 raise ImportError('To enable chardet encoding guessing, '
219 'please install the chardet library '
220 'from http://chardet.feedparser.org/') from e
221 # check for BOM first
222 decoded = None
223 for bom, encoding in _encoding_map:
224 if text.startswith(bom):
225 decoded = text[len(bom):].decode(encoding, 'replace')
226 break
227 # no BOM found, so use chardet
228 if decoded is None:
229 enc = chardet.detect(text[:1024]) # Guess using first 1KB
230 decoded = text.decode(enc.get('encoding') or 'utf-8',
231 'replace')
232 text = decoded
233 else:
234 text = text.decode(self.encoding)
235 if text.startswith('\ufeff'):
236 text = text[len('\ufeff'):]
237 else:
238 if text.startswith('\ufeff'):
239 text = text[len('\ufeff'):]
241 # text now *is* a unicode string
242 text = text.replace('\r\n', '\n')
243 text = text.replace('\r', '\n')
244 if self.stripall:
245 text = text.strip()
246 elif self.stripnl:
247 text = text.strip('\n')
248 if self.tabsize > 0:
249 text = text.expandtabs(self.tabsize)
250 if self.ensurenl and not text.endswith('\n'):
251 text += '\n'
253 return text
255 def get_tokens(self, text, unfiltered=False):
256 """
257 This method is the basic interface of a lexer. It is called by
258 the `highlight()` function. It must process the text and return an
259 iterable of ``(tokentype, value)`` pairs from `text`.
261 Normally, you don't need to override this method. The default
262 implementation processes the options recognized by all lexers
263 (`stripnl`, `stripall` and so on), and then yields all tokens
264 from `get_tokens_unprocessed()`, with the ``index`` dropped.
266 If `unfiltered` is set to `True`, the filtering mechanism is
267 bypassed even if filters are defined.
268 """
269 text = self._preprocess_lexer_input(text)
271 def streamer():
272 for _, t, v in self.get_tokens_unprocessed(text):
273 yield t, v
274 stream = streamer()
275 if not unfiltered:
276 stream = apply_filters(stream, self.filters, self)
277 return stream
279 def get_tokens_unprocessed(self, text):
280 """
281 This method should process the text and return an iterable of
282 ``(index, tokentype, value)`` tuples where ``index`` is the starting
283 position of the token within the input text.
285 It must be overridden by subclasses. It is recommended to
286 implement it as a generator to maximize effectiveness.
287 """
288 raise NotImplementedError
291class DelegatingLexer(Lexer):
292 """
293 This lexer takes two lexer as arguments. A root lexer and
294 a language lexer. First everything is scanned using the language
295 lexer, afterwards all ``Other`` tokens are lexed using the root
296 lexer.
298 The lexers from the ``template`` lexer package use this base lexer.
299 """
301 def __init__(self, _root_lexer, _language_lexer, _needle=Other, **options):
302 self.root_lexer = _root_lexer(**options)
303 self.language_lexer = _language_lexer(**options)
304 self.needle = _needle
305 Lexer.__init__(self, **options)
307 def get_tokens_unprocessed(self, text):
308 buffered = ''
309 insertions = []
310 lng_buffer = []
311 for i, t, v in self.language_lexer.get_tokens_unprocessed(text):
312 if t is self.needle:
313 if lng_buffer:
314 insertions.append((len(buffered), lng_buffer))
315 lng_buffer = []
316 buffered += v
317 else:
318 lng_buffer.append((i, t, v))
319 if lng_buffer:
320 insertions.append((len(buffered), lng_buffer))
321 return do_insertions(insertions,
322 self.root_lexer.get_tokens_unprocessed(buffered))
325# ------------------------------------------------------------------------------
326# RegexLexer and ExtendedRegexLexer
327#
330class include(str): # pylint: disable=invalid-name
331 """
332 Indicates that a state should include rules from another state.
333 """
334 pass
337class _inherit:
338 """
339 Indicates the a state should inherit from its superclass.
340 """
341 def __repr__(self):
342 return 'inherit'
344inherit = _inherit() # pylint: disable=invalid-name
347class combined(tuple): # pylint: disable=invalid-name
348 """
349 Indicates a state combined from multiple states.
350 """
352 def __new__(cls, *args):
353 return tuple.__new__(cls, args)
355 def __init__(self, *args):
356 # tuple.__init__ doesn't do anything
357 pass
360class _PseudoMatch:
361 """
362 A pseudo match object constructed from a string.
363 """
365 def __init__(self, start, text):
366 self._text = text
367 self._start = start
369 def start(self, arg=None):
370 return self._start
372 def end(self, arg=None):
373 return self._start + len(self._text)
375 def group(self, arg=None):
376 if arg:
377 raise IndexError('No such group')
378 return self._text
380 def groups(self):
381 return (self._text,)
383 def groupdict(self):
384 return {}
387def bygroups(*args):
388 """
389 Callback that yields multiple actions for each group in the match.
390 """
391 def callback(lexer, match, ctx=None):
392 for i, action in enumerate(args):
393 if action is None:
394 continue
395 elif type(action) is _TokenType:
396 data = match.group(i + 1)
397 if data:
398 yield match.start(i + 1), action, data
399 else:
400 data = match.group(i + 1)
401 if data is not None:
402 if ctx:
403 ctx.pos = match.start(i + 1)
404 for item in action(lexer,
405 _PseudoMatch(match.start(i + 1), data), ctx):
406 if item:
407 yield item
408 if ctx:
409 ctx.pos = match.end()
410 return callback
413class _This:
414 """
415 Special singleton used for indicating the caller class.
416 Used by ``using``.
417 """
419this = _This()
422def using(_other, **kwargs):
423 """
424 Callback that processes the match with a different lexer.
426 The keyword arguments are forwarded to the lexer, except `state` which
427 is handled separately.
429 `state` specifies the state that the new lexer will start in, and can
430 be an enumerable such as ('root', 'inline', 'string') or a simple
431 string which is assumed to be on top of the root state.
433 Note: For that to work, `_other` must not be an `ExtendedRegexLexer`.
434 """
435 gt_kwargs = {}
436 if 'state' in kwargs:
437 s = kwargs.pop('state')
438 if isinstance(s, (list, tuple)):
439 gt_kwargs['stack'] = s
440 else:
441 gt_kwargs['stack'] = ('root', s)
443 if _other is this:
444 def callback(lexer, match, ctx=None):
445 # if keyword arguments are given the callback
446 # function has to create a new lexer instance
447 if kwargs:
448 # XXX: cache that somehow
449 d = dict(lexer.options)
450 d.update(kwargs)
451 lx = lexer.__class__(**d)
452 else:
453 lx = lexer
454 s = match.start()
455 for i, t, v in lx.get_tokens_unprocessed(match.group(), **gt_kwargs):
456 yield i + s, t, v
457 if ctx:
458 ctx.pos = match.end()
459 else:
460 def callback(lexer, match, ctx=None):
461 # XXX: cache that somehow
462 d = dict(lexer.options)
463 d.update(kwargs)
464 lx = _other(**d)
466 s = match.start()
467 for i, t, v in lx.get_tokens_unprocessed(match.group(), **gt_kwargs):
468 yield i + s, t, v
469 if ctx:
470 ctx.pos = match.end()
471 return callback
474class default:
475 """
476 Indicates a state or state action (e.g. #pop) to apply.
477 For example default('#pop') is equivalent to ('', Token, '#pop')
478 Note that state tuples may be used as well.
480 .. versionadded:: 2.0
481 """
482 def __init__(self, state):
483 self.state = state
486class words(Future):
487 """
488 Indicates a list of literal words that is transformed into an optimized
489 regex that matches any of the words.
491 .. versionadded:: 2.0
492 """
493 def __init__(self, words, prefix='', suffix=''):
494 self.words = words
495 self.prefix = prefix
496 self.suffix = suffix
498 def get(self):
499 return regex_opt(self.words, prefix=self.prefix, suffix=self.suffix)
502class RegexLexerMeta(LexerMeta):
503 """
504 Metaclass for RegexLexer, creates the self._tokens attribute from
505 self.tokens on the first instantiation.
506 """
508 def _process_regex(cls, regex, rflags, state):
509 """Preprocess the regular expression component of a token definition."""
510 if isinstance(regex, Future):
511 regex = regex.get()
512 return re.compile(regex, rflags).match
514 def _process_token(cls, token):
515 """Preprocess the token component of a token definition."""
516 assert type(token) is _TokenType or callable(token), \
517 f'token type must be simple type or callable, not {token!r}'
518 return token
520 def _process_new_state(cls, new_state, unprocessed, processed):
521 """Preprocess the state transition action of a token definition."""
522 if isinstance(new_state, str):
523 # an existing state
524 if new_state == '#pop':
525 return -1
526 elif new_state in unprocessed:
527 return (new_state,)
528 elif new_state == '#push':
529 return new_state
530 elif new_state[:5] == '#pop:':
531 return -int(new_state[5:])
532 else:
533 assert False, f'unknown new state {new_state!r}'
534 elif isinstance(new_state, combined):
535 # combine a new state from existing ones
536 tmp_state = '_tmp_%d' % cls._tmpname
537 cls._tmpname += 1
538 itokens = []
539 for istate in new_state:
540 assert istate != new_state, f'circular state ref {istate!r}'
541 itokens.extend(cls._process_state(unprocessed,
542 processed, istate))
543 processed[tmp_state] = itokens
544 return (tmp_state,)
545 elif isinstance(new_state, tuple):
546 # push more than one state
547 for istate in new_state:
548 assert (istate in unprocessed or
549 istate in ('#pop', '#push')), \
550 'unknown new state ' + istate
551 return new_state
552 else:
553 assert False, f'unknown new state def {new_state!r}'
555 def _process_state(cls, unprocessed, processed, state):
556 """Preprocess a single state definition."""
557 assert isinstance(state, str), f"wrong state name {state!r}"
558 assert state[0] != '#', f"invalid state name {state!r}"
559 if state in processed:
560 return processed[state]
561 tokens = processed[state] = []
562 rflags = cls.flags
563 for tdef in unprocessed[state]:
564 if isinstance(tdef, include):
565 # it's a state reference
566 assert tdef != state, f"circular state reference {state!r}"
567 tokens.extend(cls._process_state(unprocessed, processed,
568 str(tdef)))
569 continue
570 if isinstance(tdef, _inherit):
571 # should be processed already, but may not in the case of:
572 # 1. the state has no counterpart in any parent
573 # 2. the state includes more than one 'inherit'
574 continue
575 if isinstance(tdef, default):
576 new_state = cls._process_new_state(tdef.state, unprocessed, processed)
577 tokens.append((re.compile('').match, None, new_state))
578 continue
580 assert type(tdef) is tuple, f"wrong rule def {tdef!r}"
582 try:
583 rex = cls._process_regex(tdef[0], rflags, state)
584 except Exception as err:
585 raise ValueError(f"uncompilable regex {tdef[0]!r} in state {state!r} of {cls!r}: {err}") from err
587 token = cls._process_token(tdef[1])
589 if len(tdef) == 2:
590 new_state = None
591 else:
592 new_state = cls._process_new_state(tdef[2],
593 unprocessed, processed)
595 tokens.append((rex, token, new_state))
596 return tokens
598 def process_tokendef(cls, name, tokendefs=None):
599 """Preprocess a dictionary of token definitions."""
600 processed = cls._all_tokens[name] = {}
601 tokendefs = tokendefs or cls.tokens[name]
602 for state in list(tokendefs):
603 cls._process_state(tokendefs, processed, state)
604 return processed
606 def get_tokendefs(cls):
607 """
608 Merge tokens from superclasses in MRO order, returning a single tokendef
609 dictionary.
611 Any state that is not defined by a subclass will be inherited
612 automatically. States that *are* defined by subclasses will, by
613 default, override that state in the superclass. If a subclass wishes to
614 inherit definitions from a superclass, it can use the special value
615 "inherit", which will cause the superclass' state definition to be
616 included at that point in the state.
617 """
618 tokens = {}
619 inheritable = {}
620 for c in cls.__mro__:
621 toks = c.__dict__.get('tokens', {})
623 for state, items in toks.items():
624 curitems = tokens.get(state)
625 if curitems is None:
626 # N.b. because this is assigned by reference, sufficiently
627 # deep hierarchies are processed incrementally (e.g. for
628 # A(B), B(C), C(RegexLexer), B will be premodified so X(B)
629 # will not see any inherits in B).
630 tokens[state] = items
631 try:
632 inherit_ndx = items.index(inherit)
633 except ValueError:
634 continue
635 inheritable[state] = inherit_ndx
636 continue
638 inherit_ndx = inheritable.pop(state, None)
639 if inherit_ndx is None:
640 continue
642 # Replace the "inherit" value with the items
643 curitems[inherit_ndx:inherit_ndx+1] = items
644 try:
645 # N.b. this is the index in items (that is, the superclass
646 # copy), so offset required when storing below.
647 new_inh_ndx = items.index(inherit)
648 except ValueError:
649 pass
650 else:
651 inheritable[state] = inherit_ndx + new_inh_ndx
653 return tokens
655 def __call__(cls, *args, **kwds):
656 """Instantiate cls after preprocessing its token definitions."""
657 if '_tokens' not in cls.__dict__:
658 cls._all_tokens = {}
659 cls._tmpname = 0
660 if hasattr(cls, 'token_variants') and cls.token_variants:
661 # don't process yet
662 pass
663 else:
664 cls._tokens = cls.process_tokendef('', cls.get_tokendefs())
666 return type.__call__(cls, *args, **kwds)
669class RegexLexer(Lexer, metaclass=RegexLexerMeta):
670 """
671 Base for simple stateful regular expression-based lexers.
672 Simplifies the lexing process so that you need only
673 provide a list of states and regular expressions.
674 """
676 #: Flags for compiling the regular expressions.
677 #: Defaults to MULTILINE.
678 flags = re.MULTILINE
680 #: At all time there is a stack of states. Initially, the stack contains
681 #: a single state 'root'. The top of the stack is called "the current state".
682 #:
683 #: Dict of ``{'state': [(regex, tokentype, new_state), ...], ...}``
684 #:
685 #: ``new_state`` can be omitted to signify no state transition.
686 #: If ``new_state`` is a string, it is pushed on the stack. This ensure
687 #: the new current state is ``new_state``.
688 #: If ``new_state`` is a tuple of strings, all of those strings are pushed
689 #: on the stack and the current state will be the last element of the list.
690 #: ``new_state`` can also be ``combined('state1', 'state2', ...)``
691 #: to signify a new, anonymous state combined from the rules of two
692 #: or more existing ones.
693 #: Furthermore, it can be '#pop' to signify going back one step in
694 #: the state stack, or '#push' to push the current state on the stack
695 #: again. Note that if you push while in a combined state, the combined
696 #: state itself is pushed, and not only the state in which the rule is
697 #: defined.
698 #:
699 #: The tuple can also be replaced with ``include('state')``, in which
700 #: case the rules from the state named by the string are included in the
701 #: current one.
702 tokens = {}
704 def get_tokens_unprocessed(self, text, stack=('root',)):
705 """
706 Split ``text`` into (tokentype, text) pairs.
708 ``stack`` is the initial stack (default: ``['root']``)
709 """
710 pos = 0
711 tokendefs = self._tokens
712 statestack = list(stack)
713 statetokens = tokendefs[statestack[-1]]
714 while 1:
715 for rexmatch, action, new_state in statetokens:
716 m = rexmatch(text, pos)
717 if m:
718 if action is not None:
719 if type(action) is _TokenType:
720 yield pos, action, m.group()
721 else:
722 yield from action(self, m)
723 pos = m.end()
724 if new_state is not None:
725 # state transition
726 if isinstance(new_state, tuple):
727 for state in new_state:
728 if state == '#pop':
729 if len(statestack) > 1:
730 statestack.pop()
731 elif state == '#push':
732 statestack.append(statestack[-1])
733 else:
734 statestack.append(state)
735 elif isinstance(new_state, int):
736 # pop, but keep at least one state on the stack
737 # (random code leading to unexpected pops should
738 # not allow exceptions)
739 if abs(new_state) >= len(statestack):
740 del statestack[1:]
741 else:
742 del statestack[new_state:]
743 elif new_state == '#push':
744 statestack.append(statestack[-1])
745 else:
746 assert False, f"wrong state def: {new_state!r}"
747 statetokens = tokendefs[statestack[-1]]
748 break
749 else:
750 # We are here only if all state tokens have been considered
751 # and there was not a match on any of them.
752 try:
753 if text[pos] == '\n':
754 # at EOL, reset state to "root"
755 statestack = ['root']
756 statetokens = tokendefs['root']
757 yield pos, Whitespace, '\n'
758 pos += 1
759 continue
760 yield pos, Error, text[pos]
761 pos += 1
762 except IndexError:
763 break
766class LexerContext:
767 """
768 A helper object that holds lexer position data.
769 """
771 def __init__(self, text, pos, stack=None, end=None):
772 self.text = text
773 self.pos = pos
774 self.end = end or len(text) # end=0 not supported ;-)
775 self.stack = stack or ['root']
777 def __repr__(self):
778 return f'LexerContext({self.text!r}, {self.pos!r}, {self.stack!r})'
781class ExtendedRegexLexer(RegexLexer):
782 """
783 A RegexLexer that uses a context object to store its state.
784 """
786 def get_tokens_unprocessed(self, text=None, context=None):
787 """
788 Split ``text`` into (tokentype, text) pairs.
789 If ``context`` is given, use this lexer context instead.
790 """
791 tokendefs = self._tokens
792 if not context:
793 ctx = LexerContext(text, 0)
794 statetokens = tokendefs['root']
795 else:
796 ctx = context
797 statetokens = tokendefs[ctx.stack[-1]]
798 text = ctx.text
799 while 1:
800 for rexmatch, action, new_state in statetokens:
801 m = rexmatch(text, ctx.pos, ctx.end)
802 if m:
803 if action is not None:
804 if type(action) is _TokenType:
805 yield ctx.pos, action, m.group()
806 ctx.pos = m.end()
807 else:
808 yield from action(self, m, ctx)
809 if not new_state:
810 # altered the state stack?
811 statetokens = tokendefs[ctx.stack[-1]]
812 # CAUTION: callback must set ctx.pos!
813 if new_state is not None:
814 # state transition
815 if isinstance(new_state, tuple):
816 for state in new_state:
817 if state == '#pop':
818 if len(ctx.stack) > 1:
819 ctx.stack.pop()
820 elif state == '#push':
821 ctx.stack.append(ctx.stack[-1])
822 else:
823 ctx.stack.append(state)
824 elif isinstance(new_state, int):
825 # see RegexLexer for why this check is made
826 if abs(new_state) >= len(ctx.stack):
827 del ctx.stack[1:]
828 else:
829 del ctx.stack[new_state:]
830 elif new_state == '#push':
831 ctx.stack.append(ctx.stack[-1])
832 else:
833 assert False, f"wrong state def: {new_state!r}"
834 statetokens = tokendefs[ctx.stack[-1]]
835 break
836 else:
837 try:
838 if ctx.pos >= ctx.end:
839 break
840 if text[ctx.pos] == '\n':
841 # at EOL, reset state to "root"
842 ctx.stack = ['root']
843 statetokens = tokendefs['root']
844 yield ctx.pos, Text, '\n'
845 ctx.pos += 1
846 continue
847 yield ctx.pos, Error, text[ctx.pos]
848 ctx.pos += 1
849 except IndexError:
850 break
853def do_insertions(insertions, tokens):
854 """
855 Helper for lexers which must combine the results of several
856 sublexers.
858 ``insertions`` is a list of ``(index, itokens)`` pairs.
859 Each ``itokens`` iterable should be inserted at position
860 ``index`` into the token stream given by the ``tokens``
861 argument.
863 The result is a combined token stream.
865 TODO: clean up the code here.
866 """
867 insertions = iter(insertions)
868 try:
869 index, itokens = next(insertions)
870 except StopIteration:
871 # no insertions
872 yield from tokens
873 return
875 realpos = None
876 insleft = True
878 # iterate over the token stream where we want to insert
879 # the tokens from the insertion list.
880 for i, t, v in tokens:
881 # first iteration. store the position of first item
882 if realpos is None:
883 realpos = i
884 oldi = 0
885 while insleft and i + len(v) >= index:
886 tmpval = v[oldi:index - i]
887 if tmpval:
888 yield realpos, t, tmpval
889 realpos += len(tmpval)
890 for it_index, it_token, it_value in itokens:
891 yield realpos, it_token, it_value
892 realpos += len(it_value)
893 oldi = index - i
894 try:
895 index, itokens = next(insertions)
896 except StopIteration:
897 insleft = False
898 break # not strictly necessary
899 if oldi < len(v):
900 yield realpos, t, v[oldi:]
901 realpos += len(v) - oldi
903 # leftover tokens
904 while insleft:
905 # no normal tokens, set realpos to zero
906 realpos = realpos or 0
907 for p, t, v in itokens:
908 yield realpos, t, v
909 realpos += len(v)
910 try:
911 index, itokens = next(insertions)
912 except StopIteration:
913 insleft = False
914 break # not strictly necessary
917class ProfilingRegexLexerMeta(RegexLexerMeta):
918 """Metaclass for ProfilingRegexLexer, collects regex timing info."""
920 def _process_regex(cls, regex, rflags, state):
921 if isinstance(regex, words):
922 rex = regex_opt(regex.words, prefix=regex.prefix,
923 suffix=regex.suffix)
924 else:
925 rex = regex
926 compiled = re.compile(rex, rflags)
928 def match_func(text, pos, endpos=sys.maxsize):
929 info = cls._prof_data[-1].setdefault((state, rex), [0, 0.0])
930 t0 = time.time()
931 res = compiled.match(text, pos, endpos)
932 t1 = time.time()
933 info[0] += 1
934 info[1] += t1 - t0
935 return res
936 return match_func
939class ProfilingRegexLexer(RegexLexer, metaclass=ProfilingRegexLexerMeta):
940 """Drop-in replacement for RegexLexer that does profiling of its regexes."""
942 _prof_data = []
943 _prof_sort_index = 4 # defaults to time per call
945 def get_tokens_unprocessed(self, text, stack=('root',)):
946 # this needs to be a stack, since using(this) will produce nested calls
947 self.__class__._prof_data.append({})
948 yield from RegexLexer.get_tokens_unprocessed(self, text, stack)
949 rawdata = self.__class__._prof_data.pop()
950 data = sorted(((s, repr(r).strip('u\'').replace('\\\\', '\\')[:65],
951 n, 1000 * t, 1000 * t / n)
952 for ((s, r), (n, t)) in rawdata.items()),
953 key=lambda x: x[self._prof_sort_index],
954 reverse=True)
955 sum_total = sum(x[3] for x in data)
957 print()
958 print('Profiling result for %s lexing %d chars in %.3f ms' %
959 (self.__class__.__name__, len(text), sum_total))
960 print('=' * 110)
961 print('%-20s %-64s ncalls tottime percall' % ('state', 'regex'))
962 print('-' * 110)
963 for d in data:
964 print('%-20s %-65s %5d %8.4f %8.4f' % d)
965 print('=' * 110)