Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/pycparser/c_lexer.py: 92%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# ------------------------------------------------------------------------------
2# pycparser: c_lexer.py
3#
4# CLexer class: lexer for the C language
5#
6# Eli Bendersky [https://eli.thegreenplace.net/]
7# License: BSD
8# ------------------------------------------------------------------------------
9import re
10from dataclasses import dataclass
11from enum import Enum
12from typing import Callable, Dict, List, Optional, Tuple
15@dataclass(slots=True)
16class Token:
17 type: str
18 value: str
19 lineno: int
20 column: int
23class CLexer:
24 """A standalone lexer for C.
26 The lexer takes the following callback functions as parameters during
27 construction:
28 error_func:
29 Called with (msg, line, column) on lexing errors.
30 on_lbrace_func:
31 Called when an LBRACE token is produced (used for scope tracking).
32 on_rbrace_func:
33 Called when an RBRACE token is produced (used for scope tracking).
34 type_lookup_func:
35 Called with an identifier name; expected to return True if it is
36 a typedef name and should be tokenized as TYPEID.
38 Call input(text) to initialize lexing, and then keep calling token() to
39 get the next token, until it returns None (at end of input).
40 """
42 def __init__(
43 self,
44 error_func: Callable[[str, int, int], None],
45 on_lbrace_func: Callable[[], None],
46 on_rbrace_func: Callable[[], None],
47 type_lookup_func: Callable[[str], bool],
48 ) -> None:
49 self.error_func = error_func
50 self.on_lbrace_func = on_lbrace_func
51 self.on_rbrace_func = on_rbrace_func
52 self.type_lookup_func = type_lookup_func
53 self._init_state()
55 def input(self, text: str, filename: str = "") -> None:
56 """Initialize the lexer to the given input text.
58 filename is an optional name identifying the initial file from which the
59 input comes. The lexer may modify it if #line directives are
60 encountered.
61 """
62 self._init_state()
63 self._lexdata = text
64 self._filename = filename
66 def _init_state(self) -> None:
67 self._lexdata = ""
68 self._filename = ""
69 self._pos = 0
70 self._line_start = 0
71 self._pending_tok: Optional[Token] = None
72 self._lineno = 1
74 @property
75 def filename(self) -> str:
76 return self._filename
78 def token(self) -> Optional[Token]:
79 # Lexing strategy overview:
80 #
81 # - We maintain a current position (self._pos), line number, and the
82 # byte offset of the current line start. The lexer is a simple loop
83 # that skips whitespace/newlines and emits one token per call.
84 #
85 # - A small amount of logic is handled manually before regex matching:
86 #
87 # * Preprocessor-style directives: if we see '#', we check whether
88 # it's a #line or #pragma directive and consume it inline. #line
89 # updates lineno/filename and produces no tokens. #pragma can yield
90 # both PPPRAGMA and PPPRAGMASTR, but token() returns a single token,
91 # so we stash the PPPRAGMASTR as _pending_tok to return on the next
92 # token() call. Otherwise we return PPHASH.
93 # * Newlines update lineno/line-start tracking so tokens can record
94 # accurate columns.
95 #
96 # - The bulk of tokens are recognized in _match_token:
97 #
98 # * _regex_rules: regex patterns for identifiers, literals, and other
99 # complex tokens (including error-producing patterns). The lexer
100 # uses a combined _regex_master to scan options at the same time.
101 # * _fixed_tokens: exact string matches for operators and punctuation,
102 # resolved by longest match.
103 #
104 # - Error patterns call the error callback and advance minimally, which
105 # keeps lexing resilient while reporting useful diagnostics.
106 text = self._lexdata
107 n = len(text)
109 if self._pending_tok is not None:
110 tok = self._pending_tok
111 self._pending_tok = None
112 return tok
114 while self._pos < n:
115 match text[self._pos]:
116 case " " | "\t":
117 self._pos += 1
118 case "\n":
119 self._lineno += 1
120 self._pos += 1
121 self._line_start = self._pos
122 case "#":
123 if _line_pattern.match(text, self._pos + 1):
124 self._pos += 1
125 self._handle_ppline()
126 continue
127 if _pragma_pattern.match(text, self._pos + 1):
128 self._pos += 1
129 toks = self._handle_pppragma()
130 if len(toks) > 1:
131 self._pending_tok = toks[1]
132 if len(toks) > 0:
133 return toks[0]
134 continue
135 tok = self._make_token("PPHASH", "#", self._pos)
136 self._pos += 1
137 return tok
138 case _:
139 if tok := self._match_token():
140 return tok
141 else:
142 continue
144 def _match_token(self) -> Optional[Token]:
145 """Match one token at the current position.
147 Returns a Token on success, or None if no token could be matched and
148 an error was reported. This method always advances _pos by the matched
149 length, or by 1 on error/no-match.
150 """
151 text = self._lexdata
152 pos = self._pos
153 # We pick the longest match between:
154 # - the master regex (identifiers, literals, error patterns, etc.)
155 # - fixed operator/punctuator literals from the bucket for text[pos]
156 #
157 # The longest match is required to ensure we properly lex something
158 # like ".123" (a floating-point constant) as a single entity (with
159 # FLOAT_CONST), rather than a PERIOD followed by a number.
160 #
161 # The fixed-literal buckets are already length-sorted, so within that
162 # bucket we can take the first match. However, we still compare its
163 # length to the regex match because the regex may have matched a longer
164 # token that should take precedence.
165 best = None
167 if m := _regex_master.match(text, pos):
168 tok_type = m.lastgroup
169 # All master-regex alternatives are named; lastgroup shouldn't be None.
170 assert tok_type is not None
171 value = m.group(tok_type)
172 length = len(value)
173 action, msg = _regex_actions[tok_type]
174 best = (length, tok_type, value, action, msg)
176 if bucket := _fixed_tokens_by_first.get(text[pos]):
177 for entry in bucket:
178 if text.startswith(entry.literal, pos):
179 length = len(entry.literal)
180 if best is None or length > best[0]:
181 best = (
182 length,
183 entry.tok_type,
184 entry.literal,
185 _RegexAction.TOKEN,
186 None,
187 )
188 break
190 if best is None:
191 self._error(f"Illegal character {repr(text[pos])}", pos)
192 self._pos += 1
193 return None
195 length, tok_type, value, action, msg = best
196 match action:
197 case _RegexAction.TOKEN:
198 pass
199 case _RegexAction.ERROR:
200 if tok_type == "BAD_CHAR_CONST":
201 msg = f"Invalid char constant {value}"
202 # All other ERROR rules provide a message.
203 assert msg is not None
204 self._error(msg, pos)
205 self._pos += max(1, length)
206 return None
207 case _RegexAction.ID:
208 tok_type = _keyword_map.get(value, "ID")
209 if tok_type == "ID" and self.type_lookup_func(value):
210 tok_type = "TYPEID"
211 case _:
212 raise RuntimeError("unreachable")
214 tok = self._make_token(tok_type, value, pos)
215 self._pos += length
217 if tok.type == "LBRACE":
218 self.on_lbrace_func()
219 elif tok.type == "RBRACE":
220 self.on_rbrace_func()
222 return tok
224 def _make_token(self, tok_type: str, value: str, pos: int) -> Token:
225 """Create a Token at an absolute input position.
227 Expects tok_type/value and the absolute byte offset pos in the current
228 input. Does not advance lexer state; callers manage _pos themselves.
229 Returns a Token with lineno/column computed from current line tracking.
230 """
231 column = pos - self._line_start + 1
232 tok = Token(tok_type, value, self._lineno, column)
233 return tok
235 def _error(self, msg: str, pos: int) -> None:
236 column = pos - self._line_start + 1
237 self.error_func(msg, self._lineno, column)
239 def _handle_ppline(self) -> None:
240 # Since #line directives aren't supposed to return tokens but should
241 # only affect the lexer's state (update line/filename for coords), this
242 # method does a bit of parsing on its own. It doesn't return anything,
243 # but its side effect is to update self._pos past the directive, and
244 # potentially update self._lineno and self._filename, based on the
245 # directive's contents.
246 #
247 # Accepted #line forms from preprocessors:
248 # - "#line 66 \"kwas\\df.h\""
249 # - "# 9"
250 # - "#line 10 \"include/me.h\" 1 2 3" (extra numeric flags)
251 # - "# 1 \"file.h\" 3"
252 # Errors we must report:
253 # - "#line \"file.h\"" (filename before line number)
254 # - "#line df" (garbage instead of number/string)
255 #
256 # We scan the directive line once (after an optional 'line' keyword),
257 # validating the order: NUMBER, optional STRING, then any NUMBERs.
258 # The NUMBERs tail is only accepted if a filename STRING was present.
259 text = self._lexdata
260 n = len(text)
261 line_end = text.find("\n", self._pos)
262 if line_end == -1:
263 line_end = n
264 line = text[self._pos : line_end]
265 pos = 0
266 line_len = len(line)
268 def skip_ws() -> None:
269 nonlocal pos
270 while pos < line_len and line[pos] in " \t":
271 pos += 1
273 skip_ws()
274 if line.startswith("line", pos):
275 pos += 4
277 def success(pp_line: Optional[str], pp_filename: Optional[str]) -> None:
278 if pp_line is None:
279 self._error("line number missing in #line", self._pos + line_len)
280 else:
281 try:
282 self._lineno = int(pp_line)
283 except ValueError:
284 self._error("invalid #line directive", self._pos + line_len)
285 return
286 if pp_filename is not None:
287 self._filename = pp_filename
288 self._pos = line_end + 1
289 self._line_start = self._pos
291 def fail(msg: str, offset: int) -> None:
292 self._error(msg, self._pos + offset)
293 self._pos = line_end + 1
294 self._line_start = self._pos
296 skip_ws()
297 if pos >= line_len:
298 success(None, None)
299 return
300 if line[pos] == '"':
301 fail("filename before line number in #line", pos)
302 return
304 m = re.match(_decimal_constant, line[pos:])
305 if not m:
306 fail("invalid #line directive", pos)
307 return
309 pp_line = m.group(0)
310 pos += len(pp_line)
311 skip_ws()
312 if pos >= line_len:
313 success(pp_line, None)
314 return
316 if line[pos] != '"':
317 fail("invalid #line directive", pos)
318 return
320 m = re.match(_string_literal, line[pos:])
321 if not m:
322 fail("invalid #line directive", pos)
323 return
325 pp_filename = m.group(0).lstrip('"').rstrip('"')
326 pos += len(m.group(0))
328 # Consume arbitrary sequence of numeric flags after the directive
329 while True:
330 skip_ws()
331 if pos >= line_len:
332 break
333 m = re.match(_decimal_constant, line[pos:])
334 if not m:
335 fail("invalid #line directive", pos)
336 return
337 pos += len(m.group(0))
339 success(pp_line, pp_filename)
341 def _handle_pppragma(self) -> List[Token]:
342 # Parse a full #pragma line; returns a list of tokens with 1 or 2
343 # tokens - PPPRAGMA and an optional PPPRAGMASTR. If an empty list is
344 # returned, it means an error occurred, or we're at the end of input.
345 #
346 # Examples:
347 # - "#pragma" -> PPPRAGMA only
348 # - "#pragma once" -> PPPRAGMA, PPPRAGMASTR("once")
349 # - "# pragma omp parallel private(th_id)" -> PPPRAGMA, PPPRAGMASTR("omp parallel private(th_id)")
350 # - "#\tpragma {pack: 2, smack: 3}" -> PPPRAGMA, PPPRAGMASTR("{pack: 2, smack: 3}")
351 text = self._lexdata
352 n = len(text)
353 pos = self._pos
355 while pos < n and text[pos] in " \t":
356 pos += 1
357 if pos >= n:
358 self._pos = pos
359 return []
361 if not text.startswith("pragma", pos):
362 self._error("invalid #pragma directive", pos)
363 self._pos = pos + 1
364 return []
366 pragma_pos = pos
367 pos += len("pragma")
368 toks = [self._make_token("PPPRAGMA", "pragma", pragma_pos)]
370 while pos < n and text[pos] in " \t":
371 pos += 1
373 start = pos
374 while pos < n and text[pos] != "\n":
375 pos += 1
376 if pos > start:
377 toks.append(self._make_token("PPPRAGMASTR", text[start:pos], start))
378 if pos < n and text[pos] == "\n":
379 self._lineno += 1
380 pos += 1
381 self._line_start = pos
382 self._pos = pos
383 return toks
386##
387## Reserved keywords
388##
389_keywords: Tuple[str, ...] = (
390 "AUTO",
391 "BREAK",
392 "CASE",
393 "CHAR",
394 "CONST",
395 "CONTINUE",
396 "DEFAULT",
397 "DO",
398 "DOUBLE",
399 "ELSE",
400 "ENUM",
401 "EXTERN",
402 "FLOAT",
403 "FOR",
404 "GOTO",
405 "IF",
406 "INLINE",
407 "INT",
408 "LONG",
409 "REGISTER",
410 "OFFSETOF",
411 "RESTRICT",
412 "RETURN",
413 "SHORT",
414 "SIGNED",
415 "SIZEOF",
416 "STATIC",
417 "STRUCT",
418 "SWITCH",
419 "TYPEDEF",
420 "UNION",
421 "UNSIGNED",
422 "VOID",
423 "VOLATILE",
424 "WHILE",
425 "__INT128",
426 "_BOOL",
427 "_COMPLEX",
428 "_NORETURN",
429 "_THREAD_LOCAL",
430 "_STATIC_ASSERT",
431 "_ATOMIC",
432 "_ALIGNOF",
433 "_ALIGNAS",
434 "_PRAGMA",
435)
437_keyword_map: Dict[str, str] = {}
439for keyword in _keywords:
440 # Keywords from new C standard are mixed-case, like _Bool, _Alignas, etc.
441 if keyword.startswith("_") and len(keyword) > 1 and keyword[1].isalpha():
442 _keyword_map[keyword[:2].upper() + keyword[2:].lower()] = keyword
443 else:
444 _keyword_map[keyword.lower()] = keyword
446##
447## Regexes for use in tokens
448##
450# valid C identifiers (K&R2: A.2.3), plus '$' (supported by some compilers)
451_identifier = r"[a-zA-Z_$][0-9a-zA-Z_$]*"
453_hex_prefix = "0[xX]"
454_hex_digits = "[0-9a-fA-F]+"
455_bin_prefix = "0[bB]"
456_bin_digits = "[01]+"
458# integer constants (K&R2: A.2.5.1)
459_integer_suffix_opt = (
460 r"(([uU]ll)|([uU]LL)|(ll[uU]?)|(LL[uU]?)|([uU][lL])|([lL][uU]?)|[uU])?"
461)
462_decimal_constant = (
463 "(0" + _integer_suffix_opt + ")|([1-9][0-9]*" + _integer_suffix_opt + ")"
464)
465_octal_constant = "0[0-7]*" + _integer_suffix_opt
466_hex_constant = _hex_prefix + _hex_digits + _integer_suffix_opt
467_bin_constant = _bin_prefix + _bin_digits + _integer_suffix_opt
469_bad_octal_constant = "0[0-7]*[89]"
471# comments are not supported
472_unsupported_c_style_comment = r"\/\*"
473_unsupported_cxx_style_comment = r"\/\/"
475# character constants (K&R2: A.2.5.2)
476# Note: a-zA-Z and '.-~^_!=&;,' are allowed as escape chars to support #line
477# directives with Windows paths as filenames (..\..\dir\file)
478# For the same reason, decimal_escape allows all digit sequences. We want to
479# parse all correct code, even if it means to sometimes parse incorrect
480# code.
481#
482# The original regexes were taken verbatim from the C syntax definition,
483# and were later modified to avoid worst-case exponential running time.
484#
485# simple_escape = r"""([a-zA-Z._~!=&\^\-\\?'"])"""
486# decimal_escape = r"""(\d+)"""
487# hex_escape = r"""(x[0-9a-fA-F]+)"""
488# bad_escape = r"""([\\][^a-zA-Z._~^!=&\^\-\\?'"x0-7])"""
489#
490# The following modifications were made to avoid the ambiguity that allowed
491# backtracking: (https://github.com/eliben/pycparser/issues/61)
492#
493# - \x was removed from simple_escape, unless it was not followed by a hex
494# digit, to avoid ambiguity with hex_escape.
495# - hex_escape allows one or more hex characters, but requires that the next
496# character(if any) is not hex
497# - decimal_escape allows one or more decimal characters, but requires that the
498# next character(if any) is not a decimal
499# - bad_escape does not allow any decimals (8-9), to avoid conflicting with the
500# permissive decimal_escape.
501#
502# Without this change, python's `re` module would recursively try parsing each
503# ambiguous escape sequence in multiple ways. e.g. `\123` could be parsed as
504# `\1`+`23`, `\12`+`3`, and `\123`.
506_simple_escape = r"""([a-wyzA-Z._~!=&\^\-\\?'"]|x(?![0-9a-fA-F]))"""
507_decimal_escape = r"""(\d+)(?!\d)"""
508_hex_escape = r"""(x[0-9a-fA-F]+)(?![0-9a-fA-F])"""
509_bad_escape = r"""([\\][^a-zA-Z._~^!=&\^\-\\?'"x0-9])"""
511_escape_sequence = (
512 r"""(\\(""" + _simple_escape + "|" + _decimal_escape + "|" + _hex_escape + "))"
513)
515# This complicated regex with lookahead might be slow for strings, so because
516# all of the valid escapes (including \x) allowed
517# 0 or more non-escaped characters after the first character,
518# simple_escape+decimal_escape+hex_escape got simplified to
520_escape_sequence_start_in_string = r"""(\\[0-9a-zA-Z._~!=&\^\-\\?'"])"""
522_cconst_char = r"""([^'\\\n]|""" + _escape_sequence + ")"
523_char_const = "'" + _cconst_char + "'"
524_wchar_const = "L" + _char_const
525_u8char_const = "u8" + _char_const
526_u16char_const = "u" + _char_const
527_u32char_const = "U" + _char_const
528_multicharacter_constant = "'" + _cconst_char + "{2,4}'"
529_unmatched_quote = "('" + _cconst_char + "*\\n)|('" + _cconst_char + "*$)"
530_bad_char_const = (
531 r"""('""" + _cconst_char + """[^'\n]+')|('')|('""" + _bad_escape + r"""[^'\n]*')"""
532)
534# string literals (K&R2: A.2.6)
535_string_char = r"""([^"\\\n]|""" + _escape_sequence_start_in_string + ")"
536_string_literal = '"' + _string_char + '*"'
537_wstring_literal = "L" + _string_literal
538_u8string_literal = "u8" + _string_literal
539_u16string_literal = "u" + _string_literal
540_u32string_literal = "U" + _string_literal
541_bad_string_literal = '"' + _string_char + "*" + _bad_escape + _string_char + '*"'
543# floating constants (K&R2: A.2.5.3)
544_exponent_part = r"""([eE][-+]?[0-9]+)"""
545_fractional_constant = r"""([0-9]*\.[0-9]+)|([0-9]+\.)"""
546_floating_constant = (
547 "(((("
548 + _fractional_constant
549 + ")"
550 + _exponent_part
551 + "?)|([0-9]+"
552 + _exponent_part
553 + "))[FfLl]?)"
554)
555_binary_exponent_part = r"""([pP][+-]?[0-9]+)"""
556_hex_fractional_constant = (
557 "(((" + _hex_digits + r""")?\.""" + _hex_digits + ")|(" + _hex_digits + r"""\.))"""
558)
559_hex_floating_constant = (
560 "("
561 + _hex_prefix
562 + "("
563 + _hex_digits
564 + "|"
565 + _hex_fractional_constant
566 + ")"
567 + _binary_exponent_part
568 + "[FfLl]?)"
569)
572class _RegexAction(Enum):
573 TOKEN = 0
574 ID = 1
575 ERROR = 2
578@dataclass(frozen=True)
579class _RegexRule:
580 # tok_type: name of the token emitted for a match
581 # regex_pattern: the raw regex (no anchors) to match at the current position
582 # action: TOKEN for normal tokens, ID for identifiers, ERROR to report
583 # error_message: message used for ERROR entries
584 tok_type: str
585 regex_pattern: str
586 action: _RegexAction
587 error_message: Optional[str]
590_regex_rules: List[_RegexRule] = [
591 _RegexRule(
592 "UNSUPPORTED_C_STYLE_COMMENT",
593 _unsupported_c_style_comment,
594 _RegexAction.ERROR,
595 "Comments are not supported, see https://github.com/eliben/pycparser#3using.",
596 ),
597 _RegexRule(
598 "UNSUPPORTED_CXX_STYLE_COMMENT",
599 _unsupported_cxx_style_comment,
600 _RegexAction.ERROR,
601 "Comments are not supported, see https://github.com/eliben/pycparser#3using.",
602 ),
603 _RegexRule(
604 "BAD_STRING_LITERAL",
605 _bad_string_literal,
606 _RegexAction.ERROR,
607 "String contains invalid escape code",
608 ),
609 _RegexRule("WSTRING_LITERAL", _wstring_literal, _RegexAction.TOKEN, None),
610 _RegexRule("U8STRING_LITERAL", _u8string_literal, _RegexAction.TOKEN, None),
611 _RegexRule("U16STRING_LITERAL", _u16string_literal, _RegexAction.TOKEN, None),
612 _RegexRule("U32STRING_LITERAL", _u32string_literal, _RegexAction.TOKEN, None),
613 _RegexRule("STRING_LITERAL", _string_literal, _RegexAction.TOKEN, None),
614 _RegexRule("HEX_FLOAT_CONST", _hex_floating_constant, _RegexAction.TOKEN, None),
615 _RegexRule("FLOAT_CONST", _floating_constant, _RegexAction.TOKEN, None),
616 _RegexRule("INT_CONST_HEX", _hex_constant, _RegexAction.TOKEN, None),
617 _RegexRule("INT_CONST_BIN", _bin_constant, _RegexAction.TOKEN, None),
618 _RegexRule(
619 "BAD_CONST_OCT",
620 _bad_octal_constant,
621 _RegexAction.ERROR,
622 "Invalid octal constant",
623 ),
624 _RegexRule("INT_CONST_OCT", _octal_constant, _RegexAction.TOKEN, None),
625 _RegexRule("INT_CONST_DEC", _decimal_constant, _RegexAction.TOKEN, None),
626 _RegexRule("INT_CONST_CHAR", _multicharacter_constant, _RegexAction.TOKEN, None),
627 _RegexRule("CHAR_CONST", _char_const, _RegexAction.TOKEN, None),
628 _RegexRule("WCHAR_CONST", _wchar_const, _RegexAction.TOKEN, None),
629 _RegexRule("U8CHAR_CONST", _u8char_const, _RegexAction.TOKEN, None),
630 _RegexRule("U16CHAR_CONST", _u16char_const, _RegexAction.TOKEN, None),
631 _RegexRule("U32CHAR_CONST", _u32char_const, _RegexAction.TOKEN, None),
632 _RegexRule("UNMATCHED_QUOTE", _unmatched_quote, _RegexAction.ERROR, "Unmatched '"),
633 _RegexRule("BAD_CHAR_CONST", _bad_char_const, _RegexAction.ERROR, None),
634 _RegexRule("ID", _identifier, _RegexAction.ID, None),
635]
637_regex_actions: Dict[str, Tuple[_RegexAction, Optional[str]]] = {}
638_regex_pattern_parts: List[str] = []
639for _rule in _regex_rules:
640 _regex_actions[_rule.tok_type] = (_rule.action, _rule.error_message)
641 _regex_pattern_parts.append(f"(?P<{_rule.tok_type}>{_rule.regex_pattern})")
642# The master regex is a single alternation of all token patterns, each wrapped
643# in a named group. We match once at the current position and then use
644# `lastgroup` to recover which token kind fired; this avoids iterating over all
645# regexes on every character while keeping the same token-level semantics.
646_regex_master: re.Pattern[str] = re.compile("|".join(_regex_pattern_parts))
649@dataclass(frozen=True)
650class _FixedToken:
651 tok_type: str
652 literal: str
655_fixed_tokens: List[_FixedToken] = [
656 _FixedToken("ELLIPSIS", "..."),
657 _FixedToken("LSHIFTEQUAL", "<<="),
658 _FixedToken("RSHIFTEQUAL", ">>="),
659 _FixedToken("PLUSPLUS", "++"),
660 _FixedToken("MINUSMINUS", "--"),
661 _FixedToken("ARROW", "->"),
662 _FixedToken("LAND", "&&"),
663 _FixedToken("LOR", "||"),
664 _FixedToken("LSHIFT", "<<"),
665 _FixedToken("RSHIFT", ">>"),
666 _FixedToken("LE", "<="),
667 _FixedToken("GE", ">="),
668 _FixedToken("EQ", "=="),
669 _FixedToken("NE", "!="),
670 _FixedToken("TIMESEQUAL", "*="),
671 _FixedToken("DIVEQUAL", "/="),
672 _FixedToken("MODEQUAL", "%="),
673 _FixedToken("PLUSEQUAL", "+="),
674 _FixedToken("MINUSEQUAL", "-="),
675 _FixedToken("ANDEQUAL", "&="),
676 _FixedToken("OREQUAL", "|="),
677 _FixedToken("XOREQUAL", "^="),
678 _FixedToken("EQUALS", "="),
679 _FixedToken("PLUS", "+"),
680 _FixedToken("MINUS", "-"),
681 _FixedToken("TIMES", "*"),
682 _FixedToken("DIVIDE", "/"),
683 _FixedToken("MOD", "%"),
684 _FixedToken("OR", "|"),
685 _FixedToken("AND", "&"),
686 _FixedToken("NOT", "~"),
687 _FixedToken("XOR", "^"),
688 _FixedToken("LNOT", "!"),
689 _FixedToken("LT", "<"),
690 _FixedToken("GT", ">"),
691 _FixedToken("CONDOP", "?"),
692 _FixedToken("LPAREN", "("),
693 _FixedToken("RPAREN", ")"),
694 _FixedToken("LBRACKET", "["),
695 _FixedToken("RBRACKET", "]"),
696 _FixedToken("LBRACE", "{"),
697 _FixedToken("RBRACE", "}"),
698 _FixedToken("COMMA", ","),
699 _FixedToken("PERIOD", "."),
700 _FixedToken("SEMI", ";"),
701 _FixedToken("COLON", ":"),
702]
704# To avoid scanning all fixed tokens on every character, we bucket them by the
705# first character. When matching at position i, we only look at the bucket for
706# text[i], and we pre-sort that bucket by token length so the first match is
707# also the longest. This preserves longest-match semantics (e.g. '>>=' before
708# '>>' before '>') while reducing the number of comparisons.
709_fixed_tokens_by_first: Dict[str, List[_FixedToken]] = {}
710for _entry in _fixed_tokens:
711 _fixed_tokens_by_first.setdefault(_entry.literal[0], []).append(_entry)
712for _bucket in _fixed_tokens_by_first.values():
713 _bucket.sort(key=lambda item: len(item.literal), reverse=True)
715_line_pattern: re.Pattern[str] = re.compile(r"([ \t]*line\W)|([ \t]*\d+)")
716_pragma_pattern: re.Pattern[str] = re.compile(r"[ \t]*pragma\W")