Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/pycparser/c

1# ------------------------------------------------------------------------------

2# pycparser: c_lexer.py

4# CLexer class: lexer for the C language

6# Eli Bendersky [https://eli.thegreenplace.net/]

7# License: BSD

8# ------------------------------------------------------------------------------

9import re

10from collections.abc import Callable

11from dataclasses import dataclass

12from enum import Enum

13from typing import Literal, assert_never

15Dialect = Literal["legacy", "c23"]

18@dataclass(slots=True)

19class Token:

20 type: str

21 value: str

22 lineno: int

23 column: int

24 filename: str

27class CLexer:

28 """A standalone lexer for C.

30 Constructor parameters:

31 error_func:

32 Called with (msg, line, column) on lexing errors.

33 dialect:

34 "legacy" preserves the existing keyword set. "c23" also recognizes

35 C23 keywords and predefined constants.

37 Call input(text) to initialize lexing, and then keep calling token() to

38 get the next token, until it returns None (at end of input).

40 Non-keyword identifiers are emitted as ID tokens. The parser handles

41 scopes and classifies typedef names as TYPEID.

42 """

44 def __init__(

45 self,

46 error_func: Callable[[str, int, int], None],

47 *,

48 dialect: Dialect = "legacy",

49 ) -> None:

50 if dialect not in ("legacy", "c23"):

51 raise ValueError(f"Unknown dialect {dialect!r}; expected 'legacy' or 'c23'")

52 # Select keywords once per lexer. input() resets the scanning state

53 # without changing this map or affecting lexers using another dialect.

54 self._keyword_map = _keyword_map.copy()

55 if dialect == "c23":

56 self._keyword_map.update(_c23_keywords)

58 self.error_func = error_func

59 self._init_state()

61 def input(self, text: str, filename: str = "") -> None:

62 """Initialize the lexer to the given input text.

64 filename is an optional name identifying the initial file from which the

65 input comes. The lexer may modify it if #line directives are

66 encountered.

67 """

68 self._init_state()

69 self._lexdata = text

70 self._filename = filename

72 def _init_state(self) -> None:

73 self._lexdata = ""

74 self._filename = ""

75 self._pos = 0

76 self._line_start = 0

77 self._pending_tok: Token | None = None

78 self._lineno = 1

79 self._in_attribute = False

81 @property

82 def filename(self) -> str:

83 return self._filename

85 def token(self) -> Token | None:

86 # Lexing strategy overview:

87 #

88 # - We maintain a current position (self._pos), line number, and the

89 # byte offset of the current line start. The lexer is a simple loop

90 # that skips whitespace/newlines and emits one token per call.

91 #

92 # - A small amount of logic is handled manually before regex matching:

93 #

94 # * Preprocessor-style directives: if we see '#', we check whether

95 # it's a #line or #pragma directive and consume it inline. #line

96 # updates lineno/filename and produces no tokens. #pragma can yield

97 # both PPPRAGMA and PPPRAGMASTR, but token() returns a single token,

98 # so we stash the PPPRAGMASTR as _pending_tok to return on the next

99 # token() call. Otherwise we return PPHASH.

100 # * Newlines update lineno/line-start tracking so tokens can record

101 # accurate columns.

102 #

103 # - The bulk of tokens are recognized in _match_token:

104 #

105 # * _regex_rules: regex patterns for identifiers, literals, and other

106 # complex tokens (including error-producing patterns). The lexer

107 # uses a combined _regex_master to scan options at the same time.

108 # * _fixed_tokens: exact string matches for operators and punctuation,

109 # resolved by longest match.

110 #

111 # - Error patterns call the error callback and advance minimally, which

112 # keeps lexing resilient while reporting useful diagnostics.

113 text = self._lexdata

114 n = len(text)

115

116 if self._pending_tok is not None:

117 tok = self._pending_tok

118 self._pending_tok = None

119 return tok

120

121 while self._pos < n:

122 match text[self._pos]:

123 case " " | "\t" | "\f" | "\v":

124 self._pos += 1

125 case "\n":

126 self._lineno += 1

127 self._pos += 1

128 self._line_start = self._pos

129 case "#":

130 if _line_pattern.match(text, self._pos + 1):

131 self._pos += 1

132 self._handle_ppline()

133 continue

134 if _pragma_pattern.match(text, self._pos + 1):

135 self._pos += 1

136 toks = self._handle_pppragma()

137 if len(toks) > 1:

138 self._pending_tok = toks[1]

139 if len(toks) > 0:

140 return toks[0]

141 continue

142 tok = self._make_token("PPHASH", "#", self._pos)

143 self._pos += 1

144 return tok

145 case _:

146 if tok := self._match_token():

147 return tok

148 else:

149 continue

150

151 def _match_token(self) -> Token | None:

152 """Match one token at the current position.

153

154 Returns a Token on success, or None if no token could be matched and

155 an error was reported. This method always advances _pos by the matched

156 length, or by 1 on error/no-match.

157 """

158 text = self._lexdata

159 pos = self._pos

160 # Attribute arguments are balanced token sequences, not C expressions.

161 # Keep them opaque so the parser does not interpret their contents as

162 # C expressions or declarations.

163 if self._in_attribute:

164 if text[pos] == "(":

165 return self._attribute_arguments()

166 if match := _attribute_end.match(text, pos):

167 self._in_attribute = False

168 return self._attribute_token("ATTRIBUTE_END", match.end())

169 # Only run regexp check if we are starting an attribute, to avoid

170 # costly regex matching for every token.

171 elif text[pos] == "[" and (match := _attribute_begin.match(text, pos)):

172 self._in_attribute = True

173 return self._attribute_token("ATTRIBUTE_BEGIN", match.end())

174 # We pick the longest match between:

175 # - the master regex (identifiers, literals, error patterns, etc.)

176 # - fixed operator/punctuator literals from the bucket for text[pos]

177 #

178 # The longest match is required to ensure we properly lex something

179 # like ".123" (a floating-point constant) as a single entity (with

180 # FLOAT_CONST), rather than a PERIOD followed by a number.

181 #

182 # The fixed-literal buckets are already length-sorted, so within that

183 # bucket we can take the first match. However, we still compare its

184 # length to the regex match because the regex may have matched a longer

185 # token that should take precedence.

186 best = None

187

188 if m := _regex_master.match(text, pos):

189 tok_type = m.lastgroup

190 # All master-regex alternatives are named; lastgroup shouldn't be None.

191 assert tok_type is not None

192 value = m.group(tok_type)

193 length = len(value)

194 action, msg = _regex_actions[tok_type]

195 best = (length, tok_type, value, action, msg)

196

197 if bucket := _fixed_tokens_by_first.get(text[pos]):

198 for entry in bucket:

199 if text.startswith(entry.literal, pos):

200 length = len(entry.literal)

201 if best is None or length > best[0]:

202 best = (

203 length,

204 entry.tok_type,

205 entry.literal,

206 _RegexAction.TOKEN,

207 None,

208 )

209 break

210

211 if best is None:

212 self._error(f"Illegal character {text[pos]!r}", pos)

213 self._pos += 1

214 return None

215

216 length, tok_type, value, action, msg = best

217 match action:

218 case _RegexAction.TOKEN:

219 pass

220 case _RegexAction.ERROR:

221 if tok_type == "BAD_CHAR_CONST":

222 msg = f"Invalid char constant {value}"

223 # All other ERROR rules provide a message.

224 assert msg is not None

225 self._error(msg, pos)

226 self._pos += max(1, length)

227 return None

228 case _RegexAction.ID:

229 tok_type = (

230 "ID" if self._in_attribute else self._keyword_map.get(value, "ID")

231 )

232 case _:

233 assert_never(action)

234

235 tok = self._make_token(tok_type, value, pos)

236 self._pos += length

237

238 return tok

239

240 def _attribute_token(self, kind: str, end: int) -> Token:

241 """Emit the text from _pos up to end and advance lexer state.

242

243 Attribute delimiters can contain whitespace, and argument clauses can

244 span lines. Record the token's starting coordinates before updating

245 line tracking for the entire consumed span.

246 """

247 value = self._lexdata[self._pos : end]

248 tok = self._make_token(kind, value, self._pos)

249 self._lineno += value.count("\n")

250 if "\n" in value:

251 self._line_start = self._pos + value.rfind("\n") + 1

252 self._pos = end

253 return tok

254

255 def _attribute_arguments(self) -> Token | None:

256 """Scan an attribute argument clause with _pos at its opening '('.

257

258 Return one ATTRIBUTE_ARGUMENTS token containing the raw text inside

259 the outer parentheses. A stack tracks the expected closing delimiters

260 for nested (), [], and {}. Comments and quoted literals are skipped

261 while balancing, but preserved in the token value. This bypasses normal

262 token classification, keeping arbitrary attribute arguments separate

263 from C expressions and scope tracking.

264

265 Report malformed clauses at the opening parenthesis. If the error

266 callback returns, consume the remaining input and return None.

267 """

268 text = self._lexdata

269 start = self._pos

270 pos = start + 1

271 closers = [")"]

272 pairs = {"(": ")", "[": "]", "{": "}"}

273 error = "Unterminated attribute arguments"

274 while pos < len(text):

275 ch = text[pos]

276 if text.startswith("/*", pos):

277 end = text.find("*/", pos + 2)

278 if end < 0:

279 break

280 pos = end + 2

281 continue

282 if text.startswith("//", pos):

283 end = text.find("\n", pos + 2)

284 pos = len(text) if end < 0 else end

285 continue

286 # Consume whole identifiers and preprocessing numbers first, so

287 # digits in literal prefixes aren't confused with numbers and

288 # digit separators aren't mistaken for character-literal quotes.

289 if match := _attribute_word.match(text, pos):

290 pos = match.end()

291 continue

292 if ch in {"'", '"'}:

293 quote = ch

294 pos += 1

295 while pos < len(text) and text[pos] not in {quote, "\n"}:

296 pos += 2 if text[pos] == "\\" else 1

297 if pos >= len(text) or text[pos] != quote:

298 break

299 pos += 1

300 continue

301 if ch in pairs:

302 closers.append(pairs[ch])

303 elif ch in ")]}":

304 if ch != closers[-1]:

305 error = "Mismatched delimiter in attribute arguments"

306 break

307 closers.pop()

308 if not closers:

309 tok = self._attribute_token("ATTRIBUTE_ARGUMENTS", pos + 1)

310 tok.value = tok.value[1:-1]

311 return tok

312 pos += 1

313 self._error(error, start)

314 self._pos = len(text)

315 return None

316

317 def _make_token(self, tok_type: str, value: str, pos: int) -> Token:

318 """Create a Token at an absolute input position.

319

320 Expects tok_type/value and the absolute byte offset pos in the current

321 input. Does not advance lexer state; callers manage _pos themselves.

322 Returns a Token with the current filename and line/column coordinates.

323 """

324 column = pos - self._line_start + 1

325 tok = Token(tok_type, value, self._lineno, column, self._filename)

326 return tok

327

328 def _error(self, msg: str, pos: int) -> None:

329 column = pos - self._line_start + 1

330 self.error_func(msg, self._lineno, column)

331

332 def _handle_ppline(self) -> None:

333 # Since #line directives aren't supposed to return tokens but should

334 # only affect the lexer's state (update line/filename for coords), this

335 # method does a bit of parsing on its own. It doesn't return anything,

336 # but its side effect is to update self._pos past the directive, and

337 # potentially update self._lineno and self._filename, based on the

338 # directive's contents.

339 #

340 # Accepted #line forms from preprocessors:

341 # - "#line 66 \"kwas\\df.h\""

342 # - "# 9"

343 # - "#line 10 \"include/me.h\" 1 2 3" (extra numeric flags)

344 # - "# 1 \"file.h\" 3"

345 # Errors we must report:

346 # - "#line \"file.h\"" (filename before line number)

347 # - "#line df" (garbage instead of number/string)

348 #

349 # We scan the directive line once (after an optional 'line' keyword),

350 # validating the order: NUMBER, optional STRING, then any NUMBERs.

351 # The NUMBERs tail is only accepted if a filename STRING was present.

352 text = self._lexdata

353 n = len(text)

354 line_end = text.find("\n", self._pos)

355 if line_end == -1:

356 line_end = n

357 line = text[self._pos : line_end]

358 pos = 0

359 line_len = len(line)

360

361 def skip_ws() -> None:

362 nonlocal pos

363 while pos < line_len and line[pos] in " \t":

364 pos += 1

365

366 skip_ws()

367 if line.startswith("line", pos):

368 pos += 4

369

370 def success(pp_line: str | None, pp_filename: str | None) -> None:

371 if pp_line is None:

372 self._error("line number missing in #line", self._pos + line_len)

373 else:

374 try:

375 self._lineno = int(pp_line.replace("'", ""))

376 except ValueError:

377 self._error("invalid #line directive", self._pos + line_len)

378 return

379 if pp_filename is not None:

380 self._filename = pp_filename

381 self._pos = line_end + 1

382 self._line_start = self._pos

383

384 def fail(msg: str, offset: int) -> None:

385 self._error(msg, self._pos + offset)

386 self._pos = line_end + 1

387 self._line_start = self._pos

388

389 skip_ws()

390 if pos >= line_len:

391 success(None, None)

392 return

393 if line[pos] == '"':

394 fail("filename before line number in #line", pos)

395 return

396

397 # Line numbers allow digit separators, but no integer suffixes.

398 m = re.match(_decimal_digits, line[pos:])

399 if not m:

400 fail("invalid #line directive", pos)

401 return

402

403 pp_line = m.group(0)

404 pos += len(pp_line)

405 skip_ws()

406 if pos >= line_len:

407 success(pp_line, None)

408 return

409

410 if line[pos] != '"':

411 fail("invalid #line directive", pos)

412 return

413

414 m = re.match(_string_literal, line[pos:])

415 if not m:

416 fail("invalid #line directive", pos)

417 return

418

419 pp_filename = m.group(0).lstrip('"').rstrip('"')

420 pos += len(m.group(0))

421

422 # Consume arbitrary sequence of numeric flags after the directive

423 while True:

424 skip_ws()

425 if pos >= line_len:

426 break

427 m = re.match(_decimal_digits, line[pos:])

428 if not m:

429 fail("invalid #line directive", pos)

430 return

431 pos += len(m.group(0))

432

433 success(pp_line, pp_filename)

434

435 def _handle_pppragma(self) -> list[Token]:

436 # Parse a full #pragma line; returns a list of tokens with 1 or 2

437 # tokens - PPPRAGMA and an optional PPPRAGMASTR. If an empty list is

438 # returned, it means an error occurred, or we're at the end of input.

439 #

440 # Examples:

441 # - "#pragma" -> PPPRAGMA only

442 # - "#pragma once" -> PPPRAGMA, PPPRAGMASTR("once")

443 # - "# pragma omp parallel private(th_id)" -> PPPRAGMA, PPPRAGMASTR("omp parallel private(th_id)")

444 # - "#\tpragma {pack: 2, smack: 3}" -> PPPRAGMA, PPPRAGMASTR("{pack: 2, smack: 3}")

445 text = self._lexdata

446 n = len(text)

447 pos = self._pos

448

449 while pos < n and text[pos] in " \t":

450 pos += 1

451 if pos >= n:

452 self._pos = pos

453 return []

454

455 if not text.startswith("pragma", pos):

456 self._error("invalid #pragma directive", pos)

457 self._pos = pos + 1

458 return []

459

460 pragma_pos = pos

461 pos += len("pragma")

462 toks = [self._make_token("PPPRAGMA", "pragma", pragma_pos)]

463

464 while pos < n and text[pos] in " \t":

465 pos += 1

466

467 start = pos

468 while pos < n and text[pos] != "\n":

469 pos += 1

470 if pos > start:

471 toks.append(self._make_token("PPPRAGMASTR", text[start:pos], start))

472 if pos < n and text[pos] == "\n":

473 self._lineno += 1

474 pos += 1

475 self._line_start = pos

476 self._pos = pos

477 return toks

478

479

480##

481## Reserved keywords

482##

483_keywords: tuple[str, ...] = (

484 "AUTO",

485 "BREAK",

486 "CASE",

487 "CHAR",

488 "CONST",

489 "CONTINUE",

490 "DEFAULT",

491 "DO",

492 "DOUBLE",

493 "ELSE",

494 "ENUM",

495 "EXTERN",

496 "FLOAT",

497 "FOR",

498 "GOTO",

499 "IF",

500 "INLINE",

501 "INT",

502 "LONG",

503 "REGISTER",

504 "OFFSETOF",

505 "RESTRICT",

506 "RETURN",

507 "SHORT",

508 "SIGNED",

509 "SIZEOF",

510 "STATIC",

511 "STRUCT",

512 "SWITCH",

513 "TYPEDEF",

514 "UNION",

515 "UNSIGNED",

516 "VOID",

517 "VOLATILE",

518 "WHILE",

519 "__BUILTIN_VA_ARG",

520 "__INT128",

521 "_BOOL",

522 "_COMPLEX",

523 "_GENERIC",

524 "_NORETURN",

525 "_THREAD_LOCAL",

526 "_STATIC_ASSERT",

527 "_ATOMIC",

528 "_ALIGNOF",

529 "_ALIGNAS",

530 "_PRAGMA",

531)

532

533_keyword_map: dict[str, str] = {}

534

535for keyword in _keywords:

536 # Keywords from new C standard are mixed-case, like _Bool, _Alignas, etc.

537 if keyword.startswith("_") and len(keyword) > 1 and keyword[1].isalpha():

538 _keyword_map[keyword[:2].upper() + keyword[2:].lower()] = keyword

539 else:

540 _keyword_map[keyword.lower()] = keyword

541

542# Alternate spellings share token types with the existing grammar. The other

543# keywords are reserved even where the parser does not yet support their syntax.

544_c23_keywords = {

545 "alignas": "_ALIGNAS",

546 "alignof": "_ALIGNOF",

547 "bool": "_BOOL",

548 "static_assert": "_STATIC_ASSERT",

549 "thread_local": "_THREAD_LOCAL",

550 "true": "BOOL_CONST",

551 "false": "BOOL_CONST",

552 "nullptr": "NULLPTR",

553 "constexpr": "CONSTEXPR",

554 "typeof": "TYPEOF",

555 "typeof_unqual": "TYPEOF_UNQUAL",

556 "_BitInt": "_BITINT",

557 "_Decimal32": "_DECIMAL32",

558 "_Decimal64": "_DECIMAL64",

559 "_Decimal128": "_DECIMAL128",

560 "_Imaginary": "_IMAGINARY",

561}

562

563##

564## Regexes for use in tokens

565##

566

567# valid C identifiers (K&R2: A.2.3), plus '$' (supported by some compilers)

568_identifier = r"[a-zA-Z_$][0-9a-zA-Z_$]*"

569_attribute_begin = re.compile(r"\[\s*\[")

570_attribute_end = re.compile(r"\]\s*\]")

571_attribute_word = re.compile(rf"{_identifier}|(?:[0-9]|\.[0-9])(?:[eEpP][+-]|[\w.'])*")

572

573# Accept digit separators in all dialects, retaining them in token values.

574# A quote must be between two digits of the appropriate base; it cannot

575# touch a prefix, decimal point, exponent marker/sign, or suffix.

576_decimal_digits = "[0-9](?:'?[0-9])*"

577_hex_prefix = "0[xX]"

578_hex_digits = "[0-9a-fA-F](?:'?[0-9a-fA-F])*"

579_bin_prefix = "0[bB]"

580_bin_digits = "[01](?:'?[01])*"

581

582# integer constants (K&R2: A.2.5.1)

583# C23 wb/WB allows u/U before or after it, but no long suffix. Recognize it

584# in every dialect and let the parser enforce C23. Keep these alternatives

585# ahead of bare u/U so that an unsigned bit-precise suffix is consumed whole.

586_integer_suffix_opt = (

587 r"(([uU](wb|WB))|((wb|WB)[uU]?)|"

589)

590_decimal_constant = (

591 "(0" + _integer_suffix_opt + ")|([1-9](?:'?[0-9])*" + _integer_suffix_opt + ")"

592)

593_octal_constant = "0(?:'?[0-7])*" + _integer_suffix_opt

594_hex_constant = _hex_prefix + _hex_digits + _integer_suffix_opt

595_bin_constant = _bin_prefix + _bin_digits + _integer_suffix_opt

596

597_bad_octal_constant = "0(?:'?[0-7])*'?[89]"

598

599# comments are not supported

600_unsupported_c_style_comment = r"\/\*"

601_unsupported_cxx_style_comment = r"\/\/"

602

603# Character constants and escape sequences.

604#

605# Tokens preserve their source spelling; the lexer does not decode escapes or

606# validate Unicode code points. Each complete escape must match as one character

607# in _cconst_char, which is used for both single and multicharacter constants.

608# Matching is deliberately permissive: simple escapes accept extra letters and

609# punctuation, and numeric escapes accept decimal digits, not just octal digits.

610#

611# _cconst_char is also repeated in the error patterns. If an escape can match in

612# several ways, long malformed constants can cause exponential backtracking

613# (https://github.com/eliben/pycparser/issues/61). The lookaheads prevent this:

614# - Decimal and hexadecimal escapes consume the whole run of digits. For

615# example, \123 cannot be split into \1 followed by the characters 2 and 3.

616# - A simple \x escape is allowed only when no hexadecimal digit follows it.

617# - A complete Unicode escape cannot fall back to a simple \u or \U escape.

618# _bad_escape also excludes all decimal digits to agree with this permissiveness.

619

620# Universal character names consume exactly four hex digits after u or eight

621# after U. A following hex digit is a separate character, unlike with \x.

622_unicode_escape = r"(u[0-9a-fA-F]{4}|U[0-9a-fA-F]{8})"

623# This lookahead runs immediately after the backslash. If a complete Unicode

624# escape follows, only _unicode_escape may match it: \u00e9 cannot be read as

625# \u followed by 0, 0, e, 9. Incomplete forms retain the permissive behavior

626# of simple escapes.

627_simple_escape = (

628 rf"(?!{_unicode_escape})" r"""([a-wyzA-Z._~!=&\^\-\\?'"]|x(?![0-9a-fA-F]))"""

629)

630_decimal_escape = r"""(\d+)(?!\d)"""

631_hex_escape = r"""(x[0-9a-fA-F]+)(?![0-9a-fA-F])"""

632_bad_escape = r"""([\\][^a-zA-Z._~^!=&\^\-\\?'"x0-9])"""

633

634_escape_sequence = (

635 r"""(\\("""

636 + _simple_escape

637 + "|"

638 + _decimal_escape

639 + "|"

640 + _hex_escape

641 + "|"

642 + _unicode_escape

643 + "))"

644)

645

646# Strings do not need to count the characters represented by escapes. Match

647# just the backslash and a permitted following character; _string_char consumes

648# any remaining numeric or Unicode digits as ordinary text. This avoids matching

649# complete escape sequences repeatedly. The permissive spellings also support

650# Windows paths in #line filenames, which are matched with _string_literal.

651_escape_sequence_start_in_string = r"""(\\[0-9a-zA-Z._~!=&\^\-\\?'"])"""

652

653_cconst_char = r"""([^'\\\n]|""" + _escape_sequence + ")"

654_char_const = "'" + _cconst_char + "'"

655_wchar_const = "L" + _char_const

656_u8char_const = "u8" + _char_const

657_u16char_const = "u" + _char_const

658_u32char_const = "U" + _char_const

659_multicharacter_constant = "'" + _cconst_char + "{2,4}'"

660_unmatched_quote = "('" + _cconst_char + "*\\n)|('" + _cconst_char + "*$)"

661_bad_char_const = (

662 r"""('""" + _cconst_char + """[^'\n]+')|('')|('""" + _bad_escape + r"""[^'\n]*')"""

663)

664

665# String literals, including wide and Unicode prefixes.

666_string_char = r"""([^"\\\n]|""" + _escape_sequence_start_in_string + ")"

667_string_literal = '"' + _string_char + '*"'

668_wstring_literal = "L" + _string_literal

669_u8string_literal = "u8" + _string_literal

670_u16string_literal = "u" + _string_literal

671_u32string_literal = "U" + _string_literal

672_bad_string_literal = '"' + _string_char + "*" + _bad_escape + _string_char + '*"'

673

674# floating constants (K&R2: A.2.5.3)

675_exponent_part = "([eE][-+]?" + _decimal_digits + ")"

676_fractional_constant = (

677 "(("

678 + _decimal_digits

679 + r")?\."

680 + _decimal_digits

681 + ")|("

682 + _decimal_digits

683 + r"\.)"

684)

685_unsuffixed_floating_constant = (

686 "((("

687 + _fractional_constant

688 + ")"

689 + _exponent_part

690 + "?)|("

691 + _decimal_digits

692 + _exponent_part

693 + "))"

694)

695# Recognize C23 decimal suffixes in every dialect; the parser rejects them

696# outside C23. Each suffix must be entirely lowercase or uppercase.

697# The hexadecimal float pattern below keeps only f/F/l/L.

698_floating_constant = _unsuffixed_floating_constant + "(df|dd|dl|DF|DD|DL|[FfLl])?"

699_binary_exponent_part = "([pP][+-]?" + _decimal_digits + ")"

700_hex_fractional_constant = (

701 "(((" + _hex_digits + r""")?\.""" + _hex_digits + ")|(" + _hex_digits + r"""\.))"""

702)

703_hex_floating_constant = (

704 "("

705 + _hex_prefix

706 + "("

707 + _hex_digits

708 + "|"

709 + _hex_fractional_constant

710 + ")"

711 + _binary_exponent_part

712 + "[FfLl]?)"

713)

714

715

716class _RegexAction(Enum):

717 TOKEN = 0

718 ID = 1

719 ERROR = 2

720

721

722@dataclass(frozen=True)

723class _RegexRule:

724 # tok_type: name of the token emitted for a match

725 # regex_pattern: the raw regex (no anchors) to match at the current position

726 # action: TOKEN for normal tokens, ID for identifiers, ERROR to report

727 # error_message: message used for ERROR entries

728 tok_type: str

729 regex_pattern: str

730 action: _RegexAction

731 error_message: str | None

732

733

734_regex_rules: list[_RegexRule] = [

735 _RegexRule(

736 "UNSUPPORTED_C_STYLE_COMMENT",

737 _unsupported_c_style_comment,

738 _RegexAction.ERROR,

739 "Comments are not supported, see https://github.com/eliben/pycparser#3using.",

740 ),

741 _RegexRule(

742 "UNSUPPORTED_CXX_STYLE_COMMENT",

743 _unsupported_cxx_style_comment,

744 _RegexAction.ERROR,

745 "Comments are not supported, see https://github.com/eliben/pycparser#3using.",

746 ),

747 _RegexRule(

748 "BAD_STRING_LITERAL",

749 _bad_string_literal,

750 _RegexAction.ERROR,

751 "String contains invalid escape code",

752 ),

753 _RegexRule("WSTRING_LITERAL", _wstring_literal, _RegexAction.TOKEN, None),

754 _RegexRule("U8STRING_LITERAL", _u8string_literal, _RegexAction.TOKEN, None),

755 _RegexRule("U16STRING_LITERAL", _u16string_literal, _RegexAction.TOKEN, None),

756 _RegexRule("U32STRING_LITERAL", _u32string_literal, _RegexAction.TOKEN, None),

757 _RegexRule("STRING_LITERAL", _string_literal, _RegexAction.TOKEN, None),

758 _RegexRule("HEX_FLOAT_CONST", _hex_floating_constant, _RegexAction.TOKEN, None),

759 _RegexRule("FLOAT_CONST", _floating_constant, _RegexAction.TOKEN, None),

760 _RegexRule("INT_CONST_HEX", _hex_constant, _RegexAction.TOKEN, None),

761 _RegexRule("INT_CONST_BIN", _bin_constant, _RegexAction.TOKEN, None),

762 _RegexRule(

763 "BAD_CONST_OCT",

764 _bad_octal_constant,

765 _RegexAction.ERROR,

766 "Invalid octal constant",

767 ),

768 _RegexRule("INT_CONST_OCT", _octal_constant, _RegexAction.TOKEN, None),

769 _RegexRule("INT_CONST_DEC", _decimal_constant, _RegexAction.TOKEN, None),

770 _RegexRule("INT_CONST_CHAR", _multicharacter_constant, _RegexAction.TOKEN, None),

771 _RegexRule("CHAR_CONST", _char_const, _RegexAction.TOKEN, None),

772 _RegexRule("WCHAR_CONST", _wchar_const, _RegexAction.TOKEN, None),

773 _RegexRule("U8CHAR_CONST", _u8char_const, _RegexAction.TOKEN, None),

774 _RegexRule("U16CHAR_CONST", _u16char_const, _RegexAction.TOKEN, None),

775 _RegexRule("U32CHAR_CONST", _u32char_const, _RegexAction.TOKEN, None),

776 _RegexRule("UNMATCHED_QUOTE", _unmatched_quote, _RegexAction.ERROR, "Unmatched '"),

777 _RegexRule("BAD_CHAR_CONST", _bad_char_const, _RegexAction.ERROR, None),

778 _RegexRule("ID", _identifier, _RegexAction.ID, None),

779]

780

781_regex_actions: dict[str, tuple[_RegexAction, str | None]] = {}

782_regex_pattern_parts: list[str] = []

783for _rule in _regex_rules:

784 _regex_actions[_rule.tok_type] = (_rule.action, _rule.error_message)

785 _regex_pattern_parts.append(f"(?P<{_rule.tok_type}>{_rule.regex_pattern})")

786# The master regex is a single alternation of all token patterns, each wrapped

787# in a named group. We match once at the current position and then use

788# `lastgroup` to recover which token kind fired; this avoids iterating over all

789# regexes on every character while keeping the same token-level semantics.

790_regex_master: re.Pattern[str] = re.compile("|".join(_regex_pattern_parts))

791

792

793@dataclass(frozen=True)

794class _FixedToken:

795 tok_type: str

796 literal: str

797

798

799_fixed_tokens: list[_FixedToken] = [

800 _FixedToken("ELLIPSIS", "..."),

801 _FixedToken("LSHIFTEQUAL", "<<="),

802 _FixedToken("RSHIFTEQUAL", ">>="),

803 _FixedToken("PLUSPLUS", "++"),

804 _FixedToken("MINUSMINUS", "--"),

805 _FixedToken("ARROW", "->"),

806 _FixedToken("LAND", "&&"),

807 _FixedToken("LOR", "||"),

808 _FixedToken("LSHIFT", "<<"),

809 _FixedToken("RSHIFT", ">>"),

810 _FixedToken("LE", "<="),

811 _FixedToken("GE", ">="),

812 _FixedToken("EQ", "=="),

813 _FixedToken("NE", "!="),

814 _FixedToken("TIMESEQUAL", "*="),

815 _FixedToken("DIVEQUAL", "/="),

816 _FixedToken("MODEQUAL", "%="),

817 _FixedToken("PLUSEQUAL", "+="),

818 _FixedToken("MINUSEQUAL", "-="),

819 _FixedToken("ANDEQUAL", "&="),

820 _FixedToken("OREQUAL", "|="),

821 _FixedToken("XOREQUAL", "^="),

822 _FixedToken("EQUALS", "="),

823 _FixedToken("PLUS", "+"),

824 _FixedToken("MINUS", "-"),

825 _FixedToken("TIMES", "*"),

826 _FixedToken("DIVIDE", "/"),

827 _FixedToken("MOD", "%"),

828 _FixedToken("OR", "|"),

829 _FixedToken("AND", "&"),

830 _FixedToken("NOT", "~"),

831 _FixedToken("XOR", "^"),

832 _FixedToken("LNOT", "!"),

833 _FixedToken("LT", "<"),

834 _FixedToken("GT", ">"),

835 _FixedToken("CONDOP", "?"),

836 _FixedToken("LPAREN", "("),

837 _FixedToken("RPAREN", ")"),

838 _FixedToken("LBRACKET", "["),

839 _FixedToken("RBRACKET", "]"),

840 _FixedToken("LBRACE", "{"),

841 _FixedToken("RBRACE", "}"),

842 _FixedToken("COMMA", ","),

843 _FixedToken("PERIOD", "."),

844 _FixedToken("SEMI", ";"),

845 _FixedToken("COLON", ":"),

846 _FixedToken("COLONCOLON", "::"),

847]

848

849# To avoid scanning all fixed tokens on every character, we bucket them by the

850# first character. When matching at position i, we only look at the bucket for

851# text[i], and we pre-sort that bucket by token length so the first match is

852# also the longest. This preserves longest-match semantics (e.g. '>>=' before

853# '>>' before '>') while reducing the number of comparisons.

854_fixed_tokens_by_first: dict[str, list[_FixedToken]] = {}

855for _entry in _fixed_tokens:

856 _fixed_tokens_by_first.setdefault(_entry.literal[0], []).append(_entry)

857for _bucket in _fixed_tokens_by_first.values():

858 _bucket.sort(key=lambda item: len(item.literal), reverse=True)

859

860_line_pattern: re.Pattern[str] = re.compile(r"([ \t]*line\W)|([ \t]*\d+)")

861_pragma_pattern: re.Pattern[str] = re.compile(r"[ \t]*pragma\W")

Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/pycparser/c_lexer.py: 92%

362 statements