Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/lark/lexer.py: 68%

1# Lexer Implementation 

2 

3from abc import abstractmethod, ABC 

4import re 

5from typing import ( 

6 TypeVar, Type, Dict, Iterator, Collection, Callable, Optional, FrozenSet, Any, 

7 AnyStr, ClassVar, TYPE_CHECKING, overload 

8) 

9from types import ModuleType 

10import warnings 

11try: 

12 import interegular 

13except ImportError: 

14 pass 

15if TYPE_CHECKING: 

16 from .common import LexerConf 

17 from .parsers.lalr_parser_state import ParserState 

18 

19from .utils import classify, get_regexp_width, Serialize, logger, TextSlice, TextOrSlice 

20from .exceptions import UnexpectedCharacters, ConfigurationError, LexError, UnexpectedToken 

21from .grammar import TOKEN_DEFAULT_PRIORITY 

22 

23 

24###{standalone 

25from contextlib import suppress 

26from copy import copy 

27from dataclasses import dataclass 

28 

29try: # For the standalone parser, we need to make sure that has_interegular is False to avoid NameErrors later on 

30 has_interegular = bool(interegular) 

31except NameError: 

32 has_interegular = False 

33 

34class Pattern(Serialize, ABC): 

35 "An abstraction over regular expressions." 

36 

37 value: str 

38 flags: Collection[str] 

39 raw: Optional[str] 

40 type: ClassVar[str] 

41 

42 def __init__(self, value: str, flags: Collection[str] = (), raw: Optional[str] = None) -> None: 

43 self.value = value 

44 self.flags = frozenset(flags) 

45 self.raw = raw 

46 

47 def __repr__(self): 

48 return repr(self.to_regexp()) 

49 

50 # Pattern Hashing assumes all subclasses have a different priority! 

51 def __hash__(self): 

52 return hash((type(self), self.value, self.flags)) 

53 

54 def __eq__(self, other): 

55 return type(self) == type(other) and self.value == other.value and self.flags == other.flags 

56 

57 @abstractmethod 

58 def to_regexp(self) -> str: 

59 raise NotImplementedError() 

60 

61 @property 

62 @abstractmethod 

63 def min_width(self) -> int: 

64 raise NotImplementedError() 

65 

66 @property 

67 @abstractmethod 

68 def max_width(self) -> int: 

69 raise NotImplementedError() 

70 

71 def _get_flags(self, value): 

72 # Sorted to remove the hash randomization in sets, that differs between processes 

73 for f in sorted(self.flags): 

74 value = ('(?%s:%s)' % (f, value)) 

75 return value 

76 

77 

78class PatternStr(Pattern): 

79 __serialize_fields__ = 'value', 'flags', 'raw' 

80 

81 type: ClassVar[str] = "str" 

82 

83 def to_regexp(self) -> str: 

84 return self._get_flags(re.escape(self.value)) 

85 

86 @property 

87 def min_width(self) -> int: 

88 return len(self.value) 

89 

90 @property 

91 def max_width(self) -> int: 

92 return len(self.value) 

93 

94 

95class PatternRE(Pattern): 

96 __serialize_fields__ = 'value', 'flags', 'raw', '_width' 

97 

98 type: ClassVar[str] = "re" 

99 

100 def to_regexp(self) -> str: 

101 return self._get_flags(self.value) 

102 

103 _width = None 

104 def _get_width(self): 

105 if self._width is None: 

106 self._width = get_regexp_width(self.to_regexp()) 

107 return self._width 

108 

109 @property 

110 def min_width(self) -> int: 

111 return self._get_width()[0] 

112 

113 @property 

114 def max_width(self) -> int: 

115 return self._get_width()[1] 

116 

117 

118class TerminalDef(Serialize): 

119 "A definition of a terminal" 

120 __serialize_fields__ = 'name', 'pattern', 'priority' 

121 __serialize_namespace__ = PatternStr, PatternRE 

122 

123 name: str 

124 pattern: Pattern 

125 priority: int 

126 

127 def __init__(self, name: str, pattern: Pattern, priority: int = TOKEN_DEFAULT_PRIORITY) -> None: 

128 assert isinstance(pattern, Pattern), pattern 

129 self.name = name 

130 self.pattern = pattern 

131 self.priority = priority 

132 

133 def __repr__(self): 

134 return '%s(%r, %r)' % (type(self).__name__, self.name, self.pattern) 

135 

136 def user_repr(self) -> str: 

137 if self.name.startswith('__'): # We represent a generated terminal 

138 return self.pattern.raw or self.name 

139 else: 

140 return self.name 

141 

142_T = TypeVar('_T', bound="Token") 

143 

144class Token(str): 

145 """A string with meta-information, that is produced by the lexer. 

146 

147 When parsing text, the resulting chunks of the input that haven't been discarded, 

148 will end up in the tree as Token instances. The Token class inherits from Python's ``str``, 

149 so normal string comparisons and operations will work as expected. 

150 

151 Attributes: 

152 type: Name of the token (as specified in grammar) 

153 value: Value of the token (redundant, as ``token.value == token`` will always be true) 

154 start_pos: The index of the token in the text 

155 line: The line of the token in the text (starting with 1) 

156 column: The column of the token in the text (starting with 1) 

157 end_line: The line where the token ends 

158 end_column: The next column after the end of the token. For example, 

159 if the token is a single character with a column value of 4, 

160 end_column will be 5. 

161 end_pos: the index where the token ends (basically ``start_pos + len(token)``) 

162 """ 

163 __slots__ = ('type', 'start_pos', 'value', 'line', 'column', 'end_line', 'end_column', 'end_pos') 

164 

165 __match_args__ = ('type', 'value') 

166 

167 type: str 

168 start_pos: Optional[int] 

169 value: Any 

170 line: Optional[int] 

171 column: Optional[int] 

172 end_line: Optional[int] 

173 end_column: Optional[int] 

174 end_pos: Optional[int] 

175 

176 

177 @overload 

178 def __new__( 

179 cls, 

180 type: str, 

181 value: Any, 

182 start_pos: Optional[int] = None, 

183 line: Optional[int] = None, 

184 column: Optional[int] = None, 

185 end_line: Optional[int] = None, 

186 end_column: Optional[int] = None, 

187 end_pos: Optional[int] = None 

188 ) -> 'Token': 

189 ... 

190 

191 @overload 

192 def __new__( 

193 cls, 

194 type_: str, 

195 value: Any, 

196 start_pos: Optional[int] = None, 

197 line: Optional[int] = None, 

198 column: Optional[int] = None, 

199 end_line: Optional[int] = None, 

200 end_column: Optional[int] = None, 

201 end_pos: Optional[int] = None 

202 ) -> 'Token': ... 

203 

204 def __new__(cls, *args, **kwargs): 

205 if "type_" in kwargs: 

206 warnings.warn("`type_` is deprecated use `type` instead", DeprecationWarning) 

207 

208 if "type" in kwargs: 

209 raise TypeError("Error: using both 'type' and the deprecated 'type_' as arguments.") 

210 kwargs["type"] = kwargs.pop("type_") 

211 

212 return cls._future_new(*args, **kwargs) 

213 

214 

215 @classmethod 

216 def _future_new(cls, type, value, start_pos=None, line=None, column=None, end_line=None, end_column=None, end_pos=None): 

217 inst = super(Token, cls).__new__(cls, value) 

218 

219 inst.type = type 

220 inst.start_pos = start_pos 

221 inst.value = value 

222 inst.line = line 

223 inst.column = column 

224 inst.end_line = end_line 

225 inst.end_column = end_column 

226 inst.end_pos = end_pos 

227 return inst 

228 

229 @overload 

230 def update(self, type: Optional[str] = None, value: Optional[Any] = None) -> 'Token': 

231 ... 

232 

233 @overload 

234 def update(self, type_: Optional[str] = None, value: Optional[Any] = None) -> 'Token': 

235 ... 

236 

237 def update(self, *args, **kwargs): 

238 if "type_" in kwargs: 

239 warnings.warn("`type_` is deprecated use `type` instead", DeprecationWarning) 

240 

241 if "type" in kwargs: 

242 raise TypeError("Error: using both 'type' and the deprecated 'type_' as arguments.") 

243 kwargs["type"] = kwargs.pop("type_") 

244 

245 return self._future_update(*args, **kwargs) 

246 

247 def _future_update(self, type: Optional[str] = None, value: Optional[Any] = None) -> 'Token': 

248 return Token.new_borrow_pos( 

249 type if type is not None else self.type, 

250 value if value is not None else self.value, 

251 self 

252 ) 

253 

254 @classmethod 

255 def new_borrow_pos(cls: Type[_T], type_: str, value: Any, borrow_t: 'Token') -> _T: 

256 return cls(type_, value, borrow_t.start_pos, borrow_t.line, borrow_t.column, borrow_t.end_line, borrow_t.end_column, borrow_t.end_pos) 

257 

258 def __reduce__(self): 

259 return (self.__class__, (self.type, self.value, self.start_pos, self.line, self.column, 

260 self.end_line, self.end_column, self.end_pos)) 

261 

262 def __repr__(self): 

263 return 'Token(%r, %r)' % (self.type, self.value) 

264 

265 def __deepcopy__(self, memo): 

266 return Token(self.type, self.value, self.start_pos, self.line, self.column, 

267 self.end_line, self.end_column, self.end_pos) 

268 

269 def __eq__(self, other): 

270 if isinstance(other, Token) and self.type != other.type: 

271 return False 

272 

273 return str.__eq__(self, other) 

274 

275 __hash__ = str.__hash__ 

276 

277 

278@dataclass(frozen=True) 

279class _TextSlice_WithLineCount(TextSlice): 

280 """Internal: a TextSlice carrying the line/column state at its ``start``, so the lexer can 

281 resume position tracking without re-counting from offset 0. 

282 """ 

283 line: int 

284 line_start_pos: int 

285 

286 

287class LineCounter: 

288 "A utility class for keeping track of line & column information" 

289 

290 __slots__ = 'char_pos', 'line', 'column', 'line_start_pos', 'newline_char' 

291 

292 def __init__(self, newline_char): 

293 self.newline_char = newline_char 

294 self.char_pos = 0 

295 self.line = 1 

296 self.column = 1 

297 self.line_start_pos = 0 

298 

299 @classmethod 

300 def from_text_slice(cls, text_slice: TextSlice) -> 'LineCounter': 

301 """Build a counter positioned at ``text_slice.start``. Resumes from a snapshot when the 

302 slice carries one (``_TextSlice_WithLineCount``); otherwise counts the prefix once. 

303 """ 

304 self = cls(b'\n' if isinstance(text_slice.text, bytes) else '\n') 

305 if isinstance(text_slice, _TextSlice_WithLineCount): 

306 self.char_pos = text_slice.start 

307 self.line = text_slice.line 

308 self.line_start_pos = text_slice.line_start_pos 

309 self.column = text_slice.start - text_slice.line_start_pos + 1 

310 elif text_slice.start > 0: 

311 self.advance_to(text_slice.text, text_slice.start) 

312 return self 

313 

314 def __eq__(self, other): 

315 if not isinstance(other, LineCounter): 

316 return NotImplemented 

317 

318 return self.char_pos == other.char_pos and self.newline_char == other.newline_char 

319 

320 def feed(self, token: AnyStr, test_newline=True): 

321 """Consume a token and calculate the new line & column. 

322 

323 As an optional optimization, set test_newline=False if token doesn't contain a newline. 

324 """ 

325 if test_newline: 

326 newlines = token.count(self.newline_char) 

327 if newlines: 

328 self.line += newlines 

329 self.line_start_pos = self.char_pos + token.rindex(self.newline_char) + 1 

330 

331 self.char_pos += len(token) 

332 self.column = self.char_pos - self.line_start_pos + 1 

333 

334 def advance_to(self, text: AnyStr, pos: int): 

335 """Advance the counter to absolute offset ``pos`` within ``text``, counting the newlines 

336 """ 

337 newlines = text.count(self.newline_char, self.char_pos, pos) 

338 if newlines: 

339 self.line += newlines 

340 self.line_start_pos = text.rindex(self.newline_char, self.char_pos, pos) + 1 

341 self.char_pos = pos 

342 self.column = self.char_pos - self.line_start_pos + 1 

343 

344 

345class UnlessCallback: 

346 def __init__(self, scanner: 'Scanner'): 

347 self.scanner = scanner 

348 

349 def __call__(self, t: Token): 

350 res = self.scanner.fullmatch(t.value) 

351 if res is not None: 

352 t.type = res 

353 return t 

354 

355 

356class CallChain: 

357 def __init__(self, callback1, callback2, cond): 

358 self.callback1 = callback1 

359 self.callback2 = callback2 

360 self.cond = cond 

361 

362 def __call__(self, t): 

363 t2 = self.callback1(t) 

364 return self.callback2(t) if self.cond(t2) else t2 

365 

366 

367def _get_match(re_, regexp, s, flags): 

368 m = re_.match(regexp, s, flags) 

369 if m: 

370 return m.group(0) 

371 

372def _create_unless(terminals, g_regex_flags, re_, use_bytes): 

373 tokens_by_type = classify(terminals, lambda t: type(t.pattern)) 

374 assert len(tokens_by_type) <= 2, tokens_by_type.keys() 

375 embedded_strs = set() 

376 callback = {} 

377 for retok in tokens_by_type.get(PatternRE, []): 

378 unless = [] 

379 for strtok in tokens_by_type.get(PatternStr, []): 

380 if strtok.priority != retok.priority: 

381 continue 

382 s = strtok.pattern.value 

383 if s == _get_match(re_, retok.pattern.to_regexp(), s, g_regex_flags): 

384 unless.append(strtok) 

385 if strtok.pattern.flags <= retok.pattern.flags: 

386 embedded_strs.add(strtok) 

387 if unless: 

388 callback[retok.name] = UnlessCallback(Scanner(unless, g_regex_flags, re_, use_bytes=use_bytes)) 

389 

390 new_terminals = [t for t in terminals if t not in embedded_strs] 

391 return new_terminals, callback 

392 

393 

394class Scanner: 

395 def __init__(self, terminals, g_regex_flags, re_, use_bytes): 

396 self.terminals = terminals 

397 self.g_regex_flags = g_regex_flags 

398 self.re_ = re_ 

399 self.use_bytes = use_bytes 

400 

401 self.allowed_types = {t.name for t in self.terminals} 

402 

403 self._mres = self._build_mres(terminals, len(terminals)) 

404 

405 def _build_mres(self, terminals, max_size): 

406 # Python sets an unreasonable group limit (currently 100) in its re module 

407 # Worse, the only way to know we reached it is by catching an AssertionError! 

408 # This function recursively tries less and less groups until it's successful. 

409 mres = [] 

410 while terminals: 

411 pattern = u'|'.join(u'(?P<%s>%s)' % (t.name, t.pattern.to_regexp()) for t in terminals[:max_size]) 

412 if self.use_bytes: 

413 pattern = pattern.encode('latin-1') 

414 try: 

415 mre = self.re_.compile(pattern, self.g_regex_flags) 

416 except AssertionError: # Yes, this is what Python provides us.. :/ 

417 return self._build_mres(terminals, max_size // 2) 

418 

419 mres.append(mre) 

420 terminals = terminals[max_size:] 

421 return mres 

422 

423 def match(self, text: TextSlice, pos): 

424 for mre in self._mres: 

425 m = mre.match(text.text, pos, text.end) 

426 if m: 

427 return m.group(0), m.lastgroup 

428 

429 

430 def fullmatch(self, text: str) -> Optional[str]: 

431 for mre in self._mres: 

432 m = mre.fullmatch(text) 

433 if m: 

434 return m.lastgroup 

435 return None 

436 

437 def search(self, text: TextSlice, pos: int) -> Optional[int]: 

438 "Find the position of the earliest match, starting at pos" 

439 best = None 

440 for mre in self._mres: 

441 m = mre.search(text.text, pos, text.end) 

442 if m and (best is None or m.start() < best.start()): 

443 best = m 

444 return best.start() if best is not None else None 

445 

446def _regexp_has_newline(r: str): 

447 r"""Expressions that may indicate newlines in a regexp: 

448 - newlines (\n) 

449 - escaped newline (\\n) 

450 - anything but ([^...]) 

451 - any-char (.) when the flag (?s) exists 

452 - spaces (\s) 

453 """ 

454 return '\n' in r or '\\n' in r or '\\s' in r or '[^' in r or ('(?s' in r and '.' in r) 

455 

456 

457class LexerState: 

458 """Represents the current state of the lexer as it scans the text 

459 (Lexer objects are only instantiated per grammar, not per text) 

460 """ 

461 

462 __slots__ = 'text', 'line_ctr', 'last_token' 

463 

464 text: TextSlice 

465 line_ctr: LineCounter 

466 last_token: Optional[Token] 

467 

468 def __init__(self, text: TextSlice, line_ctr: Optional[LineCounter] = None, last_token: Optional[Token]=None): 

469 if isinstance(text, TextSlice): 

470 if line_ctr is None: 

471 line_ctr = LineCounter.from_text_slice(text) 

472 

473 if not (text.start <= line_ctr.char_pos <= text.end): 

474 raise ValueError("LineCounter.char_pos is out of bounds") 

475 

476 self.text = text 

477 self.line_ctr = line_ctr 

478 self.last_token = last_token 

479 

480 

481 def __eq__(self, other): 

482 if not isinstance(other, LexerState): 

483 return NotImplemented 

484 

485 return self.text == other.text and self.line_ctr == other.line_ctr and self.last_token == other.last_token 

486 

487 def __copy__(self): 

488 return type(self)(self.text, copy(self.line_ctr), self.last_token) 

489 

490 

491class LexerThread: 

492 """A thread that ties a lexer instance and a lexer state, to be used by the parser 

493 """ 

494 

495 def __init__(self, lexer: 'Lexer', lexer_state: Optional[LexerState]): 

496 self.lexer = lexer 

497 self.state = lexer_state 

498 

499 @classmethod 

500 def from_text(cls, lexer: 'Lexer', text_or_slice: TextOrSlice) -> 'LexerThread': 

501 text = TextSlice.cast_from(text_or_slice) 

502 return cls(lexer, LexerState(text)) 

503 

504 @classmethod 

505 def from_custom_input(cls, lexer: 'Lexer', text: Any) -> 'LexerThread': 

506 return cls(lexer, LexerState(text)) 

507 

508 def lex(self, parser_state): 

509 if self.state is None: 

510 raise TypeError("Cannot lex: No text assigned to lexer state") 

511 return self.lexer.lex(self.state, parser_state) 

512 

513 def __copy__(self): 

514 return type(self)(self.lexer, copy(self.state)) 

515 

516 _Token = Token 

517 

518 

519_Callback = Callable[[Token], Token] 

520 

521class Lexer(ABC): 

522 """Lexer interface 

523 

524 Method Signatures: 

525 lex(self, lexer_state, parser_state) -> Iterator[Token] 

526 """ 

527 @abstractmethod 

528 def lex(self, lexer_state: LexerState, parser_state: Any) -> Iterator[Token]: 

529 return NotImplemented 

530 

531 def search_start(self, text: TextSlice, start_state: Any, pos: int) -> Optional[int]: 

532 raise ConfigurationError("scan() is not supported by %s; use the built-in 'basic' or 'contextual' lexer" 

533 % type(self).__name__) 

534 

535 def make_lexer_state(self, text: str): 

536 "Deprecated" 

537 return LexerState(TextSlice.cast_from(text)) 

538 

539 

540def _check_regex_collisions(terminal_to_regexp: Dict[TerminalDef, str], comparator, strict_mode, max_collisions_to_show=8): 

541 if not comparator: 

542 comparator = interegular.Comparator.from_regexes(terminal_to_regexp) 

543 

544 # When in strict mode, we only ever try to provide one example, so taking 

545 # a long time for that should be fine 

546 max_time = 2 if strict_mode else 0.2 

547 

548 # We don't want to show too many collisions. 

549 if comparator.count_marked_pairs() >= max_collisions_to_show: 

550 return 

551 for group in classify(terminal_to_regexp, lambda t: t.priority).values(): 

552 for a, b in comparator.check(group, skip_marked=True): 

553 assert a.priority == b.priority 

554 # Mark this pair to not repeat warnings when multiple different BasicLexers see the same collision 

555 comparator.mark(a, b) 

556 

557 # Notify the user 

558 message = f"Collision between Terminals {a.name} and {b.name}. " 

559 try: 

560 example = comparator.get_example_overlap(a, b, max_time).format_multiline() 

561 except ValueError: 

562 # Couldn't find an example within max_time steps. 

563 example = "No example could be found fast enough. However, the collision does still exists" 

564 if strict_mode: 

565 raise LexError(f"{message}\n{example}") 

566 logger.warning("%s The lexer will choose between them arbitrarily.\n%s", message, example) 

567 if comparator.count_marked_pairs() >= max_collisions_to_show: 

568 logger.warning("Found 8 regex collisions, will not check for more.") 

569 return 

570 

571 

572class AbstractBasicLexer(Lexer): 

573 terminals_by_name: Dict[str, TerminalDef] 

574 

575 @abstractmethod 

576 def __init__(self, conf: 'LexerConf', comparator=None) -> None: 

577 ... 

578 

579 @abstractmethod 

580 def next_token(self, lex_state: LexerState, parser_state: Any = None) -> Token: 

581 ... 

582 

583 def lex(self, state: LexerState, parser_state: Any) -> Iterator[Token]: 

584 with suppress(EOFError): 

585 while True: 

586 yield self.next_token(state, parser_state) 

587 

588 

589class BasicLexer(AbstractBasicLexer): 

590 terminals: Collection[TerminalDef] 

591 ignore_types: FrozenSet[str] 

592 newline_types: FrozenSet[str] 

593 user_callbacks: Dict[str, _Callback] 

594 callback: Dict[str, _Callback] 

595 re: ModuleType 

596 

597 def __init__(self, conf: 'LexerConf', comparator=None) -> None: 

598 terminals = list(conf.terminals) 

599 assert all(isinstance(t, TerminalDef) for t in terminals), terminals 

600 

601 self.re = conf.re_module 

602 

603 if not conf.skip_validation: 

604 # Sanitization 

605 terminal_to_regexp = {} 

606 for t in terminals: 

607 regexp = t.pattern.to_regexp() 

608 try: 

609 self.re.compile(regexp, conf.g_regex_flags) 

610 except self.re.error: 

611 raise LexError("Cannot compile token %s: %s" % (t.name, t.pattern)) 

612 

613 if t.pattern.min_width == 0: 

614 raise LexError("Lexer does not allow zero-width terminals. (%s: %s)" % (t.name, t.pattern)) 

615 if t.pattern.type == "re": 

616 terminal_to_regexp[t] = regexp 

617 

618 if not (set(conf.ignore) <= {t.name for t in terminals}): 

619 raise LexError("Ignore terminals are not defined: %s" % (set(conf.ignore) - {t.name for t in terminals})) 

620 

621 if has_interegular: 

622 _check_regex_collisions(terminal_to_regexp, comparator, conf.strict) 

623 elif conf.strict: 

624 raise LexError("interegular must be installed for strict mode. Use `pip install 'lark[interegular]'`.") 

625 

626 # Init 

627 self.newline_types = frozenset(t.name for t in terminals if _regexp_has_newline(t.pattern.to_regexp())) 

628 self.ignore_types = frozenset(conf.ignore) 

629 

630 terminals.sort(key=lambda x: (-x.priority, -x.pattern.max_width, -len(x.pattern.value), x.name)) 

631 self.terminals = terminals 

632 self.user_callbacks = conf.callbacks 

633 self.g_regex_flags = conf.g_regex_flags 

634 self.use_bytes = conf.use_bytes 

635 self.terminals_by_name = conf.terminals_by_name 

636 

637 self._scanner: Optional[Scanner] = None 

638 self._search_scanner: Optional[Scanner] = None 

639 

640 def _build_scanner(self) -> Scanner: 

641 terminals, self.callback = _create_unless(self.terminals, self.g_regex_flags, self.re, self.use_bytes) 

642 assert all(self.callback.values()) 

643 

644 for type_, f in self.user_callbacks.items(): 

645 if type_ in self.callback: 

646 # Already a callback there, probably UnlessCallback. 

647 # Bind ``type_`` per iteration; otherwise every CallChain's 

648 # condition closes over the loop variable and checks the last 

649 # terminal's name, silently skipping the other callbacks. 

650 self.callback[type_] = CallChain( 

651 self.callback[type_], f, lambda t, type_=type_: t.type == type_ 

652 ) 

653 else: 

654 self.callback[type_] = f 

655 

656 return Scanner(terminals, self.g_regex_flags, self.re, self.use_bytes) 

657 

658 @property 

659 def scanner(self) -> Scanner: 

660 if self._scanner is None: 

661 self._scanner = self._build_scanner() 

662 return self._scanner 

663 

664 @property 

665 def search_scanner(self) -> Scanner: 

666 # Used by search_start(): a match can only begin with a non-ignored terminal, so we 

667 # search those directly. Searching all terminals and skipping ignores would jump past 

668 # a real start hiding inside an ignore's span (e.g. the "a" in an ignored "xxa"). 

669 if self._search_scanner is None: 

670 terminals = [t for t in self.terminals if t.name not in self.ignore_types] 

671 self._search_scanner = Scanner(terminals, self.g_regex_flags, self.re, self.use_bytes) 

672 return self._search_scanner 

673 

674 def match(self, text, pos): 

675 return self.scanner.match(text, pos) 

676 

677 def next_token(self, lex_state: LexerState, parser_state: Any = None) -> Token: 

678 line_ctr = lex_state.line_ctr 

679 while line_ctr.char_pos < lex_state.text.end: 

680 res = self.match(lex_state.text, line_ctr.char_pos) 

681 if not res: 

682 allowed = self.scanner.allowed_types - self.ignore_types 

683 if not allowed: 

684 allowed = {"<END-OF-FILE>"} 

685 raise UnexpectedCharacters(lex_state.text.text, line_ctr.char_pos, line_ctr.line, line_ctr.column, 

686 allowed=allowed, token_history=lex_state.last_token and [lex_state.last_token], 

687 state=parser_state, terminals_by_name=self.terminals_by_name) 

688 

689 value, type_ = res 

690 

691 ignored = type_ in self.ignore_types 

692 t = None 

693 if not ignored or type_ in self.callback: 

694 t = Token(type_, value, line_ctr.char_pos, line_ctr.line, line_ctr.column) 

695 line_ctr.feed(value, type_ in self.newline_types) 

696 if t is not None: 

697 t.end_line = line_ctr.line 

698 t.end_column = line_ctr.column 

699 t.end_pos = line_ctr.char_pos 

700 if t.type in self.callback: 

701 t = self.callback[t.type](t) 

702 if not ignored: 

703 if not isinstance(t, Token): 

704 raise LexError("Callbacks must return a token (returned %r)" % t) 

705 lex_state.last_token = t 

706 return t 

707 

708 # EOF 

709 raise EOFError(self) 

710 

711 def search_start(self, text: TextSlice, start_state: Any, pos: int) -> Optional[int]: 

712 return self.search_scanner.search(text, pos) 

713 

714 

715class ContextualLexer(Lexer): 

716 lexers: Dict[int, AbstractBasicLexer] 

717 root_lexer: AbstractBasicLexer 

718 

719 BasicLexer: Type[AbstractBasicLexer] = BasicLexer 

720 

721 def __init__(self, conf: 'LexerConf', states: Dict[int, Collection[str]], always_accept: Collection[str]=()) -> None: 

722 terminals = list(conf.terminals) 

723 terminals_by_name = conf.terminals_by_name 

724 

725 trad_conf = copy(conf) 

726 trad_conf.terminals = terminals 

727 

728 if has_interegular and not conf.skip_validation: 

729 comparator = interegular.Comparator.from_regexes({t: t.pattern.to_regexp() for t in terminals}) 

730 else: 

731 comparator = None 

732 lexer_by_tokens: Dict[FrozenSet[str], AbstractBasicLexer] = {} 

733 self.lexers = {} 

734 for state, accepts in states.items(): 

735 key = frozenset(accepts) 

736 try: 

737 lexer = lexer_by_tokens[key] 

738 except KeyError: 

739 accepts = set(accepts) | set(conf.ignore) | set(always_accept)