Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/lark/lexer.py: 68%

1# Lexer Implementation 

2 

3from abc import abstractmethod, ABC 

4import re 

5from typing import ( 

6 TypeVar, Type, Dict, Iterator, Collection, Callable, Optional, FrozenSet, Any, 

7 AnyStr, ClassVar, TYPE_CHECKING, overload 

8) 

9from types import ModuleType 

10import warnings 

11try: 

12 import interegular 

13except ImportError: 

14 pass 

15if TYPE_CHECKING: 

16 from .common import LexerConf 

17 from .parsers.lalr_parser_state import ParserState 

18 

19from .utils import classify, get_regexp_width, Serialize, logger, TextSlice, TextOrSlice 

20from .exceptions import UnexpectedCharacters, ConfigurationError, LexError, UnexpectedToken 

21from .grammar import TOKEN_DEFAULT_PRIORITY 

22 

23 

24###{standalone 

25from contextlib import suppress 

26from copy import copy 

27from dataclasses import dataclass 

28 

29try: # For the standalone parser, we need to make sure that has_interegular is False to avoid NameErrors later on 

30 has_interegular = bool(interegular) 

31except NameError: 

32 has_interegular = False 

33 

34class Pattern(Serialize, ABC): 

35 "An abstraction over regular expressions." 

36 

37 value: str 

38 flags: Collection[str] 

39 raw: Optional[str] 

40 type: ClassVar[str] 

41 

42 def __init__(self, value: str, flags: Collection[str] = (), raw: Optional[str] = None) -> None: 

43 self.value = value 

44 self.flags = frozenset(flags) 

45 self.raw = raw 

46 

47 def __repr__(self): 

48 return repr(self.to_regexp()) 

49 

50 # Pattern Hashing assumes all subclasses have a different priority! 

51 def __hash__(self): 

52 return hash((type(self), self.value, self.flags)) 

53 

54 def __eq__(self, other): 

55 return type(self) == type(other) and self.value == other.value and self.flags == other.flags 

56 

57 @abstractmethod 

58 def to_regexp(self) -> str: 

59 raise NotImplementedError() 

60 

61 @property 

62 @abstractmethod 

63 def min_width(self) -> int: 

64 raise NotImplementedError() 

65 

66 @property 

67 @abstractmethod 

68 def max_width(self) -> int: 

69 raise NotImplementedError() 

70 

71 def _get_flags(self, value): 

72 for f in self.flags: 

73 value = ('(?%s:%s)' % (f, value)) 

74 return value 

75 

76 

77class PatternStr(Pattern): 

78 __serialize_fields__ = 'value', 'flags', 'raw' 

79 

80 type: ClassVar[str] = "str" 

81 

82 def to_regexp(self) -> str: 

83 return self._get_flags(re.escape(self.value)) 

84 

85 @property 

86 def min_width(self) -> int: 

87 return len(self.value) 

88 

89 @property 

90 def max_width(self) -> int: 

91 return len(self.value) 

92 

93 

94class PatternRE(Pattern): 

95 __serialize_fields__ = 'value', 'flags', 'raw', '_width' 

96 

97 type: ClassVar[str] = "re" 

98 

99 def to_regexp(self) -> str: 

100 return self._get_flags(self.value) 

101 

102 _width = None 

103 def _get_width(self): 

104 if self._width is None: 

105 self._width = get_regexp_width(self.to_regexp()) 

106 return self._width 

107 

108 @property 

109 def min_width(self) -> int: 

110 return self._get_width()[0] 

111 

112 @property 

113 def max_width(self) -> int: 

114 return self._get_width()[1] 

115 

116 

117class TerminalDef(Serialize): 

118 "A definition of a terminal" 

119 __serialize_fields__ = 'name', 'pattern', 'priority' 

120 __serialize_namespace__ = PatternStr, PatternRE 

121 

122 name: str 

123 pattern: Pattern 

124 priority: int 

125 

126 def __init__(self, name: str, pattern: Pattern, priority: int = TOKEN_DEFAULT_PRIORITY) -> None: 

127 assert isinstance(pattern, Pattern), pattern 

128 self.name = name 

129 self.pattern = pattern 

130 self.priority = priority 

131 

132 def __repr__(self): 

133 return '%s(%r, %r)' % (type(self).__name__, self.name, self.pattern) 

134 

135 def user_repr(self) -> str: 

136 if self.name.startswith('__'): # We represent a generated terminal 

137 return self.pattern.raw or self.name 

138 else: 

139 return self.name 

140 

141_T = TypeVar('_T', bound="Token") 

142 

143class Token(str): 

144 """A string with meta-information, that is produced by the lexer. 

145 

146 When parsing text, the resulting chunks of the input that haven't been discarded, 

147 will end up in the tree as Token instances. The Token class inherits from Python's ``str``, 

148 so normal string comparisons and operations will work as expected. 

149 

150 Attributes: 

151 type: Name of the token (as specified in grammar) 

152 value: Value of the token (redundant, as ``token.value == token`` will always be true) 

153 start_pos: The index of the token in the text 

154 line: The line of the token in the text (starting with 1) 

155 column: The column of the token in the text (starting with 1) 

156 end_line: The line where the token ends 

157 end_column: The next column after the end of the token. For example, 

158 if the token is a single character with a column value of 4, 

159 end_column will be 5. 

160 end_pos: the index where the token ends (basically ``start_pos + len(token)``) 

161 """ 

162 __slots__ = ('type', 'start_pos', 'value', 'line', 'column', 'end_line', 'end_column', 'end_pos') 

163 

164 __match_args__ = ('type', 'value') 

165 

166 type: str 

167 start_pos: Optional[int] 

168 value: Any 

169 line: Optional[int] 

170 column: Optional[int] 

171 end_line: Optional[int] 

172 end_column: Optional[int] 

173 end_pos: Optional[int] 

174 

175 

176 @overload 

177 def __new__( 

178 cls, 

179 type: str, 

180 value: Any, 

181 start_pos: Optional[int] = None, 

182 line: Optional[int] = None, 

183 column: Optional[int] = None, 

184 end_line: Optional[int] = None, 

185 end_column: Optional[int] = None, 

186 end_pos: Optional[int] = None 

187 ) -> 'Token': 

188 ... 

189 

190 @overload 

191 def __new__( 

192 cls, 

193 type_: str, 

194 value: Any, 

195 start_pos: Optional[int] = None, 

196 line: Optional[int] = None, 

197 column: Optional[int] = None, 

198 end_line: Optional[int] = None, 

199 end_column: Optional[int] = None, 

200 end_pos: Optional[int] = None 

201 ) -> 'Token': ... 

202 

203 def __new__(cls, *args, **kwargs): 

204 if "type_" in kwargs: 

205 warnings.warn("`type_` is deprecated use `type` instead", DeprecationWarning) 

206 

207 if "type" in kwargs: 

208 raise TypeError("Error: using both 'type' and the deprecated 'type_' as arguments.") 

209 kwargs["type"] = kwargs.pop("type_") 

210 

211 return cls._future_new(*args, **kwargs) 

212 

213 

214 @classmethod 

215 def _future_new(cls, type, value, start_pos=None, line=None, column=None, end_line=None, end_column=None, end_pos=None): 

216 inst = super(Token, cls).__new__(cls, value) 

217 

218 inst.type = type 

219 inst.start_pos = start_pos 

220 inst.value = value 

221 inst.line = line 

222 inst.column = column 

223 inst.end_line = end_line 

224 inst.end_column = end_column 

225 inst.end_pos = end_pos 

226 return inst 

227 

228 @overload 

229 def update(self, type: Optional[str] = None, value: Optional[Any] = None) -> 'Token': 

230 ... 

231 

232 @overload 

233 def update(self, type_: Optional[str] = None, value: Optional[Any] = None) -> 'Token': 

234 ... 

235 

236 def update(self, *args, **kwargs): 

237 if "type_" in kwargs: 

238 warnings.warn("`type_` is deprecated use `type` instead", DeprecationWarning) 

239 

240 if "type" in kwargs: 

241 raise TypeError("Error: using both 'type' and the deprecated 'type_' as arguments.") 

242 kwargs["type"] = kwargs.pop("type_") 

243 

244 return self._future_update(*args, **kwargs) 

245 

246 def _future_update(self, type: Optional[str] = None, value: Optional[Any] = None) -> 'Token': 

247 return Token.new_borrow_pos( 

248 type if type is not None else self.type, 

249 value if value is not None else self.value, 

250 self 

251 ) 

252 

253 @classmethod 

254 def new_borrow_pos(cls: Type[_T], type_: str, value: Any, borrow_t: 'Token') -> _T: 

255 return cls(type_, value, borrow_t.start_pos, borrow_t.line, borrow_t.column, borrow_t.end_line, borrow_t.end_column, borrow_t.end_pos) 

256 

257 def __reduce__(self): 

258 return (self.__class__, (self.type, self.value, self.start_pos, self.line, self.column, 

259 self.end_line, self.end_column, self.end_pos)) 

260 

261 def __repr__(self): 

262 return 'Token(%r, %r)' % (self.type, self.value) 

263 

264 def __deepcopy__(self, memo): 

265 return Token(self.type, self.value, self.start_pos, self.line, self.column, 

266 self.end_line, self.end_column, self.end_pos) 

267 

268 def __eq__(self, other): 

269 if isinstance(other, Token) and self.type != other.type: 

270 return False 

271 

272 return str.__eq__(self, other) 

273 

274 __hash__ = str.__hash__ 

275 

276 

277@dataclass(frozen=True) 

278class _TextSlice_WithLineCount(TextSlice): 

279 """Internal: a TextSlice carrying the line/column state at its ``start``, so the lexer can 

280 resume position tracking without re-counting from offset 0. 

281 """ 

282 line: int 

283 line_start_pos: int 

284 

285 

286class LineCounter: 

287 "A utility class for keeping track of line & column information" 

288 

289 __slots__ = 'char_pos', 'line', 'column', 'line_start_pos', 'newline_char' 

290 

291 def __init__(self, newline_char): 

292 self.newline_char = newline_char 

293 self.char_pos = 0 

294 self.line = 1 

295 self.column = 1 

296 self.line_start_pos = 0 

297 

298 @classmethod 

299 def from_text_slice(cls, text_slice: TextSlice) -> 'LineCounter': 

300 """Build a counter positioned at ``text_slice.start``. Resumes from a snapshot when the 

301 slice carries one (``_TextSlice_WithLineCount``); otherwise counts the prefix once. 

302 """ 

303 self = cls(b'\n' if isinstance(text_slice.text, bytes) else '\n') 

304 if isinstance(text_slice, _TextSlice_WithLineCount): 

305 self.char_pos = text_slice.start 

306 self.line = text_slice.line 

307 self.line_start_pos = text_slice.line_start_pos 

308 self.column = text_slice.start - text_slice.line_start_pos + 1 

309 elif text_slice.start > 0: 

310 self.advance_to(text_slice.text, text_slice.start) 

311 return self 

312 

313 def __eq__(self, other): 

314 if not isinstance(other, LineCounter): 

315 return NotImplemented 

316 

317 return self.char_pos == other.char_pos and self.newline_char == other.newline_char 

318 

319 def feed(self, token: AnyStr, test_newline=True): 

320 """Consume a token and calculate the new line & column. 

321 

322 As an optional optimization, set test_newline=False if token doesn't contain a newline. 

323 """ 

324 if test_newline: 

325 newlines = token.count(self.newline_char) 

326 if newlines: 

327 self.line += newlines 

328 self.line_start_pos = self.char_pos + token.rindex(self.newline_char) + 1 

329 

330 self.char_pos += len(token) 

331 self.column = self.char_pos - self.line_start_pos + 1 

332 

333 def advance_to(self, text: AnyStr, pos: int): 

334 """Advance the counter to absolute offset ``pos`` within ``text``, counting the newlines 

335 """ 

336 newlines = text.count(self.newline_char, self.char_pos, pos) 

337 if newlines: 

338 self.line += newlines 

339 self.line_start_pos = text.rindex(self.newline_char, self.char_pos, pos) + 1 

340 self.char_pos = pos 

341 self.column = self.char_pos - self.line_start_pos + 1 

342 

343 

344class UnlessCallback: 

345 def __init__(self, scanner: 'Scanner'): 

346 self.scanner = scanner 

347 

348 def __call__(self, t: Token): 

349 res = self.scanner.fullmatch(t.value) 

350 if res is not None: 

351 t.type = res 

352 return t 

353 

354 

355class CallChain: 

356 def __init__(self, callback1, callback2, cond): 

357 self.callback1 = callback1 

358 self.callback2 = callback2 

359 self.cond = cond 

360 

361 def __call__(self, t): 

362 t2 = self.callback1(t) 

363 return self.callback2(t) if self.cond(t2) else t2 

364 

365 

366def _get_match(re_, regexp, s, flags): 

367 m = re_.match(regexp, s, flags) 

368 if m: 

369 return m.group(0) 

370 

371def _create_unless(terminals, g_regex_flags, re_, use_bytes): 

372 tokens_by_type = classify(terminals, lambda t: type(t.pattern)) 

373 assert len(tokens_by_type) <= 2, tokens_by_type.keys() 

374 embedded_strs = set() 

375 callback = {} 

376 for retok in tokens_by_type.get(PatternRE, []): 

377 unless = [] 

378 for strtok in tokens_by_type.get(PatternStr, []): 

379 if strtok.priority != retok.priority: 

380 continue 

381 s = strtok.pattern.value 

382 if s == _get_match(re_, retok.pattern.to_regexp(), s, g_regex_flags): 

383 unless.append(strtok) 

384 if strtok.pattern.flags <= retok.pattern.flags: 

385 embedded_strs.add(strtok) 

386 if unless: 

387 callback[retok.name] = UnlessCallback(Scanner(unless, g_regex_flags, re_, use_bytes=use_bytes)) 

388 

389 new_terminals = [t for t in terminals if t not in embedded_strs] 

390 return new_terminals, callback 

391 

392 

393class Scanner: 

394 def __init__(self, terminals, g_regex_flags, re_, use_bytes): 

395 self.terminals = terminals 

396 self.g_regex_flags = g_regex_flags 

397 self.re_ = re_ 

398 self.use_bytes = use_bytes 

399 

400 self.allowed_types = {t.name for t in self.terminals} 

401 

402 self._mres = self._build_mres(terminals, len(terminals)) 

403 

404 def _build_mres(self, terminals, max_size): 

405 # Python sets an unreasonable group limit (currently 100) in its re module 

406 # Worse, the only way to know we reached it is by catching an AssertionError! 

407 # This function recursively tries less and less groups until it's successful. 

408 mres = [] 

409 while terminals: 

410 pattern = u'|'.join(u'(?P<%s>%s)' % (t.name, t.pattern.to_regexp()) for t in terminals[:max_size]) 

411 if self.use_bytes: 

412 pattern = pattern.encode('latin-1') 

413 try: 

414 mre = self.re_.compile(pattern, self.g_regex_flags) 

415 except AssertionError: # Yes, this is what Python provides us.. :/ 

416 return self._build_mres(terminals, max_size // 2) 

417 

418 mres.append(mre) 

419 terminals = terminals[max_size:] 

420 return mres 

421 

422 def match(self, text: TextSlice, pos): 

423 for mre in self._mres: 

424 m = mre.match(text.text, pos, text.end) 

425 if m: 

426 return m.group(0), m.lastgroup 

427 

428 

429 def fullmatch(self, text: str) -> Optional[str]: 

430 for mre in self._mres: 

431 m = mre.fullmatch(text) 

432 if m: 

433 return m.lastgroup 

434 return None 

435 

436 def search(self, text: TextSlice, pos: int) -> Optional[int]: 

437 "Find the position of the earliest match, starting at pos" 

438 best = None 

439 for mre in self._mres: 

440 m = mre.search(text.text, pos, text.end) 

441 if m and (best is None or m.start() < best.start()): 

442 best = m 

443 return best.start() if best is not None else None 

444 

445def _regexp_has_newline(r: str): 

446 r"""Expressions that may indicate newlines in a regexp: 

447 - newlines (\n) 

448 - escaped newline (\\n) 

449 - anything but ([^...]) 

450 - any-char (.) when the flag (?s) exists 

451 - spaces (\s) 

452 """ 

453 return '\n' in r or '\\n' in r or '\\s' in r or '[^' in r or ('(?s' in r and '.' in r) 

454 

455 

456class LexerState: 

457 """Represents the current state of the lexer as it scans the text 

458 (Lexer objects are only instantiated per grammar, not per text) 

459 """ 

460 

461 __slots__ = 'text', 'line_ctr', 'last_token' 

462 

463 text: TextSlice 

464 line_ctr: LineCounter 

465 last_token: Optional[Token] 

466 

467 def __init__(self, text: TextSlice, line_ctr: Optional[LineCounter] = None, last_token: Optional[Token]=None): 

468 if isinstance(text, TextSlice): 

469 if line_ctr is None: 

470 line_ctr = LineCounter.from_text_slice(text) 

471 

472 if not (text.start <= line_ctr.char_pos <= text.end): 

473 raise ValueError("LineCounter.char_pos is out of bounds") 

474 

475 self.text = text 

476 self.line_ctr = line_ctr 

477 self.last_token = last_token 

478 

479 

480 def __eq__(self, other): 

481 if not isinstance(other, LexerState): 

482 return NotImplemented 

483 

484 return self.text == other.text and self.line_ctr == other.line_ctr and self.last_token == other.last_token 

485 

486 def __copy__(self): 

487 return type(self)(self.text, copy(self.line_ctr), self.last_token) 

488 

489 

490class LexerThread: 

491 """A thread that ties a lexer instance and a lexer state, to be used by the parser 

492 """ 

493 

494 def __init__(self, lexer: 'Lexer', lexer_state: Optional[LexerState]): 

495 self.lexer = lexer 

496 self.state = lexer_state 

497 

498 @classmethod 

499 def from_text(cls, lexer: 'Lexer', text_or_slice: TextOrSlice) -> 'LexerThread': 

500 text = TextSlice.cast_from(text_or_slice) 

501 return cls(lexer, LexerState(text)) 

502 

503 @classmethod 

504 def from_custom_input(cls, lexer: 'Lexer', text: Any) -> 'LexerThread': 

505 return cls(lexer, LexerState(text)) 

506 

507 def lex(self, parser_state): 

508 if self.state is None: 

509 raise TypeError("Cannot lex: No text assigned to lexer state") 

510 return self.lexer.lex(self.state, parser_state) 

511 

512 def __copy__(self): 

513 return type(self)(self.lexer, copy(self.state)) 

514 

515 _Token = Token 

516 

517 

518_Callback = Callable[[Token], Token] 

519 

520class Lexer(ABC): 

521 """Lexer interface 

522 

523 Method Signatures: 

524 lex(self, lexer_state, parser_state) -> Iterator[Token] 

525 """ 

526 @abstractmethod 

527 def lex(self, lexer_state: LexerState, parser_state: Any) -> Iterator[Token]: 

528 return NotImplemented 

529 

530 def search_start(self, text: TextSlice, start_state: Any, pos: int) -> Optional[int]: 

531 raise ConfigurationError("scan() is not supported by %s; use the built-in 'basic' or 'contextual' lexer" 

532 % type(self).__name__) 

533 

534 def make_lexer_state(self, text: str): 

535 "Deprecated" 

536 return LexerState(TextSlice.cast_from(text)) 

537 

538 

539def _check_regex_collisions(terminal_to_regexp: Dict[TerminalDef, str], comparator, strict_mode, max_collisions_to_show=8): 

540 if not comparator: 

541 comparator = interegular.Comparator.from_regexes(terminal_to_regexp) 

542 

543 # When in strict mode, we only ever try to provide one example, so taking 

544 # a long time for that should be fine 

545 max_time = 2 if strict_mode else 0.2 

546 

547 # We don't want to show too many collisions. 

548 if comparator.count_marked_pairs() >= max_collisions_to_show: 

549 return 

550 for group in classify(terminal_to_regexp, lambda t: t.priority).values(): 

551 for a, b in comparator.check(group, skip_marked=True): 

552 assert a.priority == b.priority 

553 # Mark this pair to not repeat warnings when multiple different BasicLexers see the same collision 

554 comparator.mark(a, b) 

555 

556 # Notify the user 

557 message = f"Collision between Terminals {a.name} and {b.name}. " 

558 try: 

559 example = comparator.get_example_overlap(a, b, max_time).format_multiline() 

560 except ValueError: 

561 # Couldn't find an example within max_time steps. 

562 example = "No example could be found fast enough. However, the collision does still exists" 

563 if strict_mode: 

564 raise LexError(f"{message}\n{example}") 

565 logger.warning("%s The lexer will choose between them arbitrarily.\n%s", message, example) 

566 if comparator.count_marked_pairs() >= max_collisions_to_show: 

567 logger.warning("Found 8 regex collisions, will not check for more.") 

568 return 

569 

570 

571class AbstractBasicLexer(Lexer): 

572 terminals_by_name: Dict[str, TerminalDef] 

573 

574 @abstractmethod 

575 def __init__(self, conf: 'LexerConf', comparator=None) -> None: 

576 ... 

577 

578 @abstractmethod 

579 def next_token(self, lex_state: LexerState, parser_state: Any = None) -> Token: 

580 ... 

581 

582 def lex(self, state: LexerState, parser_state: Any) -> Iterator[Token]: 

583 with suppress(EOFError): 

584 while True: 

585 yield self.next_token(state, parser_state) 

586 

587 

588class BasicLexer(AbstractBasicLexer): 

589 terminals: Collection[TerminalDef] 

590 ignore_types: FrozenSet[str] 

591 newline_types: FrozenSet[str] 

592 user_callbacks: Dict[str, _Callback] 

593 callback: Dict[str, _Callback] 

594 re: ModuleType 

595 

596 def __init__(self, conf: 'LexerConf', comparator=None) -> None: 

597 terminals = list(conf.terminals) 

598 assert all(isinstance(t, TerminalDef) for t in terminals), terminals 

599 

600 self.re = conf.re_module 

601 

602 if not conf.skip_validation: 

603 # Sanitization 

604 terminal_to_regexp = {} 

605 for t in terminals: 

606 regexp = t.pattern.to_regexp() 

607 try: 

608 self.re.compile(regexp, conf.g_regex_flags) 

609 except self.re.error: 

610 raise LexError("Cannot compile token %s: %s" % (t.name, t.pattern)) 

611 

612 if t.pattern.min_width == 0: 

613 raise LexError("Lexer does not allow zero-width terminals. (%s: %s)" % (t.name, t.pattern)) 

614 if t.pattern.type == "re": 

615 terminal_to_regexp[t] = regexp 

616 

617 if not (set(conf.ignore) <= {t.name for t in terminals}): 

618 raise LexError("Ignore terminals are not defined: %s" % (set(conf.ignore) - {t.name for t in terminals})) 

619 

620 if has_interegular: 

621 _check_regex_collisions(terminal_to_regexp, comparator, conf.strict) 

622 elif conf.strict: 

623 raise LexError("interegular must be installed for strict mode. Use `pip install 'lark[interegular]'`.") 

624 

625 # Init 

626 self.newline_types = frozenset(t.name for t in terminals if _regexp_has_newline(t.pattern.to_regexp())) 

627 self.ignore_types = frozenset(conf.ignore) 

628 

629 terminals.sort(key=lambda x: (-x.priority, -x.pattern.max_width, -len(x.pattern.value), x.name)) 

630 self.terminals = terminals 

631 self.user_callbacks = conf.callbacks 

632 self.g_regex_flags = conf.g_regex_flags 

633 self.use_bytes = conf.use_bytes 

634 self.terminals_by_name = conf.terminals_by_name 

635 

636 self._scanner: Optional[Scanner] = None 

637 self._search_scanner: Optional[Scanner] = None 

638 

639 def _build_scanner(self) -> Scanner: 

640 terminals, self.callback = _create_unless(self.terminals, self.g_regex_flags, self.re, self.use_bytes) 

641 assert all(self.callback.values()) 

642 

643 for type_, f in self.user_callbacks.items(): 

644 if type_ in self.callback: 

645 # Already a callback there, probably UnlessCallback. 

646 # Bind ``type_`` per iteration; otherwise every CallChain's 

647 # condition closes over the loop variable and checks the last 

648 # terminal's name, silently skipping the other callbacks. 

649 self.callback[type_] = CallChain( 

650 self.callback[type_], f, lambda t, type_=type_: t.type == type_ 

651 ) 

652 else: 

653 self.callback[type_] = f 

654 

655 return Scanner(terminals, self.g_regex_flags, self.re, self.use_bytes) 

656 

657 @property 

658 def scanner(self) -> Scanner: 

659 if self._scanner is None: 

660 self._scanner = self._build_scanner() 

661 return self._scanner 

662 

663 @property 

664 def search_scanner(self) -> Scanner: 

665 # Used by search_start(): a match can only begin with a non-ignored terminal, so we 

666 # search those directly. Searching all terminals and skipping ignores would jump past 

667 # a real start hiding inside an ignore's span (e.g. the "a" in an ignored "xxa"). 

668 if self._search_scanner is None: 

669 terminals = [t for t in self.terminals if t.name not in self.ignore_types] 

670 self._search_scanner = Scanner(terminals, self.g_regex_flags, self.re, self.use_bytes) 

671 return self._search_scanner 

672 

673 def match(self, text, pos): 

674 return self.scanner.match(text, pos) 

675 

676 def next_token(self, lex_state: LexerState, parser_state: Any = None) -> Token: 

677 line_ctr = lex_state.line_ctr 

678 while line_ctr.char_pos < lex_state.text.end: 

679 res = self.match(lex_state.text, line_ctr.char_pos) 

680 if not res: 

681 allowed = self.scanner.allowed_types - self.ignore_types 

682 if not allowed: 

683 allowed = {"<END-OF-FILE>"} 

684 raise UnexpectedCharacters(lex_state.text.text, line_ctr.char_pos, line_ctr.line, line_ctr.column, 

685 allowed=allowed, token_history=lex_state.last_token and [lex_state.last_token], 

686 state=parser_state, terminals_by_name=self.terminals_by_name) 

687 

688 value, type_ = res 

689 

690 ignored = type_ in self.ignore_types 

691 t = None 

692 if not ignored or type_ in self.callback: 

693 t = Token(type_, value, line_ctr.char_pos, line_ctr.line, line_ctr.column) 

694 line_ctr.feed(value, type_ in self.newline_types) 

695 if t is not None: 

696 t.end_line = line_ctr.line 

697 t.end_column = line_ctr.column 

698 t.end_pos = line_ctr.char_pos 

699 if t.type in self.callback: 

700 t = self.callback[t.type](t) 

701 if not ignored: 

702 if not isinstance(t, Token): 

703 raise LexError("Callbacks must return a token (returned %r)" % t) 

704 lex_state.last_token = t 

705 return t 

706 

707 # EOF 

708 raise EOFError(self) 

709 

710 def search_start(self, text: TextSlice, start_state: Any, pos: int) -> Optional[int]: 

711 return self.search_scanner.search(text, pos) 

712 

713 

714class ContextualLexer(Lexer): 

715 lexers: Dict[int, AbstractBasicLexer] 

716 root_lexer: AbstractBasicLexer 

717 

718 BasicLexer: Type[AbstractBasicLexer] = BasicLexer 

719 

720 def __init__(self, conf: 'LexerConf', states: Dict[int, Collection[str]], always_accept: Collection[str]=()) -> None: 

721 terminals = list(conf.terminals) 

722 terminals_by_name = conf.terminals_by_name 

723 

724 trad_conf = copy(conf) 

725 trad_conf.terminals = terminals 

726 

727 if has_interegular and not conf.skip_validation: 

728 comparator = interegular.Comparator.from_regexes({t: t.pattern.to_regexp() for t in terminals}) 

729 else: 

730 comparator = None 

731 lexer_by_tokens: Dict[FrozenSet[str], AbstractBasicLexer] = {} 

732 self.lexers = {} 

733 for state, accepts in states.items(): 

734 key = frozenset(accepts) 

735 try: 

736 lexer = lexer_by_tokens[key] 

737 except KeyError: 

738 accepts = set(accepts) | set(conf.ignore) | set(always_accept)