Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/lark/lark.py: 51%

Shortcuts on this page

r m x   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

330 statements  

1from abc import ABC, abstractmethod 

2import getpass 

3import sys, os, pickle 

4import tempfile 

5import types 

6import re 

7from typing import ( 

8 TypeVar, Type, List, Dict, Iterator, Callable, Union, Optional, Sequence, 

9 Tuple, Iterable, IO, Any, TYPE_CHECKING, Collection 

10) 

11if TYPE_CHECKING: 

12 from .parsers.lalr_interactive_parser import InteractiveParser 

13 from .tree import ParseTree 

14 from .visitors import Transformer 

15 from typing import Literal 

16 from .parser_frontends import ParsingFrontend 

17 

18from .exceptions import ConfigurationError, assert_config, UnexpectedInput 

19from .utils import Serialize, SerializeMemoizer, FS, logger, TextOrSlice 

20from .load_grammar import load_grammar, FromPackageLoader, Grammar, verify_used_files, PackageResource, sha256_digest 

21from .tree import Tree 

22from .common import LexerConf, ParserConf, _ParserArgType, _LexerArgType 

23 

24from .lexer import Lexer, BasicLexer, TerminalDef, LexerThread, Token 

25from .parse_tree_builder import ParseTreeBuilder 

26from .parser_frontends import _validate_frontend_args, _get_lexer_callbacks, _deserialize_parsing_frontend, _construct_parsing_frontend 

27from .grammar import Rule 

28 

29 

30try: 

31 import regex 

32 _has_regex = True 

33except ImportError: 

34 _has_regex = False 

35 

36 

37###{standalone 

38 

39 

40class PostLex(ABC): 

41 @abstractmethod 

42 def process(self, stream: Iterator[Token]) -> Iterator[Token]: 

43 return stream 

44 

45 always_accept: Iterable[str] = () 

46 

47class LarkOptions(Serialize): 

48 """Specifies the options for Lark 

49 

50 """ 

51 

52 start: List[str] 

53 debug: bool 

54 strict: bool 

55 transformer: 'Optional[Transformer]' 

56 propagate_positions: Union[bool, str] 

57 maybe_placeholders: bool 

58 cache: Union[bool, str] 

59 regex: bool 

60 g_regex_flags: int 

61 keep_all_tokens: bool 

62 tree_class: Optional[Callable[[str, List], Any]] 

63 parser: _ParserArgType 

64 lexer: _LexerArgType 

65 ambiguity: 'Literal["auto", "resolve", "explicit", "forest"]' 

66 postlex: Optional[PostLex] 

67 priority: 'Optional[Literal["auto", "normal", "invert"]]' 

68 lexer_callbacks: Dict[str, Callable[[Token], Token]] 

69 use_bytes: bool 

70 ordered_sets: bool 

71 edit_terminals: Optional[Callable[[TerminalDef], TerminalDef]] 

72 import_paths: 'List[Union[str, Callable[[Union[None, str, PackageResource], str], Tuple[str, str]]]]' 

73 source_path: Optional[str] 

74 

75 OPTIONS_DOC = r""" 

76 **=== General Options ===** 

77 

78 start 

79 The start symbol. Either a string, or a list of strings for multiple possible starts (Default: "start") 

80 debug 

81 Display debug information and extra warnings. Use only when debugging (Default: ``False``) 

82 When used with Earley, it generates a forest graph as "sppf.png", if 'dot' is installed. 

83 strict 

84 Throw an exception on any potential ambiguity, including shift/reduce conflicts, and regex collisions. 

85 transformer 

86 Applies the transformer to every parse tree (equivalent to applying it after the parse, but faster) 

87 propagate_positions 

88 Propagates positional attributes into the 'meta' attribute of all tree branches. 

89 Sets attributes: (line, column, end_line, end_column, start_pos, end_pos, 

90 container_line, container_column, container_end_line, container_end_column) 

91 Accepts ``False``, ``True``, or a callable, which will filter which nodes to ignore when propagating. 

92 maybe_placeholders 

93 When ``True``, the ``[]`` operator returns ``None`` when not matched. 

94 When ``False``, ``[]`` behaves like the ``?`` operator, and returns no value at all. 

95 (default= ``True``) 

96 cache 

97 Cache the results of the Lark grammar analysis, for x2 to x3 faster loading. LALR only for now. 

98 

99 - When ``False``, does nothing (default) 

100 - When ``True``, caches to a temporary file in the local directory 

101 - When given a string, caches to the path pointed by the string 

102 regex 

103 When True, uses the ``regex`` module instead of the stdlib ``re``. 

104 g_regex_flags 

105 Flags that are applied to all terminals (both regex and strings) 

106 keep_all_tokens 

107 Prevent the tree builder from automagically removing "punctuation" tokens (Default: ``False``) 

108 tree_class 

109 Lark will produce trees comprised of instances of this class instead of the default ``lark.Tree``. 

110 

111 **=== Algorithm Options ===** 

112 

113 parser 

114 Decides which parser engine to use. Accepts "earley" or "lalr". (Default: "earley"). 

115 (there is also a "cyk" option for legacy) 

116 lexer 

117 Decides whether or not to use a lexer stage 

118 

119 - "auto" (default): Choose for me based on the parser 

120 - "basic": Use a basic lexer 

121 - "contextual": Stronger lexer (only works with parser="lalr") 

122 - "dynamic": Flexible and powerful (only with parser="earley") 

123 - "dynamic_complete": Same as dynamic, but tries *every* variation of tokenizing possible. 

124 ambiguity 

125 Decides how to handle ambiguity in the parse. Only relevant if parser="earley" 

126 

127 - "resolve": The parser will automatically choose the simplest derivation 

128 (it chooses consistently: greedy for tokens, non-greedy for rules) 

129 - "explicit": The parser will return all derivations wrapped in "_ambig" tree nodes (i.e. a forest). 

130 - "forest": The parser will return the root of the shared packed parse forest. 

131 

132 **=== Misc. / Domain Specific Options ===** 

133 

134 postlex 

135 Lexer post-processing (Default: ``None``) Only works with the basic and contextual lexers. 

136 priority 

137 How priorities should be evaluated - "auto", ``None``, "normal", "invert" (Default: "auto") 

138 lexer_callbacks 

139 Dictionary of callbacks for the lexer. May alter tokens during lexing. Use with caution. 

140 use_bytes 

141 Accept an input of type ``bytes`` instead of ``str``. 

142 ordered_sets 

143 Should Earley use ordered-sets to achieve stable output (~10% slower than regular sets. Default: True) 

144 edit_terminals 

145 A callback for editing the terminals before parse. 

146 import_paths 

147 A List of either paths or loader functions to specify from where grammars are imported 

148 source_path 

149 Override the source of from where the grammar was loaded. Useful for relative imports and unconventional grammar loading 

150 **=== End of Options ===** 

151 """ 

152 if __doc__: 

153 __doc__ += OPTIONS_DOC 

154 

155 

156 # Adding a new option needs to be done in multiple places: 

157 # - In the dictionary below. This is the primary truth of which options `Lark.__init__` accepts 

158 # - In the docstring above. It is used both for the docstring of `LarkOptions` and `Lark`, and in readthedocs 

159 # - As an attribute of `LarkOptions` above 

160 # - Potentially in `_LOAD_ALLOWED_OPTIONS` below this class, when the option doesn't change how the grammar is loaded 

161 # - Potentially in `lark.tools.__init__`, if it makes sense, and it can easily be passed as a cmd argument 

162 _defaults: Dict[str, Any] = { 

163 'debug': False, 

164 'strict': False, 

165 'keep_all_tokens': False, 

166 'tree_class': None, 

167 'cache': False, 

168 'postlex': None, 

169 'parser': 'earley', 

170 'lexer': 'auto', 

171 'transformer': None, 

172 'start': 'start', 

173 'priority': 'auto', 

174 'ambiguity': 'auto', 

175 'regex': False, 

176 'propagate_positions': False, 

177 'lexer_callbacks': {}, 

178 'maybe_placeholders': True, 

179 'edit_terminals': None, 

180 'g_regex_flags': 0, 

181 'use_bytes': False, 

182 'ordered_sets': True, 

183 'import_paths': [], 

184 'source_path': None, 

185 '_plugins': {}, 

186 } 

187 

188 def __init__(self, options_dict: Dict[str, Any]) -> None: 

189 o = dict(options_dict) 

190 

191 options = {} 

192 for name, default in self._defaults.items(): 

193 if name in o: 

194 value = o.pop(name) 

195 if isinstance(default, bool) and name not in ('cache', 'use_bytes', 'propagate_positions'): 

196 value = bool(value) 

197 else: 

198 value = default 

199 

200 options[name] = value 

201 

202 if isinstance(options['start'], str): 

203 options['start'] = [options['start']] 

204 

205 self.__dict__['options'] = options 

206 

207 

208 assert_config(self.parser, ('earley', 'lalr', 'cyk', None)) 

209 

210 if self.parser == 'earley' and self.transformer: 

211 raise ConfigurationError('Cannot specify an embedded transformer when using the Earley algorithm. ' 

212 'Please use your transformer on the resulting parse tree, or use a different algorithm (i.e. LALR)') 

213 

214 if o: 

215 raise ConfigurationError("Unknown options: %s" % o.keys()) 

216 

217 def __getattr__(self, name: str) -> Any: 

218 try: 

219 return self.__dict__['options'][name] 

220 except KeyError as e: 

221 raise AttributeError(e) 

222 

223 def __setattr__(self, name: str, value: str) -> None: 

224 assert_config(name, self.options.keys(), "%r isn't a valid option. Expected one of: %s") 

225 self.options[name] = value 

226 

227 def serialize(self, memo = None) -> Dict[str, Any]: 

228 return self.options 

229 

230 @classmethod 

231 def deserialize(cls, data: Dict[str, Any], memo: Dict[int, Union[TerminalDef, Rule]]) -> "LarkOptions": 

232 return cls(data) 

233 

234 

235# Options that can be passed to the Lark parser, even when it was loaded from cache/standalone. 

236# These options are only used outside of `load_grammar`. 

237_LOAD_ALLOWED_OPTIONS = {'postlex', 'transformer', 'lexer_callbacks', 'use_bytes', 'debug', 'g_regex_flags', 'regex', 'propagate_positions', 'tree_class', '_plugins'} 

238 

239_VALID_PRIORITY_OPTIONS = ('auto', 'normal', 'invert', None) 

240_VALID_AMBIGUITY_OPTIONS = ('auto', 'resolve', 'explicit', 'forest') 

241 

242 

243_T = TypeVar('_T', bound="Lark") 

244 

245class Lark(Serialize): 

246 """Main interface for the library. 

247 

248 It's mostly a thin wrapper for the many different parsers, and for the tree constructor. 

249 

250 Parameters: 

251 grammar: a string or file-object containing the grammar spec (using Lark's ebnf syntax) 

252 options: a dictionary controlling various aspects of Lark. 

253 

254 Example: 

255 >>> Lark(r'''start: "foo" ''') 

256 Lark(...) 

257 """ 

258 

259 source_path: str 

260 source_grammar: str 

261 grammar: 'Grammar' 

262 options: LarkOptions 

263 lexer: Lexer 

264 parser: 'ParsingFrontend' 

265 terminals: Collection[TerminalDef] 

266 

267 def __init__(self, grammar: 'Union[Grammar, str, IO[str]]', **options) -> None: 

268 self.options = LarkOptions(options) 

269 re_module: types.ModuleType 

270 

271 # Set regex or re module 

272 use_regex = self.options.regex 

273 if use_regex: 

274 if _has_regex: 

275 re_module = regex 

276 else: 

277 raise ImportError('`regex` module must be installed if calling `Lark(regex=True)`.') 

278 else: 

279 re_module = re 

280 

281 # Some, but not all file-like objects have a 'name' attribute 

282 if self.options.source_path is None: 

283 try: 

284 self.source_path = grammar.name # type: ignore[union-attr] 

285 except AttributeError: 

286 self.source_path = '<string>' 

287 else: 

288 self.source_path = self.options.source_path 

289 

290 # Drain file-like objects to get their contents 

291 try: 

292 read = grammar.read # type: ignore[union-attr] 

293 except AttributeError: 

294 pass 

295 else: 

296 grammar = read() 

297 

298 cache_fn = None 

299 cache_sha256 = None 

300 if isinstance(grammar, str): 

301 self.source_grammar = grammar 

302 if self.options.use_bytes: 

303 if not grammar.isascii(): 

304 raise ConfigurationError("Grammar must be ascii only, when use_bytes=True") 

305 

306 if self.options.cache: 

307 if self.options.parser != 'lalr': 

308 raise ConfigurationError("cache only works with parser='lalr' for now") 

309 

310 unhashable = ('transformer', 'postlex', 'lexer_callbacks', 'edit_terminals', '_plugins') 

311 options_str = ''.join(k+str(v) for k, v in options.items() if k not in unhashable) 

312 from . import __version__ 

313 s = grammar + options_str + __version__ + str(sys.version_info[:2]) 

314 cache_sha256 = sha256_digest(s) 

315 

316 if isinstance(self.options.cache, str): 

317 cache_fn = self.options.cache 

318 else: 

319 if self.options.cache is not True: 

320 raise ConfigurationError("cache argument must be bool or str") 

321 

322 try: 

323 username = getpass.getuser() 

324 except Exception: 

325 # The exception raised may be ImportError or OSError in 

326 # the future. For the cache, we don't care about the 

327 # specific reason - we just want a username. 

328 username = "unknown" 

329 

330 cache_fn = tempfile.gettempdir() + "/.lark_cache_%s_%s_%s_%s.tmp" % (username, cache_sha256, *sys.version_info[:2]) 

331 

332 old_options = self.options 

333 try: 

334 with FS.open(cache_fn, 'rb') as f: 

335 logger.debug('Loading grammar from cache: %s', cache_fn) 

336 # Remove options that aren't relevant for loading from cache 

337 for name in (set(options) - _LOAD_ALLOWED_OPTIONS): 

338 del options[name] 

339 file_sha256 = f.readline().rstrip(b'\n') 

340 cached_used_files = pickle.load(f) 

341 if file_sha256 == cache_sha256.encode('utf8') and verify_used_files(cached_used_files): 

342 cached_parser_data = pickle.load(f) 

343 self._load(cached_parser_data, **options) 

344 return 

345 except FileNotFoundError: 

346 # The cache file doesn't exist; parse and compose the grammar as normal 

347 pass 

348 except Exception: # We should probably narrow done which errors we catch here. 

349 logger.exception("Failed to load Lark from cache: %r. We will try to carry on.", cache_fn) 

350 

351 # In theory, the Lark instance might have been messed up by the call to `_load`. 

352 # In practice the only relevant thing that might have been overwritten should be `options` 

353 self.options = old_options 

354 

355 

356 # Parse the grammar file and compose the grammars 

357 self.grammar, used_files = load_grammar(grammar, self.source_path, self.options.import_paths, self.options.keep_all_tokens) 

358 else: 

359 assert isinstance(grammar, Grammar) 

360 self.grammar = grammar 

361 

362 

363 if self.options.lexer == 'auto': 

364 if self.options.parser == 'lalr': 

365 self.options.lexer = 'contextual' 

366 elif self.options.parser == 'earley': 

367 if self.options.postlex is not None: 

368 logger.info("postlex can't be used with the dynamic lexer, so we use 'basic' instead. " 

369 "Consider using lalr with contextual instead of earley") 

370 self.options.lexer = 'basic' 

371 else: 

372 self.options.lexer = 'dynamic' 

373 elif self.options.parser == 'cyk': 

374 self.options.lexer = 'basic' 

375 else: 

376 assert False, self.options.parser 

377 lexer = self.options.lexer 

378 if isinstance(lexer, type): 

379 assert issubclass(lexer, Lexer) # XXX Is this really important? Maybe just ensure interface compliance 

380 else: 

381 assert_config(lexer, ('basic', 'contextual', 'dynamic', 'dynamic_complete')) 

382 if self.options.postlex is not None and 'dynamic' in lexer: 

383 raise ConfigurationError("Can't use postlex with a dynamic lexer. Use basic or contextual instead") 

384 

385 if self.options.ambiguity == 'auto': 

386 if self.options.parser == 'earley': 

387 self.options.ambiguity = 'resolve' 

388 else: 

389 assert_config(self.options.parser, ('earley', 'cyk'), "%r doesn't support disambiguation. Use one of these parsers instead: %s") 

390 

391 if self.options.priority == 'auto': 

392 self.options.priority = 'normal' 

393 

394 if self.options.priority not in _VALID_PRIORITY_OPTIONS: 

395 raise ConfigurationError("invalid priority option: %r. Must be one of %r" % (self.options.priority, _VALID_PRIORITY_OPTIONS)) 

396 if self.options.ambiguity not in _VALID_AMBIGUITY_OPTIONS: 

397 raise ConfigurationError("invalid ambiguity option: %r. Must be one of %r" % (self.options.ambiguity, _VALID_AMBIGUITY_OPTIONS)) 

398 

399 if self.options.parser is None: 

400 terminals_to_keep = '*' 

401 elif self.options.postlex is not None: 

402 terminals_to_keep = set(self.options.postlex.always_accept) 

403 else: 

404 terminals_to_keep = set() 

405 

406 # Compile the EBNF grammar into BNF 

407 self.terminals, self.rules, self.ignore_tokens = self.grammar.compile(self.options.start, terminals_to_keep) 

408 

409 if self.options.edit_terminals: 

410 for t in self.terminals: 

411 self.options.edit_terminals(t) 

412 

413 self._terminals_dict = {t.name: t for t in self.terminals} 

414 

415 # If the user asked to invert the priorities, negate them all here. 

416 if self.options.priority == 'invert': 

417 for rule in self.rules: 

418 if rule.options.priority is not None: 

419 rule.options.priority = -rule.options.priority 

420 for term in self.terminals: 

421 term.priority = -term.priority 

422 # Else, if the user asked to disable priorities, strip them from the 

423 # rules and terminals. This allows the Earley parsers to skip an extra forest walk 

424 # for improved performance, if you don't need them (or didn't specify any). 

425 elif self.options.priority is None: 

426 for rule in self.rules: 

427 if rule.options.priority is not None: 

428 rule.options.priority = None 

429 for term in self.terminals: 

430 term.priority = 0 

431 

432 # TODO Deprecate lexer_callbacks? 

433 self.lexer_conf = LexerConf( 

434 self.terminals, re_module, self.ignore_tokens, self.options.postlex, 

435 self.options.lexer_callbacks, self.options.g_regex_flags, use_bytes=self.options.use_bytes, strict=self.options.strict 

436 ) 

437 

438 if self.options.parser: 

439 self.parser = self._build_parser() 

440 elif lexer: 

441 self.lexer = self._build_lexer() 

442 

443 if cache_fn: 

444 logger.debug('Saving grammar to cache: %s', cache_fn) 

445 try: 

446 with FS.open(cache_fn, 'wb') as f: 

447 assert cache_sha256 is not None 

448 f.write(cache_sha256.encode('utf8') + b'\n') 

449 pickle.dump(used_files, f) 

450 self.save(f, _LOAD_ALLOWED_OPTIONS) 

451 except IOError as e: 

452 logger.exception("Failed to save Lark to cache: %r.", cache_fn, e) 

453 

454 if __doc__: 

455 __doc__ += "\n\n" + LarkOptions.OPTIONS_DOC 

456 

457 __serialize_fields__ = 'parser', 'rules', 'options' 

458 

459 def _build_lexer(self, dont_ignore: bool=False) -> BasicLexer: 

460 lexer_conf = self.lexer_conf 

461 if dont_ignore: 

462 from copy import copy 

463 lexer_conf = copy(lexer_conf) 

464 lexer_conf.ignore = () 

465 return BasicLexer(lexer_conf) 

466 

467 def _prepare_callbacks(self) -> None: 

468 self._callbacks = {} 

469 # we don't need these callbacks if we aren't building a tree 

470 if self.options.ambiguity != 'forest': 

471 self._parse_tree_builder = ParseTreeBuilder( 

472 self.rules, 

473 self.options.tree_class or Tree, 

474 self.options.propagate_positions, 

475 self.options.parser != 'lalr' and self.options.ambiguity == 'explicit', 

476 self.options.maybe_placeholders 

477 ) 

478 self._callbacks = self._parse_tree_builder.create_callback(self.options.transformer) 

479 self._callbacks.update(_get_lexer_callbacks(self.options.transformer, self.terminals)) 

480 

481 def _build_parser(self) -> "ParsingFrontend": 

482 self._prepare_callbacks() 

483 _validate_frontend_args(self.options.parser, self.options.lexer) 

484 parser_conf = ParserConf(self.rules, self._callbacks, self.options.start) 

485 return _construct_parsing_frontend( 

486 self.options.parser, 

487 self.options.lexer, 

488 self.lexer_conf, 

489 parser_conf, 

490 options=self.options 

491 ) 

492 

493 def save(self, f, exclude_options: Collection[str] = ()) -> None: 

494 """Saves the instance into the given file object 

495 

496 Useful for caching and multiprocessing. 

497 """ 

498 if self.options.parser != 'lalr': 

499 raise NotImplementedError("Lark.save() is only implemented for the LALR(1) parser.") 

500 data, m = self.memo_serialize([TerminalDef, Rule]) 

501 if exclude_options: 

502 data["options"] = {n: v for n, v in data["options"].items() if n not in exclude_options} 

503 pickle.dump({'data': data, 'memo': m}, f, protocol=pickle.HIGHEST_PROTOCOL) 

504 

505 @classmethod 

506 def load(cls: Type[_T], f) -> _T: 

507 """Loads an instance from the given file object 

508 

509 Useful for caching and multiprocessing. 

510 """ 

511 inst = cls.__new__(cls) 

512 return inst._load(f) 

513 

514 def _deserialize_lexer_conf(self, data: Dict[str, Any], memo: Dict[int, Union[TerminalDef, Rule]], options: LarkOptions) -> LexerConf: 

515 lexer_conf = LexerConf.deserialize(data['lexer_conf'], memo) 

516 lexer_conf.callbacks = options.lexer_callbacks or {} 

517 lexer_conf.re_module = regex if options.regex else re 

518 lexer_conf.use_bytes = options.use_bytes 

519 lexer_conf.g_regex_flags = options.g_regex_flags 

520 lexer_conf.skip_validation = True 

521 lexer_conf.postlex = options.postlex 

522 return lexer_conf 

523 

524 def _load(self: _T, f: Any, **kwargs) -> _T: 

525 if isinstance(f, dict): 

526 d = f 

527 else: 

528 d = pickle.load(f) 

529 memo_json = d['memo'] 

530 data = d['data'] 

531 

532 assert memo_json 

533 memo = SerializeMemoizer.deserialize(memo_json, {'Rule': Rule, 'TerminalDef': TerminalDef}, {}) 

534 options = dict(data['options']) 

535 if (set(kwargs) - _LOAD_ALLOWED_OPTIONS) & set(LarkOptions._defaults): 

536 raise ConfigurationError("Some options are not allowed when loading a Parser: {}" 

537 .format(set(kwargs) - _LOAD_ALLOWED_OPTIONS)) 

538 options.update(kwargs) 

539 self.options = LarkOptions.deserialize(options, memo) 

540 self.rules = [Rule.deserialize(r, memo) for r in data['rules']] 

541 self.source_path = '<deserialized>' 

542 _validate_frontend_args(self.options.parser, self.options.lexer) 

543 self.lexer_conf = self._deserialize_lexer_conf(data['parser'], memo, self.options) 

544 self.terminals = self.lexer_conf.terminals 

545 self._prepare_callbacks() 

546 self._terminals_dict = {t.name: t for t in self.terminals} 

547 self.parser = _deserialize_parsing_frontend( 

548 data['parser'], 

549 memo, 

550 self.lexer_conf, 

551 self._callbacks, 

552 self.options, # Not all, but multiple attributes are used 

553 ) 

554 return self 

555 

556 @classmethod 

557 def _load_from_dict(cls, data, memo, **kwargs): 

558 inst = cls.__new__(cls) 

559 return inst._load({'data': data, 'memo': memo}, **kwargs) 

560 

561 @classmethod 

562 def open(cls: Type[_T], grammar_filename: str, rel_to: Optional[str]=None, **options) -> _T: 

563 """Create an instance of Lark with the grammar given by its filename 

564 

565 If ``rel_to`` is provided, the function will find the grammar filename in relation to it. 

566 

567 Example: 

568 

569 >>> Lark.open("grammar_file.lark", rel_to=__file__, parser="lalr") 

570 Lark(...) 

571 

572 """ 

573 if rel_to: 

574 basepath = os.path.dirname(rel_to) 

575 grammar_filename = os.path.join(basepath, grammar_filename) 

576 with open(grammar_filename, encoding='utf8') as f: 

577 return cls(f, **options) 

578 

579 @classmethod 

580 def open_from_package(cls: Type[_T], package: str, grammar_path: str, search_paths: 'Sequence[str]'=[""], **options) -> _T: 

581 """Create an instance of Lark with the grammar loaded from within the package `package`. 

582 This allows grammar loading from zipapps. 

583 

584 Imports in the grammar will use the `package` and `search_paths` provided, through `FromPackageLoader` 

585 

586 Example: 

587 

588 Lark.open_from_package(__name__, "example.lark", ("grammars",), parser=...) 

589 """ 

590 package_loader = FromPackageLoader(package, search_paths) 

591 full_path, text = package_loader(None, grammar_path) 

592 options.setdefault('source_path', full_path) 

593 options.setdefault('import_paths', []) 

594 options['import_paths'].append(package_loader) 

595 return cls(text, **options) 

596 

597 def __repr__(self): 

598 return 'Lark(open(%r), parser=%r, lexer=%r, ...)' % (self.source_path, self.options.parser, self.options.lexer) 

599 

600 

601 def lex(self, text: TextOrSlice, dont_ignore: bool=False) -> Iterator[Token]: 

602 """Only lex (and postlex) the text, without parsing it. Only relevant when lexer='basic' 

603 

604 When dont_ignore=True, the lexer will return all tokens, even those marked for %ignore. 

605 

606 :raises UnexpectedCharacters: In case the lexer cannot find a suitable match. 

607 """ 

608 lexer: Lexer 

609 if not hasattr(self, 'lexer') or dont_ignore: 

610 lexer = self._build_lexer(dont_ignore) 

611 else: 

612 lexer = self.lexer 

613 lexer_thread = LexerThread.from_text(lexer, text) 

614 stream = lexer_thread.lex(None) 

615 if self.options.postlex: 

616 return self.options.postlex.process(stream) 

617 return stream 

618 

619 def get_terminal(self, name: str) -> TerminalDef: 

620 """Get information about a terminal""" 

621 return self._terminals_dict[name] 

622 

623 def parse_interactive(self, text: Optional[TextOrSlice]=None, start: Optional[str]=None) -> 'InteractiveParser': 

624 """Start an interactive parsing session. Only works when parser='lalr'. 

625 

626 Parameters: 

627 text (TextOrSlice, optional): Text to be parsed. Required for ``resume_parse()``. 

628 start (str, optional): Start symbol 

629 

630 Returns: 

631 A new InteractiveParser instance. 

632 

633 See Also: ``Lark.parse()`` 

634 """ 

635 return self.parser.parse_interactive(text, start=start) 

636 

637 def parse(self, text: TextOrSlice, start: Optional[str]=None, on_error: 'Optional[Callable[[UnexpectedInput], bool]]'=None) -> 'ParseTree': 

638 """Parse the given text, according to the options provided. 

639 

640 Parameters: 

641 text (TextOrSlice): Text to be parsed, as `str` or `bytes`. 

642 TextSlice may also be used, but only when lexer='basic' or 'contextual'. 

643 start (str, optional): Required if Lark was given multiple possible start symbols (using the start option). 

644 on_error (function, optional): if provided, will be called on UnexpectedInput error, 

645 with the exception as its argument. Return true to resume parsing, or false to raise the exception. 

646 LALR only. See examples/advanced/error_handling.py for an example of how to use on_error. 

647 

648 Returns: 

649 If a transformer is supplied to ``__init__``, returns whatever is the 

650 result of the transformation. Otherwise, returns a Tree instance. 

651 

652 :raises UnexpectedInput: On a parse error, one of these sub-exceptions will rise: 

653 ``UnexpectedCharacters``, ``UnexpectedToken``, or ``UnexpectedEOF``. 

654 For convenience, these sub-exceptions also inherit from ``ParserError`` and ``LexerError``. 

655 

656 """ 

657 if on_error is not None and self.options.parser != 'lalr': 

658 raise NotImplementedError("The on_error option is only implemented for the LALR(1) parser.") 

659 return self.parser.parse(text, start=start, on_error=on_error) 

660 

661 

662###}