Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/lark/lark.py: 53%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1from abc import ABC, abstractmethod
2import getpass
3import sys, os, pickle
4import tempfile
5import types
6import re
7from typing import (
8 TypeVar, Type, List, Dict, Iterator, Callable, Union, Optional, Sequence,
9 Tuple, Iterable, IO, Any, TYPE_CHECKING, Collection, Generic, overload,
10)
11if TYPE_CHECKING:
12 from .parsers.lalr_interactive_parser import InteractiveParser
13 from .tree import ParseTree
14 from .visitors import Transformer
15 from typing import Literal
16 from .parser_frontends import ParsingFrontend, ScanMatch
18from .exceptions import ConfigurationError, assert_config, UnexpectedInput
19from .utils import Serialize, SerializeMemoizer, FS, logger, TextOrSlice, LarkInput
20from .load_grammar import load_grammar, FromPackageLoader, Grammar, verify_used_files, PackageResource, sha256_digest
22from .tree import Tree
23from .common import LexerConf, ParserConf, _ParserArgType, _LexerArgType
25from .lexer import Lexer, BasicLexer, TerminalDef, LexerThread, Token
26from .visitors import _Return_T
27from .parse_tree_builder import ParseTreeBuilder
28from .parser_frontends import _validate_frontend_args, _get_lexer_callbacks, _deserialize_parsing_frontend, _construct_parsing_frontend
29from .grammar import Rule
32try:
33 import regex
34 _has_regex = True
35except ImportError:
36 _has_regex = False
39###{standalone
42class PostLex(ABC):
43 @abstractmethod
44 def process(self, stream: Iterator[Token]) -> Iterator[Token]:
45 return stream
47 always_accept: Iterable[str] = ()
49class LarkOptions(Serialize):
50 """Specifies the options for Lark
52 """
54 start: List[str]
55 debug: bool
56 strict: bool
57 transformer: 'Optional[Transformer]'
58 propagate_positions: Union[bool, str]
59 maybe_placeholders: bool
60 cache: Union[bool, str]
61 cache_grammar: bool
62 regex: bool
63 g_regex_flags: int
64 keep_all_tokens: bool
65 tree_class: Optional[Callable[[str, List], Any]]
66 parser: _ParserArgType
67 lexer: _LexerArgType
68 ambiguity: 'Literal["auto", "resolve", "explicit", "forest"]'
69 postlex: Optional[PostLex]
70 priority: 'Optional[Literal["auto", "normal", "invert"]]'
71 lexer_callbacks: Dict[str, Callable[[Token], Token]]
72 use_bytes: bool
73 ordered_sets: bool
74 edit_terminals: Optional[Callable[[TerminalDef], TerminalDef]]
75 import_paths: 'List[Union[str, Callable[[Union[None, str, PackageResource], str], Tuple[str, str]]]]'
76 source_path: Optional[str]
78 OPTIONS_DOC = r"""
79 **=== General Options ===**
81 start
82 The start symbol. Either a string, or a list of strings for multiple possible starts (Default: "start")
83 debug
84 Display debug information and extra warnings. Use only when debugging (Default: ``False``)
85 When used with Earley, it generates a forest graph as "sppf.png", if 'dot' is installed.
86 strict
87 Throw an exception on any potential ambiguity, including shift/reduce conflicts, and regex collisions.
88 transformer
89 Applies the transformer to every parse tree (equivalent to applying it after the parse, but faster)
90 propagate_positions
91 Propagates positional attributes into the 'meta' attribute of all tree branches.
92 Sets attributes: line, column, end_line, end_column, start_pos, end_pos,
93 container_line, container_column, container_end_line, container_end_column.
94 Accepts ``False``, ``True``, or a callable, which will filter which nodes to ignore when propagating.
95 maybe_placeholders
96 When ``True``, the ``[]`` operator returns ``None`` when not matched.
97 When ``False``, ``[]`` behaves like the ``?`` operator, and returns no value at all.
98 (default= ``True``)
99 cache
100 Cache the results of the Lark grammar analysis, for x2 to x3 faster loading. LALR only for now.
102 - When ``False``, does nothing (default)
103 - When ``True``, caches to a temporary file in the local directory
104 - When given a string, caches to the path pointed by the string
105 cache_grammar
106 For use with ``cache`` option. When ``True``, the unanalyzed grammar is also included in the cache.
107 Useful for classes that require the ``Lark.grammar`` to be present (e.g. Reconstructor).
108 (default= ``False``)
109 regex
110 When True, uses the ``regex`` module instead of the stdlib ``re``.
111 g_regex_flags
112 Flags that are applied to all terminals (both regex and strings)
113 keep_all_tokens
114 Prevent the tree builder from automagically removing "punctuation" tokens (Default: ``False``)
115 tree_class
116 Lark will produce trees comprised of instances of this class instead of the default ``lark.Tree``.
118 **=== Algorithm Options ===**
120 parser
121 Decides which parser engine to use. Accepts "earley" or "lalr". (Default: "earley").
122 (there is also a "cyk" option for legacy)
123 lexer
124 Decides whether or not to use a lexer stage
126 - "auto" (default): Choose for me based on the parser
127 - "basic": Use a basic lexer
128 - "contextual": Stronger lexer (only works with parser="lalr")
129 - "dynamic": Flexible and powerful (only with parser="earley")
130 - "dynamic_complete": Same as dynamic, but tries *every* variation of tokenizing possible.
131 ambiguity
132 Decides how to handle ambiguity in the parse. Only relevant if parser="earley"
134 - "resolve": The parser will automatically choose the simplest derivation
135 (it chooses consistently: greedy for tokens, non-greedy for rules)
136 - "explicit": The parser will return all derivations wrapped in "_ambig" tree nodes (i.e. a forest).
137 - "forest": The parser will return the root of the shared packed parse forest.
139 **=== Misc. / Domain Specific Options ===**
141 postlex
142 Lexer post-processing (Default: ``None``) Only works with the basic and contextual lexers.
143 priority
144 How priorities should be evaluated - "auto", ``None``, "normal", "invert" (Default: "auto")
145 lexer_callbacks
146 Dictionary of callbacks for the lexer. May alter tokens during lexing. Use with caution.
147 use_bytes
148 Accept an input of type ``bytes`` instead of ``str``.
149 ordered_sets
150 Should Earley use ordered-sets to achieve stable output (~10% slower than regular sets. Default: True)
151 edit_terminals
152 A callback for editing the terminals before parse.
153 import_paths
154 A List of either paths or loader functions to specify from where grammars are imported
155 source_path
156 Override the source of from where the grammar was loaded. Useful for relative imports and unconventional grammar loading
158 **=== End of Options ===**
159 """
160 if __doc__:
161 __doc__ += OPTIONS_DOC
164 # Adding a new option needs to be done in multiple places:
165 # - In the dictionary below. This is the primary truth of which options `Lark.__init__` accepts
166 # - In the docstring above. It is used both for the docstring of `LarkOptions` and `Lark`, and in readthedocs
167 # - As an attribute of `LarkOptions` above
168 # - Potentially in `_LOAD_ALLOWED_OPTIONS` below this class, when the option doesn't change how the grammar is loaded
169 # - Potentially in `lark.tools.__init__`, if it makes sense, and it can easily be passed as a cmd argument
170 _defaults: Dict[str, Any] = {
171 'debug': False,
172 'strict': False,
173 'keep_all_tokens': False,
174 'tree_class': None,
175 'cache': False,
176 'cache_grammar': False,
177 'postlex': None,
178 'parser': 'earley',
179 'lexer': 'auto',
180 'transformer': None,
181 'start': 'start',
182 'priority': 'auto',
183 'ambiguity': 'auto',
184 'regex': False,
185 'propagate_positions': False,
186 'lexer_callbacks': {},
187 'maybe_placeholders': True,
188 'edit_terminals': None,
189 'g_regex_flags': 0,
190 'use_bytes': False,
191 'ordered_sets': True,
192 'import_paths': [],
193 'source_path': None,
194 '_plugins': {},
195 }
197 def __init__(self, options_dict: Dict[str, Any]) -> None:
198 o = dict(options_dict)
200 options = {}
201 for name, default in self._defaults.items():
202 if name in o:
203 value = o.pop(name)
204 if isinstance(default, bool) and name not in ('cache', 'use_bytes', 'propagate_positions'):
205 value = bool(value)
206 else:
207 value = default
209 options[name] = value
211 if isinstance(options['start'], str):
212 options['start'] = [options['start']]
214 self.__dict__['options'] = options
217 assert_config(self.parser, ('earley', 'lalr', 'cyk', None))
219 if self.parser == 'earley' and self.transformer:
220 raise ConfigurationError('Cannot specify an embedded transformer when using the Earley algorithm. '
221 'Please use your transformer on the resulting parse tree, or use a different algorithm (i.e. LALR)')
223 if self.cache_grammar and not self.cache:
224 raise ConfigurationError('cache_grammar cannot be set when cache is disabled')
226 if o:
227 raise ConfigurationError("Unknown options: %s" % o.keys())
229 def __getattr__(self, name: str) -> Any:
230 try:
231 return self.__dict__['options'][name]
232 except KeyError as e:
233 raise AttributeError(e)
235 def __setattr__(self, name: str, value: str) -> None:
236 assert_config(name, self.options.keys(), "%r isn't a valid option. Expected one of: %s")
237 self.options[name] = value
239 def serialize(self, memo = None) -> Dict[str, Any]:
240 return self.options
242 @classmethod
243 def deserialize(cls, data: Dict[str, Any], memo: Dict[int, Union[TerminalDef, Rule]]) -> "LarkOptions":
244 return cls(data)
247# Options that can be passed to the Lark parser, even when it was loaded from cache/standalone.
248# These options are only used outside of `load_grammar`.
249_LOAD_ALLOWED_OPTIONS = {'postlex', 'transformer', 'lexer_callbacks', 'use_bytes', 'debug', 'g_regex_flags', 'regex', 'propagate_positions', 'tree_class', '_plugins'}
251_VALID_PRIORITY_OPTIONS = ('auto', 'normal', 'invert', None)
252_VALID_AMBIGUITY_OPTIONS = ('auto', 'resolve', 'explicit', 'forest')
255_T = TypeVar('_T', bound="Lark")
256_InitReturn_T = TypeVar('_InitReturn_T') # __init__ self annotations must use a new type-var
258class Lark(Serialize, Generic[_Return_T]):
259 """Main interface for the library.
261 It's mostly a thin wrapper for the many different parsers, and for the tree constructor.
263 Parameters:
264 grammar: a string or file-object containing the grammar spec (using Lark's ebnf syntax)
265 options: a dictionary controlling various aspects of Lark.
267 Example:
268 >>> Lark(r'''start: "foo" ''')
269 Lark(...)
270 """
272 source_path: str
273 source_grammar: str
274 grammar: 'Grammar'
275 options: LarkOptions
276 lexer: Lexer
277 parser: 'ParsingFrontend'
278 terminals: Collection[TerminalDef]
280 __serialize_fields__ = ['parser', 'rules', 'options']
282 @overload
283 def __init__(
284 self: 'Lark[_InitReturn_T]',
285 grammar: 'Union[Grammar, str, IO[str]]',
286 *,
287 transformer: 'Transformer[Token, _InitReturn_T]',
288 **options: Any,
289 ) -> None: ...
291 @overload
292 def __init__(
293 self: 'Lark[ParseTree]',
294 grammar: 'Union[Grammar, str, IO[str]]',
295 **options: Any,
296 ) -> None: ...
298 def __init__(self, grammar: 'Union[Grammar, str, IO[str]]', **options) -> None:
299 self.options = LarkOptions(options)
300 re_module: types.ModuleType
302 # Update which fields are serialized
303 if self.options.cache_grammar:
304 self.__serialize_fields__ = self.__serialize_fields__ + ['grammar']
306 # Set regex or re module
307 use_regex = self.options.regex
308 if use_regex:
309 if _has_regex:
310 re_module = regex
311 else:
312 raise ImportError('`regex` module must be installed if calling `Lark(regex=True)`.')
313 else:
314 re_module = re
316 # Some, but not all file-like objects have a 'name' attribute
317 if self.options.source_path is None:
318 try:
319 self.source_path = grammar.name # type: ignore[union-attr]
320 except AttributeError:
321 self.source_path = '<string>'
322 else:
323 self.source_path = self.options.source_path
325 # Drain file-like objects to get their contents
326 try:
327 read = grammar.read # type: ignore[union-attr]
328 except AttributeError:
329 pass
330 else:
331 grammar = read()
333 cache_fn = None
334 cache_sha256 = None
335 if isinstance(grammar, str):
336 self.source_grammar = grammar
337 if self.options.use_bytes:
338 if not grammar.isascii():
339 raise ConfigurationError("Grammar must be ascii only, when use_bytes=True")
341 if self.options.cache:
342 if self.options.parser != 'lalr':
343 raise ConfigurationError("cache only works with parser='lalr' for now")
345 unhashable = ('transformer', 'postlex', 'lexer_callbacks', 'edit_terminals', '_plugins')
346 options_str = ''.join(k+str(v) for k, v in options.items() if k not in unhashable)
347 from . import __version__
348 s = grammar + options_str + __version__ + str(sys.version_info[:2])
349 cache_sha256 = sha256_digest(s)
351 if isinstance(self.options.cache, str):
352 cache_fn = self.options.cache
353 else:
354 if self.options.cache is not True:
355 raise ConfigurationError("cache argument must be bool or str")
357 try:
358 username = getpass.getuser()
359 except Exception:
360 # The exception raised may be ImportError or OSError in
361 # the future. For the cache, we don't care about the
362 # specific reason - we just want a username.
363 username = "unknown"
366 cache_fn = tempfile.gettempdir() + "/.lark_%s_%s_%s_%s_%s.tmp" % (
367 "cache_grammar" if self.options.cache_grammar else "cache", username, cache_sha256, *sys.version_info[:2])
369 old_options = self.options
370 try:
371 with FS.open(cache_fn, 'rb') as f:
372 logger.debug('Loading grammar from cache: %s', cache_fn)
373 # Remove options that aren't relevant for loading from cache
374 for name in (set(options) - _LOAD_ALLOWED_OPTIONS):
375 del options[name]
376 file_sha256 = f.readline().rstrip(b'\n')
377 cached_used_files = pickle.load(f)
378 if file_sha256 == cache_sha256.encode('utf8') and verify_used_files(cached_used_files):
379 cached_parser_data = pickle.load(f)
380 self._load(cached_parser_data, **options)
381 return
382 except FileNotFoundError:
383 # The cache file doesn't exist; parse and compose the grammar as normal
384 pass
385 except Exception: # We should probably narrow done which errors we catch here.
386 logger.exception("Failed to load Lark from cache: %r. We will try to carry on.", cache_fn)
388 # In theory, the Lark instance might have been messed up by the call to `_load`.
389 # In practice the only relevant thing that might have been overwritten should be `options`
390 self.options = old_options
393 # Parse the grammar file and compose the grammars
394 self.grammar, used_files = load_grammar(grammar, self.source_path, self.options.import_paths, self.options.keep_all_tokens)
395 else:
396 assert isinstance(grammar, Grammar)
397 self.grammar = grammar
400 if self.options.lexer == 'auto':
401 if self.options.parser == 'lalr':
402 self.options.lexer = 'contextual'
403 elif self.options.parser == 'earley':
404 if self.options.postlex is not None:
405 logger.info("postlex can't be used with the dynamic lexer, so we use 'basic' instead. "
406 "Consider using lalr with contextual instead of earley")
407 self.options.lexer = 'basic'
408 else:
409 self.options.lexer = 'dynamic'
410 elif self.options.parser == 'cyk':
411 self.options.lexer = 'basic'
412 else:
413 assert False, self.options.parser
414 lexer = self.options.lexer
415 if isinstance(lexer, type):
416 assert issubclass(lexer, Lexer) # XXX Is this really important? Maybe just ensure interface compliance
417 else:
418 assert_config(lexer, ('basic', 'contextual', 'dynamic', 'dynamic_complete'))
419 if self.options.postlex is not None and 'dynamic' in lexer:
420 raise ConfigurationError("Can't use postlex with a dynamic lexer. Use basic or contextual instead")
422 if self.options.ambiguity == 'auto':
423 if self.options.parser == 'earley':
424 self.options.ambiguity = 'resolve'
425 else:
426 assert_config(self.options.parser, ('earley', 'cyk'), "%r doesn't support disambiguation. Use one of these parsers instead: %s")
428 if self.options.priority == 'auto':
429 self.options.priority = 'normal'
431 if self.options.priority not in _VALID_PRIORITY_OPTIONS:
432 raise ConfigurationError("invalid priority option: %r. Must be one of %r" % (self.options.priority, _VALID_PRIORITY_OPTIONS))
433 if self.options.ambiguity not in _VALID_AMBIGUITY_OPTIONS:
434 raise ConfigurationError("invalid ambiguity option: %r. Must be one of %r" % (self.options.ambiguity, _VALID_AMBIGUITY_OPTIONS))
436 if self.options.parser is None:
437 terminals_to_keep = '*' # For lexer-only mode, keep all terminals
438 elif self.options.postlex is not None:
439 terminals_to_keep = set(self.options.postlex.always_accept)
440 else:
441 terminals_to_keep = set()
443 # Compile the EBNF grammar into BNF
444 self.terminals, self.rules, self.ignore_tokens = self.grammar.compile(self.options.start, terminals_to_keep)
446 if self.options.edit_terminals:
447 for t in self.terminals:
448 self.options.edit_terminals(t)
450 self._terminals_dict = {t.name: t for t in self.terminals}
452 # If the user asked to invert the priorities, negate them all here.
453 if self.options.priority == 'invert':
454 for rule in self.rules:
455 if rule.options.priority is not None:
456 rule.options.priority = -rule.options.priority
457 for term in self.terminals:
458 term.priority = -term.priority
459 # Else, if the user asked to disable priorities, strip them from the
460 # rules and terminals. This allows the Earley parsers to skip an extra forest walk
461 # for improved performance, if you don't need them (or didn't specify any).
462 elif self.options.priority is None:
463 for rule in self.rules:
464 if rule.options.priority is not None:
465 rule.options.priority = None
466 for term in self.terminals:
467 term.priority = 0
469 # TODO Deprecate lexer_callbacks?
470 self.lexer_conf = LexerConf(
471 self.terminals, re_module, self.ignore_tokens, self.options.postlex,
472 self.options.lexer_callbacks, self.options.g_regex_flags, use_bytes=self.options.use_bytes, strict=self.options.strict
473 )
475 if self.options.parser:
476 self.parser = self._build_parser()
477 elif lexer:
478 self.lexer = self._build_lexer()
480 if cache_fn:
481 logger.debug('Saving grammar to cache: %s', cache_fn)
482 try:
483 with FS.open(cache_fn, 'wb') as f:
484 assert cache_sha256 is not None
485 f.write(cache_sha256.encode('utf8') + b'\n')
486 pickle.dump(used_files, f)
487 self.save(f, _LOAD_ALLOWED_OPTIONS)
488 except IOError as e:
489 logger.exception("Failed to save Lark to cache: %r.", cache_fn, e)
491 if __doc__:
492 __doc__ += "\n\n" + LarkOptions.OPTIONS_DOC
494 def _build_lexer(self, dont_ignore: bool=False) -> BasicLexer:
495 lexer_conf = self.lexer_conf
496 if dont_ignore:
497 from copy import copy
498 lexer_conf = copy(lexer_conf)
499 lexer_conf.ignore = ()
500 return BasicLexer(lexer_conf)
502 def _prepare_callbacks(self) -> None:
503 self._callbacks = {}
504 # we don't need these callbacks if we aren't building a tree
505 if self.options.ambiguity != 'forest':
506 self._parse_tree_builder = ParseTreeBuilder(
507 self.rules,
508 self.options.tree_class or Tree,
509 self.options.propagate_positions,
510 self.options.parser != 'lalr' and self.options.ambiguity == 'explicit',
511 self.options.maybe_placeholders
512 )
513 self._callbacks = self._parse_tree_builder.create_callback(self.options.transformer)
514 self._callbacks.update(_get_lexer_callbacks(self.options.transformer, self.terminals))
516 def _build_parser(self) -> "ParsingFrontend":
517 self._prepare_callbacks()
518 _validate_frontend_args(self.options.parser, self.options.lexer)
519 parser_conf = ParserConf(self.rules, self._callbacks, self.options.start)
520 return _construct_parsing_frontend(
521 self.options.parser,
522 self.options.lexer,
523 self.lexer_conf,
524 parser_conf,
525 options=self.options
526 )
528 def save(self, f, exclude_options: Collection[str] = ()) -> None:
529 """Saves the instance into the given file object
531 Useful for caching and multiprocessing.
532 """
533 if self.options.parser != 'lalr':
534 raise NotImplementedError("Lark.save() is only implemented for the LALR(1) parser.")
535 data, m = self.memo_serialize([TerminalDef, Rule])
536 if exclude_options:
537 data["options"] = {n: v for n, v in data["options"].items() if n not in exclude_options}
538 pickle.dump({'data': data, 'memo': m}, f, protocol=pickle.HIGHEST_PROTOCOL)
540 @classmethod
541 def load(cls: Type[_T], f) -> _T:
542 """Loads an instance from the given file object
544 Useful for caching and multiprocessing.
545 """
546 inst = cls.__new__(cls)
547 return inst._load(f)
549 def _deserialize_lexer_conf(self, data: Dict[str, Any], memo: Dict[int, Union[TerminalDef, Rule]], options: LarkOptions) -> LexerConf:
550 lexer_conf = LexerConf.deserialize(data['lexer_conf'], memo)
551 lexer_conf.callbacks = options.lexer_callbacks or {}
552 lexer_conf.re_module = regex if options.regex else re
553 lexer_conf.use_bytes = options.use_bytes
554 lexer_conf.g_regex_flags = options.g_regex_flags
555 lexer_conf.skip_validation = True
556 lexer_conf.postlex = options.postlex
557 return lexer_conf
559 def _load(self: _T, f: Any, **kwargs) -> _T:
560 if isinstance(f, dict):
561 d = f
562 else:
563 d = pickle.load(f)
564 memo_json = d['memo']
565 data = d['data']
567 assert memo_json
568 memo = SerializeMemoizer.deserialize(memo_json, {'Rule': Rule, 'TerminalDef': TerminalDef}, {})
569 if 'grammar' in data:
570 self.grammar = Grammar.deserialize(data['grammar'], memo)
571 options = dict(data['options'])
572 if (set(kwargs) - _LOAD_ALLOWED_OPTIONS) & set(LarkOptions._defaults):
573 raise ConfigurationError("Some options are not allowed when loading a Parser: {}"
574 .format(set(kwargs) - _LOAD_ALLOWED_OPTIONS))
575 options.update(kwargs)
576 self.options = LarkOptions.deserialize(options, memo)
577 self.rules = [Rule.deserialize(r, memo) for r in data['rules']]
578 self.source_path = '<deserialized>'
579 _validate_frontend_args(self.options.parser, self.options.lexer)
580 self.lexer_conf = self._deserialize_lexer_conf(data['parser'], memo, self.options)
581 self.terminals = self.lexer_conf.terminals
582 self._prepare_callbacks()
583 self._terminals_dict = {t.name: t for t in self.terminals}
584 self.parser = _deserialize_parsing_frontend(
585 data['parser'],
586 memo,
587 self.lexer_conf,
588 self._callbacks,
589 self.options, # Not all, but multiple attributes are used
590 )
591 return self
593 @classmethod
594 def _load_from_dict(cls, data, memo, **kwargs):
595 inst = cls.__new__(cls)
596 return inst._load({'data': data, 'memo': memo}, **kwargs)
598 @overload
599 @classmethod
600 def open(
601 cls,
602 grammar_filename: str,
603 rel_to: Optional[str] = None,
604 *,
605 transformer: 'Transformer[Token, _Return_T]',
606 **options: Any,
607 ) -> 'Lark[_Return_T]': ...
609 @overload
610 @classmethod
611 def open(
612 cls,
613 grammar_filename: str,
614 rel_to: Optional[str] = None,
615 **options: Any,
616 ) -> 'Lark[ParseTree]': ...
618 @classmethod
619 def open(cls, grammar_filename: str, rel_to: Optional[str]=None, **options) -> 'Lark':
620 """Create an instance of Lark with the grammar given by its filename
622 If ``rel_to`` is provided, the function will find the grammar filename in relation to it.
624 Example:
626 >>> Lark.open("grammar_file.lark", rel_to=__file__, parser="lalr")
627 Lark(...)
628 """
629 if rel_to:
630 basepath = os.path.dirname(rel_to)
631 grammar_filename = os.path.join(basepath, grammar_filename)
632 with open(grammar_filename, encoding='utf8') as f:
633 return cls(f, **options)
635 @overload
636 @classmethod
637 def open_from_package(
638 cls,
639 package: str,
640 grammar_path: str,
641 search_paths: 'Sequence[str]' = ...,
642 *,
643 transformer: 'Transformer[Token, _Return_T]',
644 **options: Any,
645 ) -> 'Lark[_Return_T]': ...
647 @overload
648 @classmethod
649 def open_from_package(
650 cls,
651 package: str,
652 grammar_path: str,
653 search_paths: 'Sequence[str]' = ...,
654 **options: Any,
655 ) -> 'Lark[ParseTree]': ...
657 @classmethod
658 def open_from_package(cls, package: str, grammar_path: str, search_paths: 'Sequence[str]'=[""], **options) -> 'Lark':
659 """Create an instance of Lark with the grammar loaded from within the package ``package``.
660 This allows grammar loading from zipapps.
662 Imports in the grammar will use the ``package`` and ``search_paths`` provided, through ``FromPackageLoader``
664 Example:
666 Lark.open_from_package(__name__, "example.lark", ("grammars",), parser=...)
667 """
668 package_loader = FromPackageLoader(package, search_paths)
669 full_path, text = package_loader(None, grammar_path)
670 options.setdefault('source_path', full_path)
671 options.setdefault('import_paths', [])
672 options['import_paths'].append(package_loader)
673 return cls(text, **options)
675 def __repr__(self):
676 return 'Lark(open(%r), parser=%r, lexer=%r, ...)' % (self.source_path, self.options.parser, self.options.lexer)
679 def lex(self, text: TextOrSlice, dont_ignore: bool=False) -> Iterator[Token]:
680 """Only lex (and postlex) the text, without parsing it. Only relevant when lexer='basic'
682 When dont_ignore=True, the lexer will return all tokens, even those marked for %ignore.
684 :raises UnexpectedCharacters: In case the lexer cannot find a suitable match.
685 """
686 lexer: Lexer
687 if not hasattr(self, 'lexer') or dont_ignore:
688 lexer = self._build_lexer(dont_ignore)
689 else:
690 lexer = self.lexer
691 lexer_thread = LexerThread.from_text(lexer, text)
692 stream = lexer_thread.lex(None)
693 if self.options.postlex:
694 return self.options.postlex.process(stream)
695 return stream
697 def get_terminal(self, name: str) -> TerminalDef:
698 """Get information about a terminal"""
699 return self._terminals_dict[name]
701 def parse_interactive(self, text: Optional[LarkInput]=None, start: Optional[str]=None) -> 'InteractiveParser':
702 """Start an interactive parsing session. Only works when parser='lalr'.
704 Parameters:
705 text (LarkInput, optional): Text to be parsed. Required for ``resume_parse()``.
706 start (str, optional): Start symbol
708 Returns:
709 A new InteractiveParser instance.
711 See Also: ``Lark.parse()``
712 """
713 return self.parser.parse_interactive(text, start=start)
715 def parse(self, text: LarkInput, start: Optional[str]=None, on_error: 'Optional[Callable[[UnexpectedInput], bool]]'=None) -> _Return_T:
716 """Parse the given text, according to the options provided.
718 Parameters:
719 text (LarkInput): Text to be parsed, as `str` or `bytes`.
720 TextSlice may also be used, but only when lexer='basic' or 'contextual'.
721 If Lark was created with a custom lexer, this may be an object of any type.
722 start (str, optional): Required if Lark was given multiple possible start symbols (using the start option).
723 on_error (function, optional): if provided, will be called on UnexpectedInput error,
724 with the exception as its argument. Return true to resume parsing, or false to raise the exception.
725 LALR only. See examples/advanced/error_handling.py for an example of how to use on_error.
727 Returns:
728 If a transformer is supplied to ``__init__``, returns whatever is the
729 result of the transformation. Otherwise, returns a Tree instance.
731 :raises UnexpectedInput: On a parse error, one of these sub-exceptions will rise:
732 ``UnexpectedCharacters``, ``UnexpectedToken``, or ``UnexpectedEOF``.
733 For convenience, these sub-exceptions also inherit from ``ParserError`` and ``LexerError``.
735 """
736 if on_error is not None and self.options.parser != 'lalr':
737 raise NotImplementedError("The on_error option is only implemented for the LALR(1) parser.")
738 return self.parser.parse(text, start=start, on_error=on_error)
740 def scan(self, text: TextOrSlice, start: Optional[str]=None) -> Iterator['ScanMatch[_Return_T]']:
741 """Scan the input text for non-overlapping matches of this grammar.
742 Only works when ``parser='lalr'`` and without ``postlex``.
744 Greedy parsing: where multiple end positions are valid, the longest is returned.
745 Input that doesn't match the grammar is silently skipped rather than raised.
747 Performance: scan() runs a parse attempt at every position where a match could begin, so its
748 cost is the sum of those attempts' lengths -- in the worst case O(n*m), with n the input
749 length and m the longest possible parse attempt. For best results, choose leading terminals
750 that are rare in the text and reliably indicate the start of a match.
752 A returned match will never start or end with an ignored terminal.
754 User ``lexer_callbacks`` must preserve source positions on returned tokens — use
755 ``Token.update()`` rather than constructing a fresh ``Token``. A callback that raises
756 ``ValueError`` is treated as a failed lex, skipping the candidate match; any other
757 exception propagates.
759 Note: ``lexer_callbacks`` may fire while exploring the input, for regions that yield no match.
761 Note: While ``lexer='basic'`` works, it can be much slower than the contextual lexer. Use is strongly discouraged.
763 Parameters:
764 text (TextOrSlice): Text to be scanned, as ``str``, ``bytes``, or a ``TextSlice`` instance.
765 start (str, optional): Start symbol. Required if Lark was initialized with multiple start symbols.
767 Yields:
768 ``ScanMatch`` instances, each with a ``range`` (a (start, end) tuple)
769 and a ``value`` attribute. ``value`` is a ``Tree`` by default, or
770 whatever the ``transformer`` returns when one was supplied.
772 :raises ConfigurationError: If the configuration doesn't support scanning;
773 scan() requires ``parser='lalr'`` without ``postlex`` or a custom lexer.
774 :raises LexError: If a ``lexer_callback`` returns a token without source positions.
776 See Also: ``Lark.parse()``
777 """
778 if self.options.parser != 'lalr':
779 raise ConfigurationError("scan() requires parser='lalr'")
780 return self.parser.scan(text, start=start)
783###}