Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/pyparsing/core.py: 46%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#
2# core.py
3#
4from __future__ import annotations
6import collections.abc
7from collections import deque
8import os
9import typing
10from typing import (
11 Any,
12 Callable,
13 Generator,
14 NamedTuple,
15 Sequence,
16 TextIO,
17 Union,
18 cast,
19)
20from abc import ABC, abstractmethod
21from enum import Enum
22import string
23import copy
24import warnings
25import re
26import sys
27from collections.abc import Iterable
28import traceback
29import types
30from operator import itemgetter
31from functools import wraps
32from threading import RLock
33from pathlib import Path
35from .warnings import PyparsingDeprecationWarning, PyparsingDiagnosticWarning
36from .util import (
37 _FifoCache,
38 _UnboundedCache,
39 __config_flags,
40 _collapse_string_to_ranges,
41 _convert_escaped_numerics_to_char,
42 _escape_regex_range_chars,
43 _flatten,
44 LRUMemo as _LRUMemo,
45 UnboundedMemo as _UnboundedMemo,
46 deprecate_argument,
47 replaced_by_pep8,
48)
49from .exceptions import *
50from .actions import *
51from .results import ParseResults, _ParseResultsWithOffset
52from .unicode import pyparsing_unicode
54_MAX_INT = sys.maxsize
55str_type: tuple[type, ...] = (str, bytes)
57#
58# Copyright (c) 2003-2022 Paul T. McGuire
59#
60# Permission is hereby granted, free of charge, to any person obtaining
61# a copy of this software and associated documentation files (the
62# "Software"), to deal in the Software without restriction, including
63# without limitation the rights to use, copy, modify, merge, publish,
64# distribute, sublicense, and/or sell copies of the Software, and to
65# permit persons to whom the Software is furnished to do so, subject to
66# the following conditions:
67#
68# The above copyright notice and this permission notice shall be
69# included in all copies or substantial portions of the Software.
70#
71# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
72# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
73# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
74# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
75# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
76# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
77# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
78#
80from functools import cached_property
83class __compat__(__config_flags):
84 """
85 A cross-version compatibility configuration for pyparsing features that will be
86 released in a future version. By setting values in this configuration to True,
87 those features can be enabled in prior versions for compatibility development
88 and testing.
90 - ``collect_all_And_tokens`` - flag to enable fix for Issue #63 that fixes erroneous grouping
91 of results names when an :class:`And` expression is nested within an :class:`Or` or :class:`MatchFirst`;
92 maintained for compatibility, but setting to ``False`` no longer restores pre-2.3.1
93 behavior
94 """
96 _type_desc = "compatibility"
98 collect_all_And_tokens = True
100 _all_names = [__ for __ in locals() if not __.startswith("_")]
101 _fixed_names = """
102 collect_all_And_tokens
103 """.split()
106class __diag__(__config_flags):
107 _type_desc = "diagnostic"
109 warn_multiple_tokens_in_named_alternation = False
110 warn_ungrouped_named_tokens_in_collection = False
111 warn_name_set_on_empty_Forward = False
112 warn_on_parse_using_empty_Forward = False
113 warn_on_assignment_to_Forward = False
114 warn_on_multiple_string_args_to_oneof = False
115 warn_on_match_first_with_lshift_operator = False
116 enable_debug_on_named_expressions = False
118 _all_names = [__ for __ in locals() if not __.startswith("_")]
119 _warning_names = [name for name in _all_names if name.startswith("warn")]
120 _debug_names = [name for name in _all_names if name.startswith("enable_debug")]
122 @classmethod
123 def enable_all_warnings(cls) -> None:
124 for name in cls._warning_names:
125 cls.enable(name)
128class Diagnostics(Enum):
129 """
130 Diagnostic configuration (all default to disabled)
132 - ``warn_multiple_tokens_in_named_alternation`` - flag to enable warnings when a results
133 name is defined on a :class:`MatchFirst` or :class:`Or` expression with one or more :class:`And` subexpressions
134 - ``warn_ungrouped_named_tokens_in_collection`` - flag to enable warnings when a results
135 name is defined on a containing expression with ungrouped subexpressions that also
136 have results names
137 - ``warn_name_set_on_empty_Forward`` - flag to enable warnings when a :class:`Forward` is defined
138 with a results name, but has no contents defined
139 - ``warn_on_parse_using_empty_Forward`` - flag to enable warnings when a :class:`Forward` is
140 defined in a grammar but has never had an expression attached to it
141 - ``warn_on_assignment_to_Forward`` - flag to enable warnings when a :class:`Forward` is defined
142 but is overwritten by assigning using ``'='`` instead of ``'<<='`` or ``'<<'``
143 - ``warn_on_multiple_string_args_to_oneof`` - flag to enable warnings when :class:`one_of` is
144 incorrectly called with multiple str arguments
145 - ``enable_debug_on_named_expressions`` - flag to auto-enable debug on all subsequent
146 calls to :class:`ParserElement.set_name`
148 Diagnostics are enabled/disabled by calling :class:`enable_diag` and :class:`disable_diag`.
149 All warnings can be enabled by calling :class:`enable_all_warnings`.
150 """
152 warn_multiple_tokens_in_named_alternation = 0
153 warn_ungrouped_named_tokens_in_collection = 1
154 warn_name_set_on_empty_Forward = 2
155 warn_on_parse_using_empty_Forward = 3
156 warn_on_assignment_to_Forward = 4
157 warn_on_multiple_string_args_to_oneof = 5
158 warn_on_match_first_with_lshift_operator = 6
159 enable_debug_on_named_expressions = 7
162def enable_diag(diag_enum: Diagnostics) -> None:
163 """
164 Enable a global pyparsing diagnostic flag (see :class:`Diagnostics`).
165 """
166 __diag__.enable(diag_enum.name)
169def disable_diag(diag_enum: Diagnostics) -> None:
170 """
171 Disable a global pyparsing diagnostic flag (see :class:`Diagnostics`).
172 """
173 __diag__.disable(diag_enum.name)
176def enable_all_warnings() -> None:
177 """
178 Enable all global pyparsing diagnostic warnings (see :class:`Diagnostics`).
179 """
180 __diag__.enable_all_warnings()
183# hide abstract class
184del __config_flags
187def _should_enable_warnings(
188 cmd_line_warn_options: typing.Iterable[str], warn_env_var: typing.Optional[str]
189) -> bool:
190 enable = bool(warn_env_var)
191 for warn_opt in cmd_line_warn_options:
192 w_action, w_message, w_category, w_module, w_line = (warn_opt + "::::").split(
193 ":"
194 )[:5]
195 if not w_action.lower().startswith("i") and (
196 not (w_message or w_category or w_module) or w_module == "pyparsing"
197 ):
198 enable = True
199 elif w_action.lower().startswith("i") and w_module in ("pyparsing", ""):
200 enable = False
201 return enable
204if _should_enable_warnings(
205 sys.warnoptions, os.environ.get("PYPARSINGENABLEALLWARNINGS")
206):
207 enable_all_warnings()
210# build list of single arg builtins, that can be used as parse actions
211# fmt: off
212_single_arg_builtins = {
213 sum, len, sorted, reversed, list, tuple, set, any, all, min, max
214}
215# fmt: on
217_generatorType = types.GeneratorType
218ParseImplReturnType = tuple[int, Any]
219PostParseReturnType = Union[ParseResults, Sequence[ParseResults]]
221ParseCondition = Union[
222 Callable[[], bool],
223 Callable[[ParseResults], bool],
224 Callable[[int, ParseResults], bool],
225 Callable[[str, int, ParseResults], bool],
226]
227ParseFailAction = Callable[[str, int, "ParserElement", Exception], None]
228DebugStartAction = Callable[[str, int, "ParserElement", bool], None]
229DebugSuccessAction = Callable[
230 [str, int, int, "ParserElement", ParseResults, bool], None
231]
232DebugExceptionAction = Callable[[str, int, "ParserElement", Exception, bool], None]
235alphas: str = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
236identchars: str = pyparsing_unicode.Latin1.identchars
237identbodychars: str = pyparsing_unicode.Latin1.identbodychars
238nums: str = "0123456789"
239hexnums: str = "0123456789ABCDEFabcdef"
240alphanums: str = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
241printables: str = (
242 '!"'
243 "#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ"
244 "[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~"
245)
248class _ParseActionIndexError(Exception):
249 """
250 Internal wrapper around IndexError so that IndexErrors raised inside
251 parse actions aren't misinterpreted as IndexErrors raised inside
252 ParserElement parseImpl methods.
253 """
255 def __init__(self, msg: str, exc: BaseException) -> None:
256 self.msg: str = msg
257 self.exc: BaseException = exc
260_trim_arity_call_line: traceback.StackSummary = None # type: ignore[assignment]
261pa_call_line_synth = ()
264def _trim_arity(func, max_limit=3):
265 """decorator to trim function calls to match the arity of the target"""
266 global _trim_arity_call_line, pa_call_line_synth
268 if func in _single_arg_builtins:
269 return lambda s, l, t: func(t)
271 limit = 0
272 found_arity = False
274 # synthesize what would be returned by traceback.extract_stack at the call to
275 # user's parse action 'func', so that we don't incur call penalty at parse time
277 # fmt: off
278 LINE_DIFF = 9
279 # IF ANY CODE CHANGES, EVEN JUST COMMENTS OR BLANK LINES, BETWEEN THE NEXT LINE AND
280 # THE CALL TO FUNC INSIDE WRAPPER, LINE_DIFF MUST BE MODIFIED!!!!
281 _trim_arity_call_line = _trim_arity_call_line or traceback.extract_stack(limit=2)[-1]
282 pa_call_line_synth = pa_call_line_synth or (_trim_arity_call_line[0], _trim_arity_call_line[1] + LINE_DIFF)
284 def wrapper(*args):
285 nonlocal found_arity, limit
286 if found_arity:
287 return func(*args[limit:])
288 while 1:
289 try:
290 ret = func(*args[limit:])
291 found_arity = True
292 return ret
293 except TypeError as te:
294 # re-raise TypeErrors if they did not come from our arity testing
295 if found_arity:
296 raise
297 else:
298 tb = te.__traceback__
299 frames = traceback.extract_tb(tb, limit=2)
300 frame_summary = frames[-1]
301 trim_arity_type_error = (
302 [frame_summary[:2]][-1][:2] == pa_call_line_synth
303 )
304 del tb
306 if trim_arity_type_error:
307 if limit < max_limit:
308 limit += 1
309 continue
311 raise
312 except IndexError as ie:
313 # wrap IndexErrors inside a _ParseActionIndexError
314 raise _ParseActionIndexError(
315 "IndexError raised in parse action", ie
316 ).with_traceback(None)
317 # fmt: on
319 # copy func name to wrapper for sensible debug output
320 # (can't use functools.wraps, since that messes with function signature)
321 func_name = getattr(func, "__name__", getattr(func, "__class__").__name__)
322 wrapper.__name__ = func_name
323 wrapper.__doc__ = func.__doc__
325 return wrapper
328def condition_as_parse_action(
329 fn: ParseCondition, message: typing.Optional[str] = None, fatal: bool = False
330) -> ParseAction:
331 """
332 Function to convert a simple predicate function that returns ``True`` or ``False``
333 into a parse action. Can be used in places when a parse action is required
334 and :meth:`ParserElement.add_condition` cannot be used (such as when adding a condition
335 to an operator level in :class:`infix_notation`).
337 Optional keyword arguments:
339 :param message: define a custom message to be used in the raised exception
340 :param fatal: if ``True``, will raise :class:`ParseFatalException`
341 to stop parsing immediately;
342 otherwise will raise :class:`ParseException`
344 """
345 msg = message if message is not None else "failed user-defined condition"
346 exc_type = ParseFatalException if fatal else ParseException
347 fn = _trim_arity(fn)
349 @wraps(fn)
350 def pa(s, l, t):
351 if not bool(fn(s, l, t)):
352 raise exc_type(s, l, msg)
354 return pa
357# control characters escaped so they can't corrupt the printed debug line
358# (for example a stray '\r' would otherwise return the cursor to column 0)
359_debug_control_char_map = {c: repr(chr(c))[1:-1] for c in (*range(0x20), 0x7F)}
362def _default_start_debug_action(
363 instring: str, loc: int, expr: ParserElement, cache_hit: bool = False
364):
365 cache_hit_str = "*" if cache_hit else ""
366 current_col = col(loc, instring)
367 current_line = line(loc, instring)
368 escaped_line = current_line.translate(_debug_control_char_map)
369 # keep the caret under the match location after escaping widens the line
370 caret_col = (
371 len(current_line[: current_col - 1].translate(_debug_control_char_map)) + 1
372 )
373 print(
374 (
375 f"{cache_hit_str}Match {expr} at loc {loc}({lineno(loc, instring)},{current_col})\n"
376 f" {escaped_line}\n"
377 f" {'^':>{caret_col}}"
378 )
379 )
382def _default_success_debug_action(
383 instring: str,
384 startloc: int,
385 endloc: int,
386 expr: ParserElement,
387 toks: ParseResults,
388 cache_hit: bool = False,
389):
390 cache_hit_str = "*" if cache_hit else ""
391 print(f"{cache_hit_str}Matched {expr} -> {toks.as_list()}")
394def _default_exception_debug_action(
395 instring: str,
396 loc: int,
397 expr: ParserElement,
398 exc: Exception,
399 cache_hit: bool = False,
400):
401 cache_hit_str = "*" if cache_hit else ""
402 print(f"{cache_hit_str}Match {expr} failed, {type(exc).__name__} raised: {exc}")
405def null_debug_action(*args):
406 """'Do-nothing' debug action, to suppress debugging output during parsing."""
409class ParserElement(ABC):
410 """Abstract base level parser element class."""
412 DEFAULT_WHITE_CHARS: str = " \n\t\r"
413 verbose_stacktrace: bool = False
414 _literalStringClass: type = None # type: ignore[assignment]
416 @staticmethod
417 def set_default_whitespace_chars(chars: str) -> None:
418 r"""
419 Overrides the default whitespace chars
421 Example:
423 .. doctest::
425 # default whitespace chars are space, <TAB> and newline
426 >>> Word(alphas)[1, ...].parse_string("abc def\nghi jkl")
427 ParseResults(['abc', 'def', 'ghi', 'jkl'], {})
429 # change to just treat newline as significant
430 >>> ParserElement.set_default_whitespace_chars(" \t")
431 >>> Word(alphas)[1, ...].parse_string("abc def\nghi jkl")
432 ParseResults(['abc', 'def'], {})
434 # Reset to default
435 >>> ParserElement.set_default_whitespace_chars(" \n\t\r")
436 """
437 ParserElement.DEFAULT_WHITE_CHARS = chars
439 # update whitespace all parse expressions defined in this module
440 for expr in _builtin_exprs:
441 if expr.copyDefaultWhiteChars:
442 expr.whiteChars = set(chars)
444 @staticmethod
445 def inline_literals_using(cls: type) -> None:
446 """
447 Set class to be used for inclusion of string literals into a parser.
449 Example:
451 .. doctest::
452 :options: +NORMALIZE_WHITESPACE
454 # default literal class used is Literal
455 >>> integer = Word(nums)
456 >>> date_str = (
457 ... integer("year") + '/'
458 ... + integer("month") + '/'
459 ... + integer("day")
460 ... )
462 >>> date_str.parse_string("1999/12/31")
463 ParseResults(['1999', '/', '12', '/', '31'],
464 {'year': '1999', 'month': '12', 'day': '31'})
466 # change to Suppress
467 >>> ParserElement.inline_literals_using(Suppress)
468 >>> date_str = (
469 ... integer("year") + '/'
470 ... + integer("month") + '/'
471 ... + integer("day")
472 ... )
474 >>> date_str.parse_string("1999/12/31")
475 ParseResults(['1999', '12', '31'],
476 {'year': '1999', 'month': '12', 'day': '31'})
478 # Reset
479 >>> ParserElement.inline_literals_using(Literal)
480 """
481 ParserElement._literalStringClass = cls
483 @classmethod
484 def using_each(cls, seq, **class_kwargs):
485 """
486 Yields a sequence of ``class(obj, **class_kwargs)`` for obj in seq.
488 Example:
490 .. testcode::
492 LPAR, RPAR, LBRACE, RBRACE, SEMI = Suppress.using_each("(){};")
494 .. versionadded:: 3.1.0
495 """
496 yield from (cls(obj, **class_kwargs) for obj in seq)
498 class DebugActions(NamedTuple):
499 debug_try: typing.Optional[DebugStartAction]
500 debug_match: typing.Optional[DebugSuccessAction]
501 debug_fail: typing.Optional[DebugExceptionAction]
503 def __init__(self, savelist: bool = False) -> None:
504 self.parseAction: list[ParseAction] = list()
505 self.failAction: typing.Optional[ParseFailAction] = None
506 self.customName: str = None # type: ignore[assignment]
507 self._defaultName: typing.Optional[str] = None
508 self.resultsName: str = None # type: ignore[assignment]
509 self.saveAsList: bool = savelist
510 self.skipWhitespace: bool = True
511 self.whiteChars: set[str] = set(ParserElement.DEFAULT_WHITE_CHARS)
512 self.copyDefaultWhiteChars: bool = True
513 # used when checking for left-recursion
514 self._may_return_empty: bool = False
515 self.keepTabs: bool = False
516 self.ignoreExprs: list[ParserElement] = list()
517 self.debug: bool = False
518 self.streamlined: bool = False
519 # optimize exception handling for subclasses that don't advance parse index
520 self.mayIndexError: bool = True
521 self.errmsg: Union[str, None] = ""
522 # mark results names as modal (report only last) or cumulative (list all)
523 self.modalResults: bool = True
524 # custom debug actions
525 self.debugActions = self.DebugActions(None, None, None)
526 # avoid redundant calls to preParse
527 self.callPreparse: bool = True
528 self.callDuringTry: bool = False
529 self.suppress_warnings_: list[Diagnostics] = []
530 self.show_in_diagram: bool = True
532 @property
533 def mayReturnEmpty(self) -> bool:
534 """
535 .. deprecated:: 3.3.0
536 use _may_return_empty instead.
537 """
538 return self._may_return_empty
540 @mayReturnEmpty.setter
541 def mayReturnEmpty(self, value) -> None:
542 """
543 .. deprecated:: 3.3.0
544 use _may_return_empty instead.
545 """
546 self._may_return_empty = value
548 def suppress_warning(self, warning_type: Diagnostics) -> ParserElement:
549 """
550 Suppress warnings emitted for a particular diagnostic on this expression.
552 Example:
554 .. doctest::
556 >>> label = pp.Word(pp.alphas)
558 # Normally using an empty Forward in a grammar
559 # would print a warning, but we can suppress that
560 >>> base = pp.Forward().suppress_warning(
561 ... pp.Diagnostics.warn_on_parse_using_empty_Forward)
563 >>> grammar = base | label
564 >>> print(grammar.parse_string("x"))
565 ['x']
566 """
567 self.suppress_warnings_.append(warning_type)
568 return self
570 def visit_all(self):
571 """General-purpose method to yield all expressions and sub-expressions
572 in a grammar. Typically just for internal use.
573 """
574 to_visit = deque([self])
575 seen = set()
576 while to_visit:
577 cur = to_visit.popleft()
579 # guard against looping forever through recursive grammars
580 if cur in seen:
581 continue
582 seen.add(cur)
584 to_visit.extend(cur.recurse())
585 yield cur
587 def copy(self) -> ParserElement:
588 """
589 Make a copy of this :class:`ParserElement`. Useful for defining
590 different parse actions for the same parsing pattern, using copies of
591 the original parse element.
593 Example:
595 .. testcode::
597 integer = Word(nums).set_parse_action(
598 lambda toks: int(toks[0]))
599 integerK = integer.copy().add_parse_action(
600 lambda toks: toks[0] * 1024) + Suppress("K")
601 integerM = integer.copy().add_parse_action(
602 lambda toks: toks[0] * 1024 * 1024) + Suppress("M")
604 print(
605 (integerK | integerM | integer)[1, ...].parse_string(
606 "5K 100 640K 256M")
607 )
609 prints:
611 .. testoutput::
613 [5120, 100, 655360, 268435456]
615 Equivalent form of ``expr.copy()`` is just ``expr()``:
617 .. testcode::
619 integerM = integer().add_parse_action(
620 lambda toks: toks[0] * 1024 * 1024) + Suppress("M")
621 """
622 cpy = copy.copy(self)
623 cpy.parseAction = self.parseAction[:]
624 cpy.ignoreExprs = self.ignoreExprs[:]
625 if self.copyDefaultWhiteChars:
626 cpy.whiteChars = set(ParserElement.DEFAULT_WHITE_CHARS)
627 return cpy
629 def set_results_name(
630 self, name: str, list_all_matches: bool = False, **kwargs
631 ) -> ParserElement:
632 """
633 Define name for referencing matching tokens as a nested attribute
634 of the returned parse results.
636 Normally, results names are assigned as you would assign keys in a dict:
637 any existing value is overwritten by later values. If it is necessary to
638 keep all values captured for a particular results name, call ``set_results_name``
639 with ``list_all_matches`` = True.
641 NOTE: ``set_results_name`` returns a *copy* of the original :class:`ParserElement` object;
642 this is so that the client can define a basic element, such as an
643 integer, and reference it in multiple places with different names.
645 You can also set results names using the abbreviated syntax,
646 ``expr("name")`` in place of ``expr.set_results_name("name")``
647 - see :meth:`__call__`. If ``list_all_matches`` is required, use
648 ``expr("name*")``.
650 Example:
652 .. testcode::
654 integer = Word(nums)
655 date_str = (integer.set_results_name("year") + '/'
656 + integer.set_results_name("month") + '/'
657 + integer.set_results_name("day"))
659 # equivalent form:
660 date_str = integer("year") + '/' + integer("month") + '/' + integer("day")
661 """
662 listAllMatches: bool = deprecate_argument(kwargs, "listAllMatches", False)
664 list_all_matches = listAllMatches or list_all_matches
665 return self._setResultsName(name, list_all_matches)
667 def _setResultsName(self, name, list_all_matches=False) -> ParserElement:
668 if name is None:
669 return self
670 newself = self.copy()
671 if name.endswith("*"):
672 name = name[:-1]
673 list_all_matches = True
674 newself.resultsName = name
675 newself.modalResults = not list_all_matches
676 return newself
678 def set_break(self, break_flag: bool = True) -> ParserElement:
679 """
680 Method to invoke the Python pdb debugger when this element is
681 about to be parsed. Set ``break_flag`` to ``True`` to enable, ``False`` to
682 disable.
683 """
684 if break_flag:
685 _parseMethod = self._parse
687 def breaker(instring, loc, do_actions=True, callPreParse=True):
688 # this call to breakpoint() is intentional, not a checkin error
689 breakpoint()
690 return _parseMethod(instring, loc, do_actions, callPreParse)
692 breaker._originalParseMethod = _parseMethod # type: ignore [attr-defined]
693 self._parse = breaker # type: ignore [method-assign]
694 elif hasattr(self._parse, "_originalParseMethod"):
695 self._parse = self._parse._originalParseMethod # type: ignore [method-assign]
696 return self
698 def set_parse_action(
699 self, *fns: ParseAction, call_during_try: bool = False, **kwargs: Any
700 ) -> ParserElement:
701 """
702 Define one or more actions to perform when successfully matching parse element definition.
704 Parse actions can be called to perform data conversions, do extra validation,
705 update external data structures, or enhance or replace the parsed tokens.
706 Each parse action ``fn`` is a callable method with 0-3 arguments, called as
707 ``fn(s, loc, toks)`` , ``fn(loc, toks)`` , ``fn(toks)`` , or just ``fn()`` , where:
709 - ``s`` = the original string being parsed (see note below)
710 - ``loc`` = the location of the matching substring
711 - ``toks`` = a list of the matched tokens, packaged as a :class:`ParseResults` object
713 The parsed tokens are passed to the parse action as ParseResults. They can be
714 modified in place using list-style append, extend, and pop operations to update
715 the parsed list elements; and with dictionary-style item set and del operations
716 to add, update, or remove any named results. If the tokens are modified in place,
717 it is not necessary to return them with a return statement.
719 Parse actions can also completely replace the given tokens, with another ``ParseResults``
720 object, or with some entirely different object (common for parse actions that perform data
721 conversions). A convenient way to build a new parse result is to define the values
722 using a dict, and then create the return value using :class:`ParseResults.from_dict`.
724 If None is passed as the ``fn`` parse action, all previously added parse actions for this
725 expression are cleared.
727 Optional keyword arguments:
729 :param call_during_try: (default= ``False``) indicate if parse action
730 should be run during lookaheads and alternate
731 testing. For parse actions that have side
732 effects, it is important to only call the parse
733 action once it is determined that it is being
734 called as part of a successful parse.
735 For parse actions that perform additional
736 validation, then ``call_during_try`` should
737 be passed as True, so that the validation code
738 is included in the preliminary "try" parses.
740 .. Note::
741 The default parsing behavior is to expand tabs in the input string
742 before starting the parsing process.
743 See :meth:`parse_string` for more information on parsing strings
744 containing ``<TAB>`` s, and suggested methods to maintain a
745 consistent view of the parsed string, the parse location, and
746 line and column positions within the parsed string.
748 Example: Parse dates in the form ``YYYY/MM/DD``
749 -----------------------------------------------
751 Setup code:
753 .. testcode::
755 def convert_to_int(toks):
756 '''a parse action to convert toks from str to int
757 at parse time'''
758 return int(toks[0])
760 def is_valid_date(instring, loc, toks):
761 '''a parse action to verify that the date is a valid date'''
762 from datetime import date
763 year, month, day = toks[::2]
764 try:
765 date(year, month, day)
766 except ValueError:
767 raise ParseException(instring, loc, "invalid date given")
769 integer = Word(nums)
770 date_str = integer + '/' + integer + '/' + integer
772 # add parse actions
773 integer.set_parse_action(convert_to_int)
774 date_str.set_parse_action(is_valid_date)
776 Successful parse - note that integer fields are converted to ints:
778 .. testcode::
780 print(date_str.parse_string("1999/12/31"))
782 prints:
784 .. testoutput::
786 [1999, '/', 12, '/', 31]
788 Failure - invalid date:
790 .. testcode::
792 date_str.parse_string("1999/13/31")
794 prints:
796 .. testoutput::
798 Traceback (most recent call last):
799 ParseException: invalid date given, found '1999' ...
800 """
801 callDuringTry: bool = deprecate_argument(kwargs, "callDuringTry", False)
803 if list(fns) == [None]:
804 self.parseAction.clear()
805 return self
807 if not all(callable(fn) for fn in fns):
808 raise TypeError("parse actions must be callable")
809 self.parseAction[:] = [_trim_arity(fn) for fn in fns]
810 self.callDuringTry = self.callDuringTry or call_during_try or callDuringTry
812 return self
814 def add_parse_action(
815 self, *fns: ParseAction, call_during_try: bool = False, **kwargs: Any
816 ) -> ParserElement:
817 """
818 Add one or more parse actions to expression's list of parse actions. See :class:`set_parse_action`.
820 See examples in :class:`copy`.
821 """
822 callDuringTry: bool = deprecate_argument(kwargs, "callDuringTry", False)
824 self.parseAction += [_trim_arity(fn) for fn in fns]
825 self.callDuringTry = self.callDuringTry or callDuringTry or call_during_try
826 return self
828 def add_condition(
829 self, *fns: ParseCondition, call_during_try: bool = False, **kwargs: Any
830 ) -> ParserElement:
831 """Add a boolean predicate function to expression's list of parse actions. See
832 :class:`set_parse_action` for function call signatures. Unlike ``set_parse_action``,
833 functions passed to ``add_condition`` need to return boolean success/fail of the condition.
835 Optional keyword arguments:
837 - ``message`` = define a custom message to be used in the raised exception
838 - ``fatal`` = if True, will raise ParseFatalException to stop parsing immediately; otherwise will raise
839 ParseException
840 - ``call_during_try`` = boolean to indicate if this method should be called during internal tryParse calls,
841 default=False
843 Example:
845 .. doctest::
846 :options: +NORMALIZE_WHITESPACE
848 >>> integer = Word(nums).set_parse_action(lambda toks: int(toks[0]))
849 >>> year_int = integer.copy().add_condition(
850 ... lambda toks: toks[0] >= 2000,
851 ... message="Only support years 2000 and later")
852 >>> date_str = year_int + '/' + integer + '/' + integer
854 >>> result = date_str.parse_string("1999/12/31")
855 Traceback (most recent call last):
856 ParseException: Only support years 2000 and later...
857 """
858 callDuringTry: bool = deprecate_argument(kwargs, "callDuringTry", False)
860 for fn in fns:
861 self.parseAction.append(
862 condition_as_parse_action(
863 fn,
864 message=str(kwargs.get("message")),
865 fatal=bool(kwargs.get("fatal", False)),
866 )
867 )
869 self.callDuringTry = self.callDuringTry or call_during_try or callDuringTry
870 return self
872 def set_fail_action(self, fn: ParseFailAction) -> ParserElement:
873 """
874 Define action to perform if parsing fails at this expression.
875 Fail acton fn is a callable function that takes the arguments
876 ``fn(s, loc, expr, err)`` where:
878 - ``s`` = string being parsed
879 - ``loc`` = location where expression match was attempted and failed
880 - ``expr`` = the parse expression that failed
881 - ``err`` = the exception thrown
883 The function returns no value. It may throw :class:`ParseFatalException`
884 if it is desired to stop parsing immediately."""
885 self.failAction = fn
886 return self
888 def _skipIgnorables(self, instring: str, loc: int) -> int:
889 if not self.ignoreExprs:
890 return loc
891 exprsFound = True
892 ignore_expr_fns = [e._parse for e in self.ignoreExprs]
893 last_loc = loc
894 while exprsFound:
895 exprsFound = False
896 for ignore_fn in ignore_expr_fns:
897 try:
898 while 1:
899 loc, dummy = ignore_fn(instring, loc)
900 exprsFound = True
901 except ParseException:
902 pass
903 # check if all ignore exprs matched but didn't actually advance the parse location
904 if loc == last_loc:
905 break
906 last_loc = loc
907 return loc
909 def preParse(self, instring: str, loc: int) -> int:
910 if self.ignoreExprs:
911 loc = self._skipIgnorables(instring, loc)
913 if self.skipWhitespace:
914 instrlen = len(instring)
915 white_chars = self.whiteChars
916 while loc < instrlen and instring[loc] in white_chars:
917 loc += 1
919 return loc
921 def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType:
922 return loc, []
924 def postParse(self, instring, loc, tokenlist):
925 return tokenlist
927 # @profile
928 def _parseNoCache(
929 self, instring, loc, do_actions=True, callPreParse=True
930 ) -> tuple[int, ParseResults]:
931 debugging = self.debug # and do_actions)
932 len_instring = len(instring)
934 if debugging or self.failAction:
935 # print("Match {} at loc {}({}, {})".format(self, loc, lineno(loc, instring), col(loc, instring)))
936 try:
937 if callPreParse and self.callPreparse:
938 pre_loc = self.preParse(instring, loc)
939 else:
940 pre_loc = loc
941 tokens_start = pre_loc
942 if self.debugActions.debug_try:
943 self.debugActions.debug_try(instring, tokens_start, self, False)
944 if self.mayIndexError or pre_loc >= len_instring:
945 try:
946 loc, tokens = self.parseImpl(instring, pre_loc, do_actions)
947 except IndexError:
948 raise ParseException(instring, len_instring, self.errmsg, self)
949 else:
950 loc, tokens = self.parseImpl(instring, pre_loc, do_actions)
951 except Exception as err:
952 # print("Exception raised:", err)
953 if self.debugActions.debug_fail:
954 self.debugActions.debug_fail(
955 instring, tokens_start, self, err, False
956 )
957 if self.failAction:
958 self.failAction(instring, tokens_start, self, err)
959 raise
960 else:
961 if callPreParse and self.callPreparse:
962 pre_loc = self.preParse(instring, loc)
963 else:
964 pre_loc = loc
965 tokens_start = pre_loc
966 if self.mayIndexError or pre_loc >= len_instring:
967 try:
968 loc, tokens = self.parseImpl(instring, pre_loc, do_actions)
969 except IndexError:
970 raise ParseException(instring, len_instring, self.errmsg, self)
971 else:
972 loc, tokens = self.parseImpl(instring, pre_loc, do_actions)
974 tokens = self.postParse(instring, loc, tokens)
976 ret_tokens = ParseResults(
977 tokens, self.resultsName, aslist=self.saveAsList, modal=self.modalResults
978 )
979 if self.parseAction and (do_actions or self.callDuringTry):
980 if debugging:
981 try:
982 for fn in self.parseAction:
983 try:
984 tokens = fn(instring, tokens_start, ret_tokens) # type: ignore [call-arg, arg-type]
985 except IndexError as parse_action_exc:
986 exc = ParseException("exception raised in parse action")
987 raise exc from parse_action_exc
989 if tokens is not None and tokens is not ret_tokens:
990 if isinstance(tokens, (ParseResults, list)) or tokens == ():
991 ret_tokens = ParseResults(
992 tokens,
993 self.resultsName,
994 aslist=self.saveAsList
995 and isinstance(tokens, (ParseResults, list)),
996 modal=self.modalResults,
997 )
998 else:
999 ret_tokens = ParseResults(
1000 [tokens],
1001 self.resultsName,
1002 aslist=False,
1003 modal=self.modalResults,
1004 )
1005 except Exception as err:
1006 # print "Exception raised in user parse action:", err
1007 if self.debugActions.debug_fail:
1008 self.debugActions.debug_fail(
1009 instring, tokens_start, self, err, False
1010 )
1011 raise
1012 else:
1013 for fn in self.parseAction:
1014 try:
1015 tokens = fn(instring, tokens_start, ret_tokens) # type: ignore [call-arg, arg-type]
1016 except IndexError as parse_action_exc:
1017 exc = ParseException("exception raised in parse action")
1018 raise exc from parse_action_exc
1020 if tokens is not None and tokens is not ret_tokens:
1021 if isinstance(tokens, (ParseResults, list)) or tokens == ():
1022 ret_tokens = ParseResults(
1023 tokens,
1024 self.resultsName,
1025 aslist=self.saveAsList
1026 and isinstance(tokens, (ParseResults, list)),
1027 modal=self.modalResults,
1028 )
1029 else:
1030 ret_tokens = ParseResults(
1031 [tokens],
1032 self.resultsName,
1033 aslist=False,
1034 modal=self.modalResults,
1035 )
1036 if debugging:
1037 # print("Matched", self, "->", ret_tokens.as_list())
1038 if self.debugActions.debug_match:
1039 self.debugActions.debug_match(
1040 instring, tokens_start, loc, self, ret_tokens, False
1041 )
1043 return loc, ret_tokens
1045 def try_parse(
1046 self,
1047 instring: str,
1048 loc: int,
1049 *,
1050 raise_fatal: bool = False,
1051 do_actions: bool = False,
1052 ) -> int:
1053 try:
1054 return self._parse(instring, loc, do_actions=do_actions)[0]
1055 except ParseFatalException:
1056 if raise_fatal:
1057 raise
1058 raise ParseException(instring, loc, self.errmsg, self)
1060 def can_parse_next(self, instring: str, loc: int, do_actions: bool = False) -> bool:
1061 try:
1062 self.try_parse(instring, loc, do_actions=do_actions)
1063 except (ParseException, IndexError):
1064 return False
1065 else:
1066 return True
1068 # cache for left-recursion in Forward references
1069 recursion_lock = RLock()
1070 recursion_memos: collections.abc.MutableMapping[
1071 tuple[int, Forward, bool], tuple[int, Union[ParseResults, Exception]]
1072 ] = {}
1074 class _CacheType(typing.Protocol):
1075 """
1076 Class to be used for packrat and left-recursion cacheing of results
1077 and exceptions.
1078 """
1080 not_in_cache: bool
1082 def get(self, *args) -> typing.Any: ...
1084 def set(self, *args) -> None: ...
1086 def clear(self) -> None: ...
1088 class NullCache(dict):
1089 """
1090 A null cache type for initialization of the packrat_cache class variable.
1091 If/when enable_packrat() is called, this null cache will be replaced by a
1092 proper _CacheType class instance.
1093 """
1095 not_in_cache: bool = True
1097 def get(self, *args) -> typing.Any: ...
1099 def set(self, *args) -> None: ...
1101 def clear(self) -> None: ...
1103 # class-level argument cache for optimizing repeated calls when backtracking
1104 # through recursive expressions
1105 packrat_cache: _CacheType = NullCache()
1106 packrat_cache_lock = RLock()
1107 packrat_cache_stats = [0, 0]
1109 # this method gets repeatedly called during backtracking with the same arguments -
1110 # we can cache these arguments and save ourselves the trouble of re-parsing the contained expression
1111 def _parseCache(
1112 self, instring, loc, do_actions=True, callPreParse=True
1113 ) -> tuple[int, ParseResults]:
1114 HIT, MISS = 0, 1
1115 lookup = (self, instring, loc, callPreParse, do_actions)
1116 with ParserElement.packrat_cache_lock:
1117 cache = ParserElement.packrat_cache
1118 value = cache.get(lookup)
1119 if value is cache.not_in_cache:
1120 ParserElement.packrat_cache_stats[MISS] += 1
1121 try:
1122 value = self._parseNoCache(instring, loc, do_actions, callPreParse)
1123 except ParseBaseException as pe:
1124 # cache a copy of the exception, without the traceback
1125 cache.set(lookup, pe.__class__(*pe.args))
1126 raise
1127 else:
1128 cache.set(lookup, (value[0], value[1].copy(), loc))
1129 return value
1130 else:
1131 ParserElement.packrat_cache_stats[HIT] += 1
1132 if self.debug and self.debugActions.debug_try:
1133 try:
1134 self.debugActions.debug_try(instring, loc, self, cache_hit=True) # type: ignore [call-arg]
1135 except TypeError:
1136 pass
1137 if isinstance(value, Exception):
1138 if self.debug and self.debugActions.debug_fail:
1139 try:
1140 self.debugActions.debug_fail(
1141 instring, loc, self, value, cache_hit=True # type: ignore [call-arg]
1142 )
1143 except TypeError:
1144 pass
1145 raise value
1147 value = cast(tuple[int, ParseResults, int], value)
1148 loc_, result, endloc = value[0], value[1].copy(), value[2]
1149 if self.debug and self.debugActions.debug_match:
1150 try:
1151 self.debugActions.debug_match(
1152 instring, loc_, endloc, self, result, cache_hit=True # type: ignore [call-arg]
1153 )
1154 except TypeError:
1155 pass
1157 return loc_, result
1159 _parse = _parseNoCache
1161 @staticmethod
1162 def reset_cache() -> None:
1163 """
1164 Clears caches used by packrat and left-recursion.
1165 """
1166 with ParserElement.packrat_cache_lock:
1167 ParserElement.packrat_cache.clear()
1168 ParserElement.packrat_cache_stats[:] = [0] * len(
1169 ParserElement.packrat_cache_stats
1170 )
1171 ParserElement.recursion_memos.clear()
1173 # class attributes to keep caching status
1174 _packratEnabled = False
1175 _left_recursion_enabled = False
1177 @staticmethod
1178 def disable_memoization() -> None:
1179 """
1180 Disables active Packrat or Left Recursion parsing and their memoization
1182 This method also works if neither Packrat nor Left Recursion are enabled.
1183 This makes it safe to call before activating Packrat nor Left Recursion
1184 to clear any previous settings.
1185 """
1186 with ParserElement.packrat_cache_lock:
1187 ParserElement.reset_cache()
1188 ParserElement._left_recursion_enabled = False
1189 ParserElement._packratEnabled = False
1190 ParserElement._parse = ParserElement._parseNoCache
1192 @staticmethod
1193 def enable_left_recursion(
1194 cache_size_limit: typing.Optional[int] = None, *, force=False
1195 ) -> None:
1196 """
1197 Enables "bounded recursion" parsing, which allows for both direct and indirect
1198 left-recursion. During parsing, left-recursive :class:`Forward` elements are
1199 repeatedly matched with a fixed recursion depth that is gradually increased
1200 until finding the longest match.
1202 Example:
1204 .. testcode::
1206 import pyparsing as pp
1207 pp.ParserElement.enable_left_recursion()
1209 E = pp.Forward("E")
1210 num = pp.Word(pp.nums)
1212 # match `num`, or `num '+' num`, or `num '+' num '+' num`, ...
1213 E <<= E + '+' - num | num
1215 print(E.parse_string("1+2+3+4"))
1217 prints:
1219 .. testoutput::
1221 ['1', '+', '2', '+', '3', '+', '4']
1223 Recursion search naturally memoizes matches of ``Forward`` elements and may
1224 thus skip reevaluation of parse actions during backtracking. This may break
1225 programs with parse actions which rely on strict ordering of side-effects.
1227 Parameters:
1229 - ``cache_size_limit`` - (default=``None``) - memoize at most this many
1230 ``Forward`` elements during matching; if ``None`` (the default),
1231 memoize all ``Forward`` elements.
1233 Bounded Recursion parsing works similar but not identical to Packrat parsing,
1234 thus the two cannot be used together. Use ``force=True`` to disable any
1235 previous, conflicting settings.
1236 """
1237 with ParserElement.packrat_cache_lock:
1238 if force:
1239 ParserElement.disable_memoization()
1240 elif ParserElement._packratEnabled:
1241 raise RuntimeError("Packrat and Bounded Recursion are not compatible")
1242 if cache_size_limit is None:
1243 ParserElement.recursion_memos = _UnboundedMemo()
1244 elif cache_size_limit > 0:
1245 ParserElement.recursion_memos = _LRUMemo(capacity=cache_size_limit) # type: ignore[assignment]
1246 else:
1247 raise NotImplementedError(f"Memo size of {cache_size_limit}")
1248 ParserElement._left_recursion_enabled = True
1250 @staticmethod
1251 def enable_packrat(
1252 cache_size_limit: Union[int, None] = 128, *, force: bool = False
1253 ) -> None:
1254 """
1255 Enables "packrat" parsing, which adds memoizing to the parsing logic.
1256 Repeated parse attempts at the same string location (which happens
1257 often in many complex grammars) can immediately return a cached value,
1258 instead of re-executing parsing/validating code. Memoizing is done of
1259 both valid results and parsing exceptions.
1261 Parameters:
1263 - ``cache_size_limit`` - (default= ``128``) - if an integer value is provided
1264 will limit the size of the packrat cache; if None is passed, then
1265 the cache size will be unbounded; if 0 is passed, the cache will
1266 be effectively disabled.
1268 This speedup may break existing programs that use parse actions that
1269 have side-effects. For this reason, packrat parsing is disabled when
1270 you first import pyparsing. To activate the packrat feature, your
1271 program must call the class method :class:`ParserElement.enable_packrat`.
1272 For best results, call ``enable_packrat()`` immediately after
1273 importing pyparsing.
1275 .. Can't really be doctested, alas
1277 Example::
1279 import pyparsing
1280 pyparsing.ParserElement.enable_packrat()
1282 Packrat parsing works similar but not identical to Bounded Recursion parsing,
1283 thus the two cannot be used together. Use ``force=True`` to disable any
1284 previous, conflicting settings.
1285 """
1286 with ParserElement.packrat_cache_lock:
1287 if force:
1288 ParserElement.disable_memoization()
1289 elif ParserElement._left_recursion_enabled:
1290 raise RuntimeError("Packrat and Bounded Recursion are not compatible")
1292 if ParserElement._packratEnabled:
1293 return
1295 ParserElement._packratEnabled = True
1296 if cache_size_limit is None:
1297 ParserElement.packrat_cache = _UnboundedCache()
1298 else:
1299 ParserElement.packrat_cache = _FifoCache(cache_size_limit)
1300 ParserElement._parse = ParserElement._parseCache
1302 def parse_string(
1303 self, instring: str, parse_all: bool = False, **kwargs
1304 ) -> ParseResults:
1305 """
1306 Parse a string with respect to the parser definition. This function is intended as the primary interface to the
1307 client code.
1309 :param instring: The input string to be parsed.
1310 :param parse_all: If set, the entire input string must match the grammar.
1311 :param parseAll: retained for pre-PEP8 compatibility, will be removed in a future release.
1312 :raises ParseException: Raised if ``parse_all`` is set and the input string does not match the whole grammar.
1313 :returns: the parsed data as a :class:`ParseResults` object, which may be accessed as a `list`, a `dict`, or
1314 an object with attributes if the given parser includes results names.
1316 If the input string is required to match the entire grammar, ``parse_all`` flag must be set to ``True``. This
1317 is also equivalent to ending the grammar with :class:`StringEnd`\\ ().
1319 To report proper column numbers, ``parse_string`` operates on a copy of the input string where all tabs are
1320 converted to spaces (8 spaces per tab, as per the default in ``string.expandtabs``). If the input string
1321 contains tabs and the grammar uses parse actions that use the ``loc`` argument to index into the string
1322 being parsed, one can ensure a consistent view of the input string by doing one of the following:
1324 - calling ``parse_with_tabs`` on your grammar before calling ``parse_string`` (see :class:`parse_with_tabs`),
1325 - define your parse action using the full ``(s,loc,toks)`` signature, and reference the input string using the
1326 parse action's ``s`` argument, or
1327 - explicitly expand the tabs in your input string before calling ``parse_string``.
1329 Examples:
1331 By default, partial matches are OK.
1333 .. doctest::
1335 >>> res = Word('a').parse_string('aaaaabaaa')
1336 >>> print(res)
1337 ['aaaaa']
1339 The parsing behavior varies by the inheriting class of this abstract class. Please refer to the children
1340 directly to see more examples.
1342 It raises an exception if parse_all flag is set and instring does not match the whole grammar.
1344 .. doctest::
1346 >>> res = Word('a').parse_string('aaaaabaaa', parse_all=True)
1347 Traceback (most recent call last):
1348 ParseException: Expected end of text, found 'b' ...
1349 """
1350 parseAll: bool = deprecate_argument(kwargs, "parseAll", False)
1352 parse_all = parse_all or parseAll
1354 ParserElement.reset_cache()
1355 if not self.streamlined:
1356 self.streamline()
1357 for e in self.ignoreExprs:
1358 e.streamline()
1359 if not self.keepTabs:
1360 instring = instring.expandtabs()
1361 try:
1362 loc, tokens = self._parse(instring, 0)
1363 if parse_all:
1364 loc = self.preParse(instring, loc)
1365 se = Empty() + StringEnd().set_debug(False)
1366 se._parse(instring, loc)
1367 except _ParseActionIndexError as pa_exc:
1368 raise pa_exc.exc
1369 except ParseBaseException as exc:
1370 if ParserElement.verbose_stacktrace:
1371 raise
1373 # catch and re-raise exception from here, clearing out pyparsing internal stack trace
1374 raise exc.with_traceback(None)
1375 else:
1376 return tokens
1378 def scan_string(
1379 self,
1380 instring: str,
1381 max_matches: int = _MAX_INT,
1382 overlap: bool = False,
1383 always_skip_whitespace=True,
1384 *,
1385 debug: bool = False,
1386 **kwargs,
1387 ) -> Generator[tuple[ParseResults, int, int], None, None]:
1388 """
1389 Scan the input string for expression matches. Each match will return the
1390 matching tokens, start location, and end location. May be called with optional
1391 ``max_matches`` argument, to clip scanning after 'n' matches are found. If
1392 ``overlap`` is specified, then overlapping matches will be reported.
1394 Note that the start and end locations are reported relative to the string
1395 being parsed. See :class:`parse_string` for more information on parsing
1396 strings with embedded tabs.
1398 Example:
1400 .. testcode::
1402 source = "sldjf123lsdjjkf345sldkjf879lkjsfd987"
1403 print(source)
1404 for tokens, start, end in Word(alphas).scan_string(source):
1405 print(' '*start + '^'*(end-start))
1406 print(' '*start + tokens[0])
1408 prints:
1410 .. testoutput::
1412 sldjf123lsdjjkf345sldkjf879lkjsfd987
1413 ^^^^^
1414 sldjf
1415 ^^^^^^^
1416 lsdjjkf
1417 ^^^^^^
1418 sldkjf
1419 ^^^^^^
1420 lkjsfd
1421 """
1422 maxMatches: int = deprecate_argument(kwargs, "maxMatches", _MAX_INT)
1424 max_matches = min(maxMatches, max_matches)
1425 if not self.streamlined:
1426 self.streamline()
1427 for e in self.ignoreExprs:
1428 e.streamline()
1430 if not self.keepTabs:
1431 instring = str(instring).expandtabs()
1432 instrlen = len(instring)
1433 loc = 0
1434 if always_skip_whitespace:
1435 preparser = Empty()
1436 preparser.ignoreExprs = self.ignoreExprs
1437 preparser.whiteChars = self.whiteChars
1438 preparseFn = preparser.preParse
1439 else:
1440 preparseFn = self.preParse
1441 parseFn = self._parse
1442 ParserElement.reset_cache()
1443 matches = 0
1444 try:
1445 while loc <= instrlen and matches < max_matches:
1446 try:
1447 preloc: int = preparseFn(instring, loc)
1448 nextLoc: int
1449 tokens: ParseResults
1450 nextLoc, tokens = parseFn(instring, preloc, callPreParse=False)
1451 except ParseException:
1452 loc = preloc + 1
1453 else:
1454 if nextLoc > loc:
1455 matches += 1
1456 if debug:
1457 print(
1458 {
1459 "tokens": tokens.as_list(),
1460 "start": preloc,
1461 "end": nextLoc,
1462 }
1463 )
1464 yield tokens, preloc, nextLoc
1465 if overlap:
1466 nextloc = preparseFn(instring, loc)
1467 if nextloc > loc:
1468 loc = nextLoc
1469 else:
1470 loc += 1
1471 else:
1472 loc = nextLoc
1473 else:
1474 loc = preloc + 1
1475 except ParseBaseException as exc:
1476 if ParserElement.verbose_stacktrace:
1477 raise
1479 # catch and re-raise exception from here, clears out pyparsing internal stack trace
1480 raise exc.with_traceback(None)
1482 def transform_string(self, instring: str, *, debug: bool = False) -> str:
1483 """
1484 Extension to :class:`scan_string`, to modify matching text with modified tokens that may
1485 be returned from a parse action. To use ``transform_string``, define a grammar and
1486 attach a parse action to it that modifies the returned token list.
1487 Invoking ``transform_string()`` on a target string will then scan for matches,
1488 and replace the matched text patterns according to the logic in the parse
1489 action. ``transform_string()`` returns the resulting transformed string.
1491 Example:
1493 .. testcode::
1495 quote = '''now is the winter of our discontent,
1496 made glorious summer by this sun of york.'''
1498 wd = Word(alphas)
1499 wd.set_parse_action(lambda toks: toks[0].title())
1501 print(wd.transform_string(quote))
1503 prints:
1505 .. testoutput::
1507 Now Is The Winter Of Our Discontent,
1508 Made Glorious Summer By This Sun Of York.
1509 """
1510 out: list[str] = []
1511 lastE = 0
1512 # force preservation of <TAB>s, to minimize unwanted transformation of string, and to
1513 # keep string locs straight between transform_string and scan_string
1514 self.keepTabs = True
1515 try:
1516 for t, s, e in self.scan_string(instring, debug=debug):
1517 if s > lastE:
1518 out.append(instring[lastE:s])
1519 lastE = e
1521 if not t:
1522 continue
1524 if isinstance(t, ParseResults):
1525 out += t.as_list()
1526 elif isinstance(t, Iterable) and not isinstance(t, str_type):
1527 out.extend(t)
1528 else:
1529 out.append(t)
1531 out.append(instring[lastE:])
1532 out = [o for o in out if o]
1533 return "".join([str(s) for s in _flatten(out)])
1534 except ParseBaseException as exc:
1535 if ParserElement.verbose_stacktrace:
1536 raise
1538 # catch and re-raise exception from here, clears out pyparsing internal stack trace
1539 raise exc.with_traceback(None)
1541 def search_string(
1542 self,
1543 instring: str,
1544 max_matches: int = _MAX_INT,
1545 *,
1546 debug: bool = False,
1547 **kwargs,
1548 ) -> ParseResults:
1549 """
1550 Another extension to :class:`scan_string`, simplifying the access to the tokens found
1551 to match the given parse expression. May be called with optional
1552 ``max_matches`` argument, to clip searching after 'n' matches are found.
1554 Example:
1556 .. testcode::
1558 quote = '''More than Iron, more than Lead,
1559 more than Gold I need Electricity'''
1561 # a capitalized word starts with an uppercase letter,
1562 # followed by zero or more lowercase letters
1563 cap_word = Word(alphas.upper(), alphas.lower())
1565 print(cap_word.search_string(quote))
1567 # the sum() builtin can be used to merge results
1568 # into a single ParseResults object
1569 print(sum(cap_word.search_string(quote)))
1571 prints:
1573 .. testoutput::
1575 [['More'], ['Iron'], ['Lead'], ['Gold'], ['I'], ['Electricity']]
1576 ['More', 'Iron', 'Lead', 'Gold', 'I', 'Electricity']
1577 """
1578 maxMatches: int = deprecate_argument(kwargs, "maxMatches", _MAX_INT)
1580 max_matches = min(maxMatches, max_matches)
1581 try:
1582 return ParseResults(
1583 [
1584 t
1585 for t, s, e in self.scan_string(
1586 instring,
1587 max_matches=max_matches,
1588 always_skip_whitespace=False,
1589 debug=debug,
1590 )
1591 ]
1592 )
1593 except ParseBaseException as exc:
1594 if ParserElement.verbose_stacktrace:
1595 raise
1597 # catch and re-raise exception from here, clears out pyparsing internal stack trace
1598 raise exc.with_traceback(None)
1600 def split(
1601 self,
1602 instring: str,
1603 maxsplit: int = _MAX_INT,
1604 include_separators: bool = False,
1605 **kwargs,
1606 ) -> Generator[str, None, None]:
1607 """
1608 Generator method to split a string using the given expression as a separator.
1609 May be called with optional ``maxsplit`` argument, to limit the number of splits;
1610 and the optional ``include_separators`` argument (default= ``False``), if the separating
1611 matching text should be included in the split results.
1613 Example:
1615 .. testcode::
1617 punc = one_of(list(".,;:/-!?"))
1618 print(list(punc.split(
1619 "This, this?, this sentence, is badly punctuated!")))
1621 prints:
1623 .. testoutput::
1625 ['This', ' this', '', ' this sentence', ' is badly punctuated', '']
1626 """
1627 includeSeparators: bool = deprecate_argument(kwargs, "includeSeparators", False)
1629 include_separators = includeSeparators or include_separators
1630 last = 0
1631 for t, s, e in self.scan_string(instring, max_matches=maxsplit):
1632 yield instring[last:s]
1633 if include_separators:
1634 yield t[0]
1635 last = e
1636 yield instring[last:]
1638 def __add__(self, other) -> ParserElement:
1639 """
1640 Implementation of ``+`` operator - returns :class:`And`. Adding strings to a :class:`ParserElement`
1641 converts them to :class:`Literal`\\ s by default.
1643 Example:
1645 .. testcode::
1647 greet = Word(alphas) + "," + Word(alphas) + "!"
1648 hello = "Hello, World!"
1649 print(hello, "->", greet.parse_string(hello))
1651 prints:
1653 .. testoutput::
1655 Hello, World! -> ['Hello', ',', 'World', '!']
1657 ``...`` may be used as a parse expression as a short form of :class:`SkipTo`:
1659 .. testcode::
1661 Literal('start') + ... + Literal('end')
1663 is equivalent to:
1665 .. testcode::
1667 Literal('start') + SkipTo('end')("_skipped*") + Literal('end')
1669 Note that the skipped text is returned with '_skipped' as a results name,
1670 and to support having multiple skips in the same parser, the value returned is
1671 a list of all skipped text.
1672 """
1673 if other is Ellipsis:
1674 return _PendingSkip(self)
1676 if isinstance(other, str_type):
1677 other = self._literalStringClass(other)
1678 if not isinstance(other, ParserElement):
1679 return NotImplemented
1680 return And([self, other])
1682 def __radd__(self, other) -> ParserElement:
1683 """
1684 Implementation of ``+`` operator when left operand is not a :class:`ParserElement`
1685 """
1686 if other is Ellipsis:
1687 return SkipTo(self)("_skipped*") + self
1689 if isinstance(other, str_type):
1690 other = self._literalStringClass(other)
1691 if not isinstance(other, ParserElement):
1692 return NotImplemented
1693 return other + self
1695 def __sub__(self, other) -> ParserElement:
1696 """
1697 Implementation of ``-`` operator, returns :class:`And` with error stop
1698 """
1699 if isinstance(other, str_type):
1700 other = self._literalStringClass(other)
1701 if not isinstance(other, ParserElement):
1702 return NotImplemented
1703 return self + And._ErrorStop() + other
1705 def __rsub__(self, other) -> ParserElement:
1706 """
1707 Implementation of ``-`` operator when left operand is not a :class:`ParserElement`
1708 """
1709 if isinstance(other, str_type):
1710 other = self._literalStringClass(other)
1711 if not isinstance(other, ParserElement):
1712 return NotImplemented
1713 return other - self
1715 def __mul__(self, other) -> ParserElement:
1716 """
1717 Implementation of ``*`` operator, allows use of ``expr * 3`` in place of
1718 ``expr + expr + expr``. Expressions may also be multiplied by a 2-integer
1719 tuple, similar to ``{min, max}`` multipliers in regular expressions. Tuples
1720 may also include ``None`` as in:
1722 - ``expr*(n, None)`` or ``expr*(n, )`` is equivalent
1723 to ``expr*n + ZeroOrMore(expr)``
1724 (read as "at least n instances of ``expr``")
1725 - ``expr*(None, n)`` is equivalent to ``expr*(0, n)``
1726 (read as "0 to n instances of ``expr``")
1727 - ``expr*(None, None)`` is equivalent to ``ZeroOrMore(expr)``
1728 - ``expr*(1, None)`` is equivalent to ``OneOrMore(expr)``
1730 Note that ``expr*(None, n)`` does not raise an exception if
1731 more than n exprs exist in the input stream; that is,
1732 ``expr*(None, n)`` does not enforce a maximum number of expr
1733 occurrences. If this behavior is desired, then write
1734 ``expr*(None, n) + ~expr``
1735 """
1736 if other is Ellipsis:
1737 other = (0, None)
1738 elif isinstance(other, tuple) and other[:1] == (Ellipsis,):
1739 other = (0, *other[1:2], None)[:2]
1741 if not isinstance(other, (int, tuple)):
1742 return NotImplemented
1744 if isinstance(other, int):
1745 minElements, optElements = other, 0
1746 else:
1747 other = tuple(o if o is not Ellipsis else None for o in other)
1748 other = (*other, None, None)[:2]
1749 if other[0] is None:
1750 other = (0, other[1])
1751 if isinstance(other[0], int) and other[1] is None:
1752 if other[0] == 0:
1753 return ZeroOrMore(self)
1754 if other[0] == 1:
1755 return OneOrMore(self)
1756 else:
1757 return self * other[0] + ZeroOrMore(self)
1758 elif isinstance(other[0], int) and isinstance(other[1], int):
1759 minElements, optElements = other
1760 optElements -= minElements
1761 else:
1762 return NotImplemented
1764 if minElements < 0:
1765 raise ValueError("cannot multiply ParserElement by negative value")
1766 if optElements < 0:
1767 raise ValueError(
1768 "second tuple value must be greater or equal to first tuple value"
1769 )
1770 if minElements == optElements == 0:
1771 return And([])
1773 if optElements:
1774 # Build the optional tail as a bounded ``ZeroOrMore`` instead of a
1775 # deeply nested ``Opt(self + Opt(self + ...))`` chain. The nested
1776 # form recursed ``optElements`` levels deep, which raised
1777 # RecursionError for large upper bounds (e.g. ``expr[..., 1000]``)
1778 # -- see issue #332. ``ZeroOrMore(..., max=optElements)`` is a flat
1779 # loop that still *exits early* (it stops at the first non-match,
1780 # just like the recursive form), so it preserves the original
1781 # early-exit behavior while no longer scaling the call stack with
1782 # the upper bound.
1783 optionalTail = ZeroOrMore(self, max=optElements)
1785 if minElements:
1786 if minElements == 1:
1787 ret = self + optionalTail
1788 else:
1789 ret = And([self] * minElements) + optionalTail
1790 else:
1791 ret = optionalTail
1792 else:
1793 if minElements == 1:
1794 ret = self
1795 else:
1796 ret = And([self] * minElements)
1797 return ret
1799 def __rmul__(self, other) -> ParserElement:
1800 return self.__mul__(other)
1802 def __or__(self, other) -> ParserElement:
1803 """
1804 Implementation of ``|`` operator - returns :class:`MatchFirst`
1806 .. versionchanged:: 3.1.0
1807 Support ``expr | ""`` as a synonym for ``Optional(expr)``.
1808 """
1809 if other is Ellipsis:
1810 return _PendingSkip(self, must_skip=True)
1812 if isinstance(other, str_type):
1813 # `expr | ""` is equivalent to `Opt(expr)`
1814 if other == "":
1815 return Opt(self)
1816 other = self._literalStringClass(other)
1817 if not isinstance(other, ParserElement):
1818 return NotImplemented
1819 return MatchFirst([self, other])
1821 def __ror__(self, other) -> ParserElement:
1822 """
1823 Implementation of ``|`` operator when left operand is not a :class:`ParserElement`
1824 """
1825 if isinstance(other, str_type):
1826 other = self._literalStringClass(other)
1827 if not isinstance(other, ParserElement):
1828 return NotImplemented
1829 return other | self
1831 def __xor__(self, other) -> ParserElement:
1832 """
1833 Implementation of ``^`` operator - returns :class:`Or`
1834 """
1835 if isinstance(other, str_type):
1836 other = self._literalStringClass(other)
1837 if not isinstance(other, ParserElement):
1838 return NotImplemented
1839 return Or([self, other])
1841 def __rxor__(self, other) -> ParserElement:
1842 """
1843 Implementation of ``^`` operator when left operand is not a :class:`ParserElement`
1844 """
1845 if isinstance(other, str_type):
1846 other = self._literalStringClass(other)
1847 if not isinstance(other, ParserElement):
1848 return NotImplemented
1849 return other ^ self
1851 def __and__(self, other) -> ParserElement:
1852 """
1853 Implementation of ``&`` operator - returns :class:`Each`
1854 """
1855 if isinstance(other, str_type):
1856 other = self._literalStringClass(other)
1857 if not isinstance(other, ParserElement):
1858 return NotImplemented
1859 return Each([self, other])
1861 def __rand__(self, other) -> ParserElement:
1862 """
1863 Implementation of ``&`` operator when left operand is not a :class:`ParserElement`
1864 """
1865 if isinstance(other, str_type):
1866 other = self._literalStringClass(other)
1867 if not isinstance(other, ParserElement):
1868 return NotImplemented
1869 return other & self
1871 def __invert__(self) -> ParserElement:
1872 """
1873 Implementation of ``~`` operator - returns :class:`NotAny`
1874 """
1875 return NotAny(self)
1877 # disable __iter__ to override legacy use of sequential access to __getitem__ to
1878 # iterate over a sequence
1879 __iter__ = None
1881 def __getitem__(self, key):
1882 """
1883 use ``[]`` indexing notation as a short form for expression repetition:
1885 - ``expr[n]`` is equivalent to ``expr*n``
1886 - ``expr[m, n]`` is equivalent to ``expr*(m, n)``
1887 - ``expr[n, ...]`` or ``expr[n,]`` is equivalent
1888 to ``expr*n + ZeroOrMore(expr)``
1889 (read as "at least n instances of ``expr``")
1890 - ``expr[..., n]`` is equivalent to ``expr*(0, n)``
1891 (read as "0 to n instances of ``expr``")
1892 - ``expr[...]`` and ``expr[0, ...]`` are equivalent to ``ZeroOrMore(expr)``
1893 - ``expr[1, ...]`` is equivalent to ``OneOrMore(expr)``
1895 ``None`` may be used in place of ``...``.
1897 Note that ``expr[..., n]`` and ``expr[m, n]`` do not raise an exception
1898 if more than ``n`` ``expr``\\ s exist in the input stream. If this behavior is
1899 desired, then write ``expr[..., n] + ~expr``.
1901 For repetition with a stop_on expression, use slice notation:
1903 - ``expr[...: end_expr]`` and ``expr[0, ...: end_expr]`` are equivalent to ``ZeroOrMore(expr, stop_on=end_expr)``
1904 - ``expr[1, ...: end_expr]`` is equivalent to ``OneOrMore(expr, stop_on=end_expr)``
1906 .. versionchanged:: 3.1.0
1907 Support for slice notation.
1908 """
1910 stop_on_defined = False
1911 stop_on = NoMatch()
1912 if isinstance(key, slice):
1913 key, stop_on = key.start, key.stop
1914 if key is None:
1915 key = ...
1916 stop_on_defined = True
1917 elif isinstance(key, tuple) and isinstance(key[-1], slice):
1918 key, stop_on = (key[0], key[1].start), key[1].stop
1919 stop_on_defined = True
1921 # convert single arg keys to tuples
1922 if isinstance(key, str_type):
1923 key = (key,)
1924 try:
1925 iter(key)
1926 except TypeError:
1927 key = (key, key)
1929 if len(key) > 2:
1930 raise TypeError(
1931 f"only 1 or 2 index arguments supported ({key[:5]}{f'... [{len(key)}]' if len(key) > 5 else ''})"
1932 )
1934 # clip to 2 elements
1935 ret = self * tuple(key[:2])
1936 ret = typing.cast(_MultipleMatch, ret)
1938 if stop_on_defined:
1939 ret.stopOn(stop_on)
1941 return ret
1943 def __call__(self, name: typing.Optional[str] = None) -> ParserElement:
1944 """
1945 Shortcut for :class:`set_results_name`, with ``list_all_matches=False``.
1947 If ``name`` is given with a trailing ``'*'`` character, then ``list_all_matches`` will be
1948 passed as ``True``.
1950 If ``name`` is omitted, same as calling :class:`copy`.
1952 Example:
1954 .. testcode::
1956 # these are equivalent
1957 userdata = (
1958 Word(alphas).set_results_name("name")
1959 + Word(nums + "-").set_results_name("socsecno")
1960 )
1962 userdata = Word(alphas)("name") + Word(nums + "-")("socsecno")
1963 """
1964 if name is not None:
1965 return self._setResultsName(name)
1967 return self.copy()
1969 def suppress(self) -> ParserElement:
1970 """
1971 Suppresses the output of this :class:`ParserElement`; useful to keep punctuation from
1972 cluttering up returned output.
1973 """
1974 return Suppress(self)
1976 def ignore_whitespace(self, recursive: bool = True) -> ParserElement:
1977 """
1978 Enables the skipping of whitespace before matching the characters in the
1979 :class:`ParserElement`'s defined pattern.
1981 :param recursive: If ``True`` (the default), also enable whitespace skipping in child elements (if any)
1982 """
1983 self.skipWhitespace = True
1984 return self
1986 def leave_whitespace(self, recursive: bool = True) -> ParserElement:
1987 """
1988 Disables the skipping of whitespace before matching the characters in the
1989 :class:`ParserElement`'s defined pattern. This is normally only used internally by
1990 the pyparsing module, but may be needed in some whitespace-sensitive grammars.
1992 :param recursive: If true (the default), also disable whitespace skipping in child elements (if any)
1993 """
1994 self.skipWhitespace = False
1995 return self
1997 def set_whitespace_chars(
1998 self, chars: Union[set[str], str], copy_defaults: bool = False
1999 ) -> ParserElement:
2000 """
2001 Overrides the default whitespace chars
2002 """
2003 self.skipWhitespace = True
2004 self.whiteChars = set(chars)
2005 self.copyDefaultWhiteChars = copy_defaults
2006 return self
2008 def parse_with_tabs(self) -> ParserElement:
2009 """
2010 Overrides default behavior to expand ``<TAB>`` s to spaces before parsing the input string.
2011 Must be called before ``parse_string`` when the input grammar contains elements that
2012 match ``<TAB>`` characters.
2013 """
2014 self.keepTabs = True
2015 return self
2017 def ignore(self, other: ParserElement) -> ParserElement:
2018 """
2019 Define expression to be ignored (e.g., comments) while doing pattern
2020 matching; may be called repeatedly, to define multiple comment or other
2021 ignorable patterns.
2023 Example:
2025 .. doctest::
2027 >>> patt = Word(alphas)[...]
2028 >>> print(patt.parse_string('ablaj /* comment */ lskjd'))
2029 ['ablaj']
2031 >>> patt = Word(alphas)[...].ignore(c_style_comment)
2032 >>> print(patt.parse_string('ablaj /* comment */ lskjd'))
2033 ['ablaj', 'lskjd']
2034 """
2035 if isinstance(other, str_type):
2036 other = Suppress(other)
2038 if isinstance(other, Suppress):
2039 if other not in self.ignoreExprs:
2040 self.ignoreExprs.append(other)
2041 else:
2042 self.ignoreExprs.append(Suppress(other.copy()))
2043 return self
2045 def set_debug_actions(
2046 self,
2047 start_action: DebugStartAction,
2048 success_action: DebugSuccessAction,
2049 exception_action: DebugExceptionAction,
2050 ) -> ParserElement:
2051 """
2052 Customize display of debugging messages while doing pattern matching:
2054 :param start_action: method to be called when an expression is about to be parsed;
2055 should have the signature::
2057 fn(input_string: str,
2058 location: int,
2059 expression: ParserElement,
2060 cache_hit: bool)
2062 :param success_action: method to be called when an expression has successfully parsed;
2063 should have the signature::
2065 fn(input_string: str,
2066 start_location: int,
2067 end_location: int,
2068 expression: ParserELement,
2069 parsed_tokens: ParseResults,
2070 cache_hit: bool)
2072 :param exception_action: method to be called when expression fails to parse;
2073 should have the signature::
2075 fn(input_string: str,
2076 location: int,
2077 expression: ParserElement,
2078 exception: Exception,
2079 cache_hit: bool)
2080 """
2081 self.debugActions = self.DebugActions(
2082 start_action or _default_start_debug_action, # type: ignore[truthy-function]
2083 success_action or _default_success_debug_action, # type: ignore[truthy-function]
2084 exception_action or _default_exception_debug_action, # type: ignore[truthy-function]
2085 )
2086 self.debug = any(self.debugActions)
2087 return self
2089 def set_debug(self, flag: bool = True, recurse: bool = False) -> ParserElement:
2090 """
2091 Enable display of debugging messages while doing pattern matching.
2092 Set ``flag`` to ``True`` to enable, ``False`` to disable.
2093 Set ``recurse`` to ``True`` to set the debug flag on this expression and all sub-expressions.
2095 Example:
2097 .. testcode::
2099 wd = Word(alphas).set_name("alphaword")
2100 integer = Word(nums).set_name("numword")
2101 term = wd | integer
2103 # turn on debugging for wd
2104 wd.set_debug()
2106 term[1, ...].parse_string("abc 123 xyz 890")
2108 prints:
2110 .. testoutput::
2111 :options: +NORMALIZE_WHITESPACE
2113 Match alphaword at loc 0(1,1)
2114 abc 123 xyz 890
2115 ^
2116 Matched alphaword -> ['abc']
2117 Match alphaword at loc 4(1,5)
2118 abc 123 xyz 890
2119 ^
2120 Match alphaword failed, ParseException raised: Expected alphaword, ...
2121 Match alphaword at loc 8(1,9)
2122 abc 123 xyz 890
2123 ^
2124 Matched alphaword -> ['xyz']
2125 Match alphaword at loc 12(1,13)
2126 abc 123 xyz 890
2127 ^
2128 Match alphaword failed, ParseException raised: Expected alphaword, ...
2129 abc 123 xyz 890
2130 ^
2131 Match alphaword failed, ParseException raised: Expected alphaword, found end of text ...
2133 The output shown is that produced by the default debug actions - custom debug actions can be
2134 specified using :meth:`set_debug_actions`. Prior to attempting
2135 to match the ``wd`` expression, the debugging message ``"Match <exprname> at loc <n>(<line>,<col>)"``
2136 is shown. Then if the parse succeeds, a ``"Matched"`` message is shown, or an ``"Exception raised"``
2137 message is shown. Also note the use of :meth:`set_name` to assign a human-readable name to the expression,
2138 which makes debugging and exception messages easier to understand - for instance, the default
2139 name created for the :class:`Word` expression without calling :meth:`set_name` is ``"W:(A-Za-z)"``.
2141 .. versionchanged:: 3.1.0
2142 ``recurse`` argument added.
2143 """
2144 if recurse:
2145 for expr in self.visit_all():
2146 expr.set_debug(flag, recurse=False)
2147 return self
2149 if flag:
2150 self.set_debug_actions(
2151 _default_start_debug_action,
2152 _default_success_debug_action,
2153 _default_exception_debug_action,
2154 )
2155 else:
2156 self.debug = False
2157 return self
2159 @property
2160 def default_name(self) -> str:
2161 if self._defaultName is None:
2162 self._defaultName = self._generateDefaultName()
2163 return self._defaultName
2165 @abstractmethod
2166 def _generateDefaultName(self) -> str:
2167 """
2168 Child classes must define this method, which defines how the ``default_name`` is set.
2169 """
2171 def set_name(self, name: typing.Optional[str]) -> ParserElement:
2172 """
2173 Define name for this expression, makes debugging and exception messages clearer. If
2174 `__diag__.enable_debug_on_named_expressions` is set to True, setting a name will also
2175 enable debug for this expression.
2177 If `name` is None, clears any custom name for this expression, and clears the
2178 debug flag is it was enabled via `__diag__.enable_debug_on_named_expressions`.
2180 Example:
2182 .. doctest::
2184 >>> integer = Word(nums)
2185 >>> integer.parse_string("ABC")
2186 Traceback (most recent call last):
2187 ParseException: Expected W:(0-9) (at char 0), (line:1, col:1)
2189 >>> integer.set_name("integer")
2190 integer
2191 >>> integer.parse_string("ABC")
2192 Traceback (most recent call last):
2193 ParseException: Expected integer (at char 0), (line:1, col:1)
2195 .. versionchanged:: 3.1.0
2196 Accept ``None`` as the ``name`` argument.
2197 """
2198 self.customName = name # type: ignore[assignment]
2199 self.errmsg = f"Expected {str(self)}"
2201 if __diag__.enable_debug_on_named_expressions:
2202 self.set_debug(name is not None)
2204 return self
2206 @property
2207 def name(self) -> str:
2208 """
2209 Returns a user-defined name if available, but otherwise defaults back to the auto-generated name
2210 """
2211 return self.customName if self.customName is not None else self.default_name
2213 @name.setter
2214 def name(self, new_name) -> None:
2215 self.set_name(new_name)
2217 def __str__(self) -> str:
2218 return self.name
2220 def __repr__(self) -> str:
2221 return str(self)
2223 def streamline(self) -> ParserElement:
2224 self.streamlined = True
2225 self._defaultName = None
2226 return self
2228 def recurse(self) -> list[ParserElement]:
2229 return []
2231 def _checkRecursion(self, parseElementList):
2232 subRecCheckList = parseElementList[:] + [self]
2233 for e in self.recurse():
2234 e._checkRecursion(subRecCheckList)
2236 def validate(self, validateTrace=None) -> None:
2237 """
2238 .. deprecated:: 3.0.0
2239 Do not use to check for left recursion.
2241 Check defined expressions for valid structure, check for infinite recursive definitions.
2243 """
2244 warnings.warn(
2245 "ParserElement.validate() is deprecated, and should not be used to check for left recursion",
2246 PyparsingDeprecationWarning,
2247 stacklevel=2,
2248 )
2249 self._checkRecursion([])
2251 def parse_file(
2252 self,
2253 file_or_filename: Union[str, Path, TextIO],
2254 encoding: str = "utf-8",
2255 parse_all: bool = False,
2256 **kwargs,
2257 ) -> ParseResults:
2258 """
2259 Execute the parse expression on the given file or filename.
2260 If a filename is specified (instead of a file object),
2261 the entire file is opened, read, and closed before parsing.
2262 """
2263 parseAll: bool = deprecate_argument(kwargs, "parseAll", False)
2265 parse_all = parse_all or parseAll
2266 try:
2267 file_or_filename = typing.cast(TextIO, file_or_filename)
2268 file_contents = file_or_filename.read()
2269 except AttributeError:
2270 file_or_filename = typing.cast(str, file_or_filename)
2271 with open(file_or_filename, "r", encoding=encoding) as f:
2272 file_contents = f.read()
2273 try:
2274 return self.parse_string(file_contents, parse_all)
2275 except ParseBaseException as exc:
2276 if ParserElement.verbose_stacktrace:
2277 raise
2279 # catch and re-raise exception from here, clears out pyparsing internal stack trace
2280 raise exc.with_traceback(None)
2282 def __eq__(self, other):
2283 if self is other:
2284 return True
2285 elif isinstance(other, str_type):
2286 return self.matches(other, parse_all=True)
2287 elif isinstance(other, ParserElement):
2288 return vars(self) == vars(other)
2289 return False
2291 def __hash__(self):
2292 return id(self)
2294 def matches(self, test_string: str, parse_all: bool = True, **kwargs) -> bool:
2295 """
2296 Method for quick testing of a parser against a test string. Good for simple
2297 inline microtests of sub expressions while building up larger parser.
2299 :param test_string: to test against this expression for a match
2300 :param parse_all: flag to pass to :meth:`parse_string` when running tests
2302 Example:
2304 .. doctest::
2306 >>> expr = Word(nums)
2307 >>> expr.matches("100")
2308 True
2309 """
2310 parseAll: bool = deprecate_argument(kwargs, "parseAll", True)
2312 parse_all = parse_all and parseAll
2313 try:
2314 self.parse_string(str(test_string), parse_all=parse_all)
2315 return True
2316 except ParseBaseException:
2317 return False
2319 def run_tests(
2320 self,
2321 tests: Union[str, list[str]],
2322 parse_all: bool = True,
2323 comment: typing.Optional[Union[ParserElement, str]] = "#",
2324 full_dump: bool = True,
2325 print_results: bool = True,
2326 failure_tests: bool = False,
2327 post_parse: typing.Optional[
2328 Callable[[str, ParseResults], typing.Optional[str]]
2329 ] = None,
2330 file: typing.Optional[TextIO] = None,
2331 with_line_numbers: bool = False,
2332 *,
2333 parseAll: bool = True,
2334 fullDump: bool = True,
2335 printResults: bool = True,
2336 failureTests: bool = False,
2337 postParse: typing.Optional[
2338 Callable[[str, ParseResults], typing.Optional[str]]
2339 ] = None,
2340 ) -> tuple[bool, list[tuple[str, Union[ParseResults, Exception]]]]:
2341 """
2342 Execute the parse expression on a series of test strings, showing each
2343 test, the parsed results or where the parse failed. Quick and easy way to
2344 run a parse expression against a list of sample strings.
2346 Parameters:
2348 - ``tests`` - a list of separate test strings, or a multiline string of test strings
2349 - ``parse_all`` - (default= ``True``) - flag to pass to :class:`parse_string` when running tests
2350 - ``comment`` - (default= ``'#'``) - expression for indicating embedded comments in the test
2351 string; pass None to disable comment filtering
2352 - ``full_dump`` - (default= ``True``) - dump results as list followed by results names in nested outline;
2353 if False, only dump nested list
2354 - ``print_results`` - (default= ``True``) prints test output to stdout
2355 - ``failure_tests`` - (default= ``False``) indicates if these tests are expected to fail parsing
2356 - ``post_parse`` - (default= ``None``) optional callback for successful parse results; called as
2357 `fn(test_string, parse_results)` and returns a string to be added to the test output
2358 - ``file`` - (default= ``None``) optional file-like object to which test output will be written;
2359 if None, will default to ``sys.stdout``
2360 - ``with_line_numbers`` - default= ``False``) show test strings with line and column numbers
2362 Returns: a (success, results) tuple, where success indicates that all tests succeeded
2363 (or failed if ``failure_tests`` is True), and the results contain a list of lines of each
2364 test's output
2366 Passing example:
2368 .. testcode::
2370 number_expr = pyparsing_common.number.copy()
2372 result = number_expr.run_tests('''
2373 # unsigned integer
2374 100
2375 # negative integer
2376 -100
2377 # float with scientific notation
2378 6.02e23
2379 # integer with scientific notation
2380 1e-12
2381 # negative decimal number without leading digit
2382 -.100
2383 ''')
2384 print("Success" if result[0] else "Failed!")
2386 prints:
2388 .. testoutput::
2389 :options: +NORMALIZE_WHITESPACE
2392 # unsigned integer
2393 100
2394 [100]
2396 # negative integer
2397 -100
2398 [-100]
2400 # float with scientific notation
2401 6.02e23
2402 [6.02e+23]
2404 # integer with scientific notation
2405 1e-12
2406 [1e-12]
2408 # negative decimal number without leading digit
2409 -.100
2410 [-0.1]
2411 Success
2413 Failure-test example:
2415 .. testcode::
2417 result = number_expr.run_tests('''
2418 # stray character
2419 100Z
2420 # too many '.'
2421 3.14.159
2422 ''', failure_tests=True)
2423 print("Success" if result[0] else "Failed!")
2425 prints:
2427 .. testoutput::
2428 :options: +NORMALIZE_WHITESPACE
2431 # stray character
2432 100Z
2433 100Z
2434 ^
2435 ParseException: Expected end of text, found 'Z' ...
2437 # too many '.'
2438 3.14.159
2439 3.14.159
2440 ^
2441 ParseException: Expected end of text, found '.' ...
2442 FAIL: Expected end of text, found '.' ...
2443 Success
2445 Each test string must be on a single line. If you want to test a string that spans multiple
2446 lines, create a test like this:
2448 .. testcode::
2450 expr = Word(alphanums)[1,...]
2451 expr.run_tests(r"this is a test\\n of strings that spans \\n 3 lines")
2453 .. testoutput::
2454 :options: +NORMALIZE_WHITESPACE
2455 :hide:
2458 this is a test\\n of strings that spans \\n 3 lines
2459 ['this', 'is', 'a', 'test', 'of', 'strings', 'that', 'spans', '3', 'lines']
2461 (Note that this is a raw string literal, you must include the leading ``'r'``.)
2462 """
2463 from .testing import pyparsing_test
2465 parseAll = parseAll and parse_all
2466 fullDump = fullDump and full_dump
2467 printResults = printResults and print_results
2468 failureTests = failureTests or failure_tests
2469 postParse = postParse or post_parse
2470 if isinstance(tests, str_type):
2471 tests = typing.cast(str, tests)
2472 line_strip = type(tests).strip
2473 tests = [line_strip(test_line) for test_line in tests.rstrip().splitlines()]
2474 comment_specified = comment is not None
2475 if comment_specified:
2476 if isinstance(comment, str_type):
2477 comment = typing.cast(str, comment)
2478 comment = Literal(comment)
2479 comment = typing.cast(ParserElement, comment)
2480 if file is None:
2481 file = sys.stdout
2482 print_ = file.write
2484 result: Union[ParseResults, Exception]
2485 allResults: list[tuple[str, Union[ParseResults, Exception]]] = []
2486 comments: list[str] = []
2487 success = True
2488 NL = Literal(r"\n").add_parse_action(replace_with("\n")).ignore(quoted_string)
2489 BOM = "\ufeff"
2490 nlstr = "\n"
2491 for t in tests:
2492 if comment_specified and comment.matches(t, False) or comments and not t:
2493 comments.append(
2494 pyparsing_test.with_line_numbers(t) if with_line_numbers else t
2495 )
2496 continue
2497 if not t:
2498 continue
2499 out = [
2500 f"{nlstr}{nlstr.join(comments) if comments else ''}",
2501 pyparsing_test.with_line_numbers(t) if with_line_numbers else t,
2502 ]
2503 comments.clear()
2504 try:
2505 # convert newline marks to actual newlines, and strip leading BOM if present
2506 t = NL.transform_string(t.lstrip(BOM))
2507 result = self.parse_string(t, parse_all=parse_all)
2508 except ParseBaseException as pe:
2509 fatal = "(FATAL) " if isinstance(pe, ParseFatalException) else ""
2510 out.append(pe.explain())
2511 out.append(f"FAIL: {fatal}{pe}")
2512 if ParserElement.verbose_stacktrace:
2513 out.extend(traceback.format_tb(pe.__traceback__))
2514 success = success and failureTests
2515 result = pe
2516 except Exception as exc:
2517 tag = "FAIL-EXCEPTION"
2519 # see if this exception was raised in a parse action
2520 tb = exc.__traceback__
2521 it = iter(traceback.walk_tb(tb))
2522 for f, line in it:
2523 if (f.f_code.co_filename, line) == pa_call_line_synth:
2524 next_f = next(it)[0]
2525 tag += f" (raised in parse action {next_f.f_code.co_name!r})"
2526 break
2528 out.append(f"{tag}: {type(exc).__name__}: {exc}")
2529 if ParserElement.verbose_stacktrace:
2530 out.extend(traceback.format_tb(exc.__traceback__))
2531 success = success and failureTests
2532 result = exc
2533 else:
2534 success = success and not failureTests
2535 if postParse is not None:
2536 try:
2537 pp_value = postParse(t, result)
2538 if pp_value is not None:
2539 if isinstance(pp_value, ParseResults):
2540 out.append(pp_value.dump())
2541 else:
2542 out.append(str(pp_value))
2543 else:
2544 out.append(result.dump())
2545 except Exception as e:
2546 out.append(result.dump(full=fullDump))
2547 out.append(
2548 f"{postParse.__name__} failed: {type(e).__name__}: {e}"
2549 )
2550 else:
2551 out.append(result.dump(full=fullDump))
2552 out.append("")
2554 if printResults:
2555 print_("\n".join(out))
2557 allResults.append((t, result))
2559 return success, allResults
2561 def create_diagram(
2562 self,
2563 output_html: Union[TextIO, Path, str],
2564 vertical: int = 3,
2565 show_results_names: bool = False,
2566 show_groups: bool = False,
2567 embed: bool = False,
2568 show_hidden: bool = False,
2569 **kwargs,
2570 ) -> None:
2571 """
2572 Create a railroad diagram for the parser.
2574 Parameters:
2576 - ``output_html`` (str or file-like object) - output target for generated
2577 diagram HTML
2578 - ``vertical`` (int) - threshold for formatting multiple alternatives vertically
2579 instead of horizontally (default=3)
2580 - ``show_results_names`` - bool flag whether diagram should show annotations for
2581 defined results names
2582 - ``show_groups`` - bool flag whether groups should be highlighted with an unlabeled surrounding box
2583 - ``show_hidden`` - bool flag to show diagram elements for internal elements that are usually hidden
2584 - ``embed`` - bool flag whether generated HTML should omit <HEAD>, <BODY>, and <DOCTYPE> tags to embed
2585 the resulting HTML in an enclosing HTML source
2586 - ``head`` - str containing additional HTML to insert into the <HEAD> section of the generated code;
2587 can be used to insert custom CSS styling
2588 - ``body`` - str containing additional HTML to insert at the beginning of the <BODY> section of the
2589 generated code
2591 Additional diagram-formatting keyword arguments can also be included;
2592 see railroad.Diagram class.
2594 .. versionchanged:: 3.1.0
2595 ``embed`` argument added.
2596 """
2598 try:
2599 from .diagram import to_railroad, railroad_to_html
2600 except ImportError as ie:
2601 raise Exception(
2602 "must ``pip install pyparsing[diagrams]`` to generate parser railroad diagrams"
2603 ) from ie
2605 self.streamline()
2607 railroad = to_railroad(
2608 self,
2609 vertical=vertical,
2610 show_results_names=show_results_names,
2611 show_groups=show_groups,
2612 show_hidden=show_hidden,
2613 diagram_kwargs=kwargs,
2614 )
2615 if not isinstance(output_html, (str, Path)):
2616 # we were passed a file-like object, just write to it
2617 output_html.write(railroad_to_html(railroad, embed=embed, **kwargs))
2618 return
2620 with open(output_html, "w", encoding="utf-8") as diag_file:
2621 diag_file.write(railroad_to_html(railroad, embed=embed, **kwargs))
2623 # Compatibility synonyms
2624 # fmt: off
2625 inlineLiteralsUsing = staticmethod(replaced_by_pep8("inlineLiteralsUsing", inline_literals_using))
2626 setDefaultWhitespaceChars = staticmethod(replaced_by_pep8(
2627 "setDefaultWhitespaceChars", set_default_whitespace_chars
2628 ))
2629 disableMemoization = staticmethod(replaced_by_pep8("disableMemoization", disable_memoization))
2630 enableLeftRecursion = staticmethod(replaced_by_pep8("enableLeftRecursion", enable_left_recursion))
2631 enablePackrat = staticmethod(replaced_by_pep8("enablePackrat", enable_packrat))
2632 resetCache = staticmethod(replaced_by_pep8("resetCache", reset_cache))
2634 setResultsName = replaced_by_pep8("setResultsName", set_results_name)
2635 setBreak = replaced_by_pep8("setBreak", set_break)
2636 setParseAction = replaced_by_pep8("setParseAction", set_parse_action)
2637 addParseAction = replaced_by_pep8("addParseAction", add_parse_action)
2638 addCondition = replaced_by_pep8("addCondition", add_condition)
2639 setFailAction = replaced_by_pep8("setFailAction", set_fail_action)
2640 tryParse = replaced_by_pep8("tryParse", try_parse)
2641 parseString = replaced_by_pep8("parseString", parse_string)
2642 scanString = replaced_by_pep8("scanString", scan_string)
2643 transformString = replaced_by_pep8("transformString", transform_string)
2644 searchString = replaced_by_pep8("searchString", search_string)
2645 ignoreWhitespace = replaced_by_pep8("ignoreWhitespace", ignore_whitespace)
2646 leaveWhitespace = replaced_by_pep8("leaveWhitespace", leave_whitespace)
2647 setWhitespaceChars = replaced_by_pep8("setWhitespaceChars", set_whitespace_chars)
2648 parseWithTabs = replaced_by_pep8("parseWithTabs", parse_with_tabs)
2649 setDebugActions = replaced_by_pep8("setDebugActions", set_debug_actions)
2650 setDebug = replaced_by_pep8("setDebug", set_debug)
2651 setName = replaced_by_pep8("setName", set_name)
2652 parseFile = replaced_by_pep8("parseFile", parse_file)
2653 runTests = replaced_by_pep8("runTests", run_tests)
2654 canParseNext = replaced_by_pep8("canParseNext", can_parse_next)
2655 defaultName = default_name
2656 # fmt: on
2659class _PendingSkip(ParserElement):
2660 # internal placeholder class to hold a place were '...' is added to a parser element,
2661 # once another ParserElement is added, this placeholder will be replaced with a SkipTo
2662 def __init__(self, expr: ParserElement, must_skip: bool = False) -> None:
2663 super().__init__()
2664 self.anchor = expr
2665 self.must_skip = must_skip
2667 def _generateDefaultName(self) -> str:
2668 return str(self.anchor + Empty()).replace("Empty", "...")
2670 def __add__(self, other) -> ParserElement:
2671 skipper = SkipTo(other).set_name("...")("_skipped*")
2672 if self.must_skip:
2674 def must_skip(t):
2675 if not t._skipped or t._skipped.as_list() == [""]:
2676 del t[0]
2677 t.pop("_skipped", None)
2679 def show_skip(t):
2680 if t._skipped.as_list()[-1:] == [""]:
2681 t.pop("_skipped")
2682 t["_skipped"] = f"missing <{self.anchor!r}>"
2684 return (
2685 self.anchor + skipper().add_parse_action(must_skip)
2686 | skipper().add_parse_action(show_skip)
2687 ) + other
2689 return self.anchor + skipper + other
2691 def __repr__(self):
2692 return self.defaultName
2694 def parseImpl(self, *args) -> ParseImplReturnType:
2695 raise Exception(
2696 "use of `...` expression without following SkipTo target expression"
2697 )
2700class Token(ParserElement):
2701 """Abstract :class:`ParserElement` subclass, for defining atomic
2702 matching patterns.
2703 """
2705 def __init__(self) -> None:
2706 super().__init__(savelist=False)
2708 def _generateDefaultName(self) -> str:
2709 return type(self).__name__
2712class NoMatch(Token):
2713 """
2714 A token that will never match.
2715 """
2717 def __init__(self) -> None:
2718 super().__init__()
2719 self._may_return_empty = True
2720 self.mayIndexError = False
2721 self.errmsg = "Unmatchable token"
2723 def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType:
2724 raise ParseException(instring, loc, self.errmsg, self)
2727class Literal(Token):
2728 """
2729 Token to exactly match a specified string.
2731 Example:
2733 .. doctest::
2735 >>> Literal('abc').parse_string('abc')
2736 ParseResults(['abc'], {})
2737 >>> Literal('abc').parse_string('abcdef')
2738 ParseResults(['abc'], {})
2739 >>> Literal('abc').parse_string('ab')
2740 Traceback (most recent call last):
2741 ParseException: Expected 'abc', found 'ab' (at char 0), (line: 1, col: 1)
2743 For case-insensitive matching, use :class:`CaselessLiteral`.
2745 For keyword matching (force word break before and after the matched string),
2746 use :class:`Keyword` or :class:`CaselessKeyword`.
2747 """
2749 def __new__(cls, match_string: str = "", **kwargs):
2750 # Performance tuning: select a subclass with optimized parseImpl
2751 if cls is Literal:
2752 matchString: str = deprecate_argument(kwargs, "matchString", "")
2754 match_string = matchString or match_string
2755 if not match_string:
2756 return super().__new__(Empty)
2757 if len(match_string) == 1:
2758 return super().__new__(_SingleCharLiteral)
2760 # Default behavior
2761 return super().__new__(cls)
2763 # Needed to make copy.copy() work correctly if we customize __new__
2764 def __getnewargs__(self):
2765 return (self.match,)
2767 def __init__(self, match_string: str = "", **kwargs) -> None:
2768 matchString: str = deprecate_argument(kwargs, "matchString", "")
2770 super().__init__()
2771 match_string = matchString or match_string
2772 self.match = match_string
2773 self.matchLen = len(match_string)
2774 self.firstMatchChar = match_string[:1]
2775 self.errmsg = f"Expected {self.name}"
2776 self._may_return_empty = False
2777 self.mayIndexError = False
2779 def _generateDefaultName(self) -> str:
2780 return repr(self.match)
2782 def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType:
2783 if instring[loc] == self.firstMatchChar and instring.startswith(
2784 self.match, loc
2785 ):
2786 return loc + self.matchLen, self.match
2787 raise ParseException(instring, loc, self.errmsg, self)
2790class Empty(Literal):
2791 """
2792 An empty token, will always match.
2793 """
2795 def __init__(self, match_string="", *, matchString="") -> None:
2796 super().__init__("")
2797 self._may_return_empty = True
2798 self.mayIndexError = False
2800 def _generateDefaultName(self) -> str:
2801 return "Empty"
2803 def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType:
2804 return loc, []
2807class _SingleCharLiteral(Literal):
2808 def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType:
2809 if instring[loc] == self.firstMatchChar:
2810 return loc + 1, self.match
2811 raise ParseException(instring, loc, self.errmsg, self)
2814ParserElement._literalStringClass = Literal
2817class Keyword(Token):
2818 """
2819 Token to exactly match a specified string as a keyword, that is,
2820 it must be immediately preceded and followed by whitespace or
2821 non-keyword characters. Compare with :class:`Literal`:
2823 - ``Literal("if")`` will match the leading ``'if'`` in
2824 ``'ifAndOnlyIf'``.
2825 - ``Keyword("if")`` will not; it will only match the leading
2826 ``'if'`` in ``'if x=1'``, or ``'if(y==2)'``
2828 Accepts two optional constructor arguments in addition to the
2829 keyword string:
2831 - ``ident_chars`` is a string of characters that would be valid
2832 identifier characters, defaulting to all alphanumerics + "_" and
2833 "$"
2834 - ``caseless`` allows case-insensitive matching, default is ``False``.
2836 Example:
2838 .. doctest::
2839 :options: +NORMALIZE_WHITESPACE
2841 >>> Keyword("start").parse_string("start")
2842 ParseResults(['start'], {})
2843 >>> Keyword("start").parse_string("starting")
2844 Traceback (most recent call last):
2845 ParseException: Expected Keyword 'start', keyword was immediately
2846 followed by keyword character, found 'ing' (at char 5), (line:1, col:6)
2848 .. doctest::
2849 :options: +NORMALIZE_WHITESPACE
2851 >>> Keyword("start").parse_string("starting").debug()
2852 Traceback (most recent call last):
2853 ParseException: Expected Keyword "start", keyword was immediately
2854 followed by keyword character, found 'ing' ...
2856 For case-insensitive matching, use :class:`CaselessKeyword`.
2857 """
2859 DEFAULT_KEYWORD_CHARS = f"{alphanums}_$"
2861 def __init__(
2862 self,
2863 match_string: str = "",
2864 ident_chars: typing.Optional[str] = None,
2865 caseless: bool = False,
2866 **kwargs,
2867 ) -> None:
2868 matchString = deprecate_argument(kwargs, "matchString", "")
2869 identChars = deprecate_argument(kwargs, "identChars", None)
2871 super().__init__()
2872 identChars = identChars or ident_chars
2873 if identChars is None:
2874 identChars = Keyword.DEFAULT_KEYWORD_CHARS
2875 match_string = matchString or match_string
2876 self.match = match_string
2877 self.matchLen = len(match_string)
2878 self.firstMatchChar = match_string[:1]
2879 if not self.firstMatchChar:
2880 raise ValueError("null string passed to Keyword; use Empty() instead")
2881 self.errmsg = f"Expected {type(self).__name__} {self.name}"
2882 self._may_return_empty = False
2883 self.mayIndexError = False
2884 self.caseless = caseless
2885 if caseless:
2886 self.caselessmatch = match_string.upper()
2887 identChars = identChars.upper()
2888 self.ident_chars = set(identChars)
2890 @property
2891 def identChars(self) -> set[str]:
2892 """
2893 .. deprecated:: 3.3.0
2894 use ident_chars instead.
2896 Property returning the characters being used as keyword characters for this expression.
2897 """
2898 return self.ident_chars
2900 def _generateDefaultName(self) -> str:
2901 return repr(self.match)
2903 def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType:
2904 errmsg = self.errmsg or ""
2905 errloc = loc
2906 if self.caseless:
2907 if instring[loc : loc + self.matchLen].upper() == self.caselessmatch:
2908 if loc == 0 or instring[loc - 1].upper() not in self.identChars:
2909 if (
2910 loc >= len(instring) - self.matchLen
2911 or instring[loc + self.matchLen].upper() not in self.identChars
2912 ):
2913 return loc + self.matchLen, self.match
2915 # followed by keyword char
2916 errmsg += ", was immediately followed by keyword character"
2917 errloc = loc + self.matchLen
2918 else:
2919 # preceded by keyword char
2920 errmsg += ", keyword was immediately preceded by keyword character"
2921 errloc = loc - 1
2922 # else no match just raise plain exception
2924 elif (
2925 instring[loc] == self.firstMatchChar
2926 and self.matchLen == 1
2927 or instring.startswith(self.match, loc)
2928 ):
2929 if loc == 0 or instring[loc - 1] not in self.identChars:
2930 if (
2931 loc >= len(instring) - self.matchLen
2932 or instring[loc + self.matchLen] not in self.identChars
2933 ):
2934 return loc + self.matchLen, self.match
2936 # followed by keyword char
2937 errmsg += ", keyword was immediately followed by keyword character"
2938 errloc = loc + self.matchLen
2939 else:
2940 # preceded by keyword char
2941 errmsg += ", keyword was immediately preceded by keyword character"
2942 errloc = loc - 1
2943 # else no match just raise plain exception
2945 raise ParseException(instring, errloc, errmsg, self)
2947 @staticmethod
2948 def set_default_keyword_chars(chars) -> None:
2949 """
2950 Overrides the default characters used by :class:`Keyword` expressions.
2951 """
2952 Keyword.DEFAULT_KEYWORD_CHARS = chars
2954 # Compatibility synonyms
2955 setDefaultKeywordChars = staticmethod(
2956 replaced_by_pep8("setDefaultKeywordChars", set_default_keyword_chars)
2957 )
2960class CaselessLiteral(Literal):
2961 """
2962 Token to match a specified string, ignoring case of letters.
2963 Note: the matched results will always be in the case of the given
2964 match string, NOT the case of the input text.
2966 Example:
2968 .. doctest::
2970 >>> CaselessLiteral("CMD")[1, ...].parse_string("cmd CMD Cmd10")
2971 ParseResults(['CMD', 'CMD', 'CMD'], {})
2973 (Contrast with example for :class:`CaselessKeyword`.)
2974 """
2976 def __init__(self, match_string: str = "", **kwargs) -> None:
2977 matchString: str = deprecate_argument(kwargs, "matchString", "")
2979 match_string = matchString or match_string
2980 super().__init__(match_string.upper())
2981 # Preserve the defining literal.
2982 self.returnString = match_string
2983 self.errmsg = f"Expected {self.name}"
2985 def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType:
2986 if instring[loc : loc + self.matchLen].upper() == self.match:
2987 return loc + self.matchLen, self.returnString
2988 raise ParseException(instring, loc, self.errmsg, self)
2991class CaselessKeyword(Keyword):
2992 """
2993 Caseless version of :class:`Keyword`.
2995 Example:
2997 .. doctest::
2999 >>> CaselessKeyword("CMD")[1, ...].parse_string("cmd CMD Cmd10")
3000 ParseResults(['CMD', 'CMD'], {})
3002 (Contrast with example for :class:`CaselessLiteral`.)
3003 """
3005 def __init__(
3006 self, match_string: str = "", ident_chars: typing.Optional[str] = None, **kwargs
3007 ) -> None:
3008 matchString: str = deprecate_argument(kwargs, "matchString", "")
3009 identChars: typing.Optional[str] = deprecate_argument(
3010 kwargs, "identChars", None
3011 )
3013 identChars = identChars or ident_chars
3014 match_string = matchString or match_string
3015 super().__init__(match_string, identChars, caseless=True)
3018class CloseMatch(Token):
3019 """A variation on :class:`Literal` which matches "close" matches,
3020 that is, strings with at most 'n' mismatching characters.
3021 :class:`CloseMatch` takes parameters:
3023 - ``match_string`` - string to be matched
3024 - ``caseless`` - a boolean indicating whether to ignore casing when comparing characters
3025 - ``max_mismatches`` - (``default=1``) maximum number of
3026 mismatches allowed to count as a match
3028 The results from a successful parse will contain the matched text
3029 from the input string and the following named results:
3031 - ``mismatches`` - a list of the positions within the
3032 match_string where mismatches were found
3033 - ``original`` - the original match_string used to compare
3034 against the input string
3036 If ``mismatches`` is an empty list, then the match was an exact
3037 match.
3039 Example:
3041 .. doctest::
3042 :options: +NORMALIZE_WHITESPACE
3044 >>> patt = CloseMatch("ATCATCGAATGGA")
3045 >>> patt.parse_string("ATCATCGAAXGGA")
3046 ParseResults(['ATCATCGAAXGGA'],
3047 {'original': 'ATCATCGAATGGA', 'mismatches': [9]})
3049 >>> patt.parse_string("ATCAXCGAAXGGA")
3050 Traceback (most recent call last):
3051 ParseException: Expected 'ATCATCGAATGGA' (with up to 1 mismatches),
3052 found 'ATCAXCGAAXGGA' (at char 0), (line:1, col:1)
3054 # exact match
3055 >>> patt.parse_string("ATCATCGAATGGA")
3056 ParseResults(['ATCATCGAATGGA'],
3057 {'original': 'ATCATCGAATGGA', 'mismatches': []})
3059 # close match allowing up to 2 mismatches
3060 >>> patt = CloseMatch("ATCATCGAATGGA", max_mismatches=2)
3061 >>> patt.parse_string("ATCAXCGAAXGGA")
3062 ParseResults(['ATCAXCGAAXGGA'],
3063 {'original': 'ATCATCGAATGGA', 'mismatches': [4, 9]})
3064 """
3066 def __init__(
3067 self,
3068 match_string: str,
3069 max_mismatches: typing.Optional[int] = None,
3070 *,
3071 caseless=False,
3072 **kwargs,
3073 ) -> None:
3074 maxMismatches: int = deprecate_argument(kwargs, "maxMismatches", 1)
3076 maxMismatches = max_mismatches if max_mismatches is not None else maxMismatches
3077 super().__init__()
3078 self.match_string = match_string
3079 self.maxMismatches = maxMismatches
3080 self.errmsg = f"Expected {self.match_string!r} (with up to {self.maxMismatches} mismatches)"
3081 self.caseless = caseless
3082 self.mayIndexError = False
3083 self._may_return_empty = False
3085 def _generateDefaultName(self) -> str:
3086 return f"{type(self).__name__}:{self.match_string!r}"
3088 def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType:
3089 start = loc
3090 instrlen = len(instring)
3091 maxloc = start + len(self.match_string)
3093 if maxloc <= instrlen:
3094 match_string = self.match_string
3095 match_stringloc = 0
3096 mismatches = []
3097 maxMismatches = self.maxMismatches
3099 for match_stringloc, s_m in enumerate(
3100 zip(instring[loc:maxloc], match_string)
3101 ):
3102 src, mat = s_m
3103 if self.caseless:
3104 src, mat = src.lower(), mat.lower()
3106 if src != mat:
3107 mismatches.append(match_stringloc)
3108 if len(mismatches) > maxMismatches:
3109 break
3110 else:
3111 loc = start + match_stringloc + 1
3112 results = ParseResults([instring[start:loc]])
3113 results["original"] = match_string
3114 results["mismatches"] = mismatches
3115 return loc, results
3117 raise ParseException(instring, loc, self.errmsg, self)
3120class Word(Token):
3121 """Token for matching words composed of allowed character sets.
3123 Parameters:
3125 - ``init_chars`` - string of all characters that should be used to
3126 match as a word; "ABC" will match "AAA", "ABAB", "CBAC", etc.;
3127 if ``body_chars`` is also specified, then this is the string of
3128 initial characters
3129 - ``body_chars`` - string of characters that
3130 can be used for matching after a matched initial character as
3131 given in ``init_chars``; if omitted, same as the initial characters
3132 (default=``None``)
3133 - ``min`` - minimum number of characters to match (default=1)
3134 - ``max`` - maximum number of characters to match (default=0)
3135 - ``exact`` - exact number of characters to match (default=0)
3136 - ``as_keyword`` - match as a keyword (default=``False``)
3137 - ``exclude_chars`` - characters that might be
3138 found in the input ``body_chars`` string but which should not be
3139 accepted for matching ;useful to define a word of all
3140 printables except for one or two characters, for instance
3141 (default=``None``)
3143 :class:`srange` is useful for defining custom character set strings
3144 for defining :class:`Word` expressions, using range notation from
3145 regular expression character sets.
3147 A common mistake is to use :class:`Word` to match a specific literal
3148 string, as in ``Word("Address")``. Remember that :class:`Word`
3149 uses the string argument to define *sets* of matchable characters.
3150 This expression would match "Add", "AAA", "dAred", or any other word
3151 made up of the characters 'A', 'd', 'r', 'e', and 's'. To match an
3152 exact literal string, use :class:`Literal` or :class:`Keyword`.
3154 pyparsing includes helper strings for building Words:
3156 - :attr:`alphas`
3157 - :attr:`nums`
3158 - :attr:`alphanums`
3159 - :attr:`hexnums`
3160 - :attr:`alphas8bit` (alphabetic characters in ASCII range 128-255
3161 - accented, tilded, umlauted, etc.)
3162 - :attr:`punc8bit` (non-alphabetic characters in ASCII range
3163 128-255 - currency, symbols, superscripts, diacriticals, etc.)
3164 - :attr:`printables` (any non-whitespace character)
3166 ``alphas``, ``nums``, and ``printables`` are also defined in several
3167 Unicode sets - see :class:`pyparsing_unicode`.
3169 Example:
3171 .. testcode::
3173 # a word composed of digits
3174 integer = Word(nums)
3175 # Two equivalent alternate forms:
3176 Word("0123456789")
3177 Word(srange("[0-9]"))
3179 # a word with a leading capital, and zero or more lowercase
3180 capitalized_word = Word(alphas.upper(), alphas.lower())
3182 # hostnames are alphanumeric, with leading alpha, and '-'
3183 hostname = Word(alphas, alphanums + '-')
3185 # roman numeral
3186 # (not a strict parser, accepts invalid mix of characters)
3187 roman = Word("IVXLCDM")
3189 # any string of non-whitespace characters, except for ','
3190 csv_value = Word(printables, exclude_chars=",")
3192 :raises ValueError: If ``min`` and ``max`` are both specified
3193 and the test ``min <= max`` fails.
3195 .. versionchanged:: 3.1.0
3196 Raises :exc:`ValueError` if ``min`` > ``max``.
3197 """
3199 def __init__(
3200 self,
3201 init_chars: str = "",
3202 body_chars: typing.Optional[str] = None,
3203 min: int = 1,
3204 max: int = 0,
3205 exact: int = 0,
3206 as_keyword: bool = False,
3207 exclude_chars: typing.Optional[str] = None,
3208 **kwargs,
3209 ) -> None:
3210 initChars: typing.Optional[str] = deprecate_argument(kwargs, "initChars", None)
3211 bodyChars: typing.Optional[str] = deprecate_argument(kwargs, "bodyChars", None)
3212 asKeyword: bool = deprecate_argument(kwargs, "asKeyword", False)
3213 excludeChars: typing.Optional[str] = deprecate_argument(
3214 kwargs, "excludeChars", None
3215 )
3217 initChars = initChars or init_chars
3218 bodyChars = bodyChars or body_chars
3219 asKeyword = asKeyword or as_keyword
3220 excludeChars = excludeChars or exclude_chars
3221 super().__init__()
3222 if not initChars:
3223 raise ValueError(
3224 f"invalid {type(self).__name__}, initChars cannot be empty string"
3225 )
3227 initChars_set = set(initChars)
3228 if excludeChars:
3229 excludeChars_set = set(excludeChars)
3230 initChars_set -= excludeChars_set
3231 if bodyChars:
3232 bodyChars = "".join(set(bodyChars) - excludeChars_set)
3233 self.init_chars = initChars_set
3234 self.initCharsOrig = "".join(sorted(initChars_set))
3236 if bodyChars:
3237 self.bodyChars = set(bodyChars)
3238 self.bodyCharsOrig = "".join(sorted(bodyChars))
3239 else:
3240 self.bodyChars = initChars_set
3241 self.bodyCharsOrig = self.initCharsOrig
3243 self.maxSpecified = max > 0
3245 if min < 1:
3246 raise ValueError(
3247 "cannot specify a minimum length < 1; use Opt(Word()) if zero-length word is permitted"
3248 )
3250 if self.maxSpecified and min > max:
3251 raise ValueError(
3252 f"invalid args, if min and max both specified min must be <= max (min={min}, max={max})"
3253 )
3255 self.minLen = min
3257 if max > 0:
3258 self.maxLen = max
3259 else:
3260 self.maxLen = _MAX_INT
3262 if exact > 0:
3263 min = max = exact
3264 self.maxLen = exact
3265 self.minLen = exact
3267 self.errmsg = f"Expected {self.name}"
3268 self.mayIndexError = False
3269 self.asKeyword = asKeyword
3270 if self.asKeyword:
3271 self.errmsg += " as a keyword"
3273 # see if we can make a regex for this Word
3274 if " " not in (self.initChars | self.bodyChars):
3275 if len(self.initChars) == 1:
3276 re_leading_fragment = re.escape(self.initCharsOrig)
3277 else:
3278 re_leading_fragment = f"[{_collapse_string_to_ranges(self.initChars)}]"
3280 if self.bodyChars == self.initChars:
3281 if max == 0 and self.minLen == 1:
3282 repeat = "+"
3283 elif max == 1:
3284 repeat = ""
3285 else:
3286 if self.minLen != self.maxLen:
3287 repeat = f"{{{self.minLen},{'' if self.maxLen == _MAX_INT else self.maxLen}}}"
3288 else:
3289 repeat = f"{{{self.minLen}}}"
3290 self.reString = f"{re_leading_fragment}{repeat}"
3291 else:
3292 if max == 1:
3293 re_body_fragment = ""
3294 repeat = ""
3295 else:
3296 re_body_fragment = f"[{_collapse_string_to_ranges(self.bodyChars)}]"
3297 if max == 0 and self.minLen == 1:
3298 repeat = "*"
3299 elif max == 2:
3300 repeat = "?" if min <= 1 else ""
3301 else:
3302 if min != max:
3303 repeat = f"{{{min - 1 if min > 0 else ''},{max - 1 if max > 0 else ''}}}"
3304 else:
3305 repeat = f"{{{min - 1 if min > 0 else ''}}}"
3307 self.reString = f"{re_leading_fragment}{re_body_fragment}{repeat}"
3309 if self.asKeyword:
3310 self.reString = rf"\b{self.reString}\b"
3312 try:
3313 self.re = re.compile(self.reString)
3314 except re.error:
3315 self.re = None # type: ignore[assignment]
3316 else:
3317 self.re_match = self.re.match
3318 self.parseImpl = self.parseImpl_regex # type: ignore[method-assign]
3320 @property
3321 def initChars(self) -> set[str]:
3322 """
3323 .. deprecated:: 3.3.0
3324 use `init_chars` instead.
3326 Property returning the initial chars to be used when matching this
3327 Word expression. If no body chars were specified, the initial characters
3328 will also be the body characters.
3329 """
3330 return set(self.init_chars)
3332 def copy(self) -> Word:
3333 """
3334 Returns a copy of this expression.
3336 Generally only used internally by pyparsing.
3337 """
3338 ret: Word = cast(Word, super().copy())
3339 if hasattr(self, "re_match"):
3340 ret.re_match = self.re_match
3341 ret.parseImpl = ret.parseImpl_regex # type: ignore[method-assign]
3342 return ret
3344 def _generateDefaultName(self) -> str:
3345 def charsAsStr(s):
3346 max_repr_len = 16
3347 s = _collapse_string_to_ranges(s, re_escape=False)
3349 if len(s) > max_repr_len:
3350 return f"{s[:max_repr_len - 3]}..."
3352 return s
3354 if self.initChars != self.bodyChars:
3355 base = f"W:({charsAsStr(self.initChars)}, {charsAsStr(self.bodyChars)})"
3356 else:
3357 base = f"W:({charsAsStr(self.initChars)})"
3359 # add length specification
3360 if self.minLen > 1 or self.maxLen != _MAX_INT:
3361 if self.minLen == self.maxLen:
3362 if self.minLen == 1:
3363 return base[2:]
3364 else:
3365 return base + f"{{{self.minLen}}}"
3366 elif self.maxLen == _MAX_INT:
3367 return base + f"{{{self.minLen},...}}"
3368 else:
3369 return base + f"{{{self.minLen},{self.maxLen}}}"
3370 return base
3372 def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType:
3373 if instring[loc] not in self.initChars:
3374 raise ParseException(instring, loc, self.errmsg, self)
3376 start = loc
3377 loc += 1
3378 instrlen = len(instring)
3379 body_chars: set[str] = self.bodyChars
3380 maxloc = start + self.maxLen
3381 maxloc = min(maxloc, instrlen)
3382 while loc < maxloc and instring[loc] in body_chars:
3383 loc += 1
3385 throw_exception = False
3386 if loc - start < self.minLen:
3387 throw_exception = True
3388 elif self.asKeyword and (
3389 (start > 0 and instring[start - 1] in body_chars)
3390 or (loc < instrlen and instring[loc] in body_chars)
3391 ):
3392 throw_exception = True
3394 if throw_exception:
3395 raise ParseException(instring, loc, self.errmsg, self)
3397 return loc, instring[start:loc]
3399 def parseImpl_regex(self, instring, loc, do_actions=True) -> ParseImplReturnType:
3400 result = self.re_match(instring, loc)
3401 if not result:
3402 raise ParseException(instring, loc, self.errmsg, self)
3404 loc = result.end()
3405 return loc, result[0]
3408class Char(Word):
3409 """A short-cut class for defining :class:`Word` ``(characters, exact=1)``,
3410 when defining a match of any single character in a string of
3411 characters.
3412 """
3414 def __init__(
3415 self,
3416 charset: str,
3417 as_keyword: bool = False,
3418 exclude_chars: typing.Optional[str] = None,
3419 **kwargs,
3420 ) -> None:
3421 asKeyword: bool = deprecate_argument(kwargs, "asKeyword", False)
3422 excludeChars: typing.Optional[str] = deprecate_argument(
3423 kwargs, "excludeChars", None
3424 )
3426 asKeyword = asKeyword or as_keyword
3427 excludeChars = excludeChars or exclude_chars
3428 super().__init__(
3429 charset, exact=1, as_keyword=asKeyword, exclude_chars=excludeChars
3430 )
3433class Regex(Token):
3434 r"""Token for matching strings that match a given regular
3435 expression. Defined with string specifying the regular expression in
3436 a form recognized by the stdlib Python `re module <https://docs.python.org/3/library/re.html>`_.
3437 If the given regex contains named groups (defined using ``(?P<name>...)``),
3438 these will be preserved as named :class:`ParseResults`.
3440 If instead of the Python stdlib ``re`` module you wish to use a different RE module
3441 (such as the ``regex`` module), you can do so by building your ``Regex`` object with
3442 a compiled RE that was compiled using ``regex``.
3444 The parameters ``pattern`` and ``flags`` are passed
3445 to the ``re.compile()`` function as-is. See the Python
3446 `re module <https://docs.python.org/3/library/re.html>`_ module for an
3447 explanation of the acceptable patterns and flags.
3449 Example:
3451 .. testcode::
3453 realnum = Regex(r"[+-]?\d+\.\d*")
3454 # ref: https://stackoverflow.com/questions/267399/how-do-you-match-only-valid-roman-numerals-with-a-regular-expression
3455 roman = Regex(r"M{0,4}(CM|CD|D?{0,3})(XC|XL|L?X{0,3})(IX|IV|V?I{0,3})")
3457 # named fields in a regex will be returned as named results
3458 date = Regex(r'(?P<year>\d{4})-(?P<month>\d\d?)-(?P<day>\d\d?)')
3460 # the Regex class will accept regular expressions compiled using the
3461 # re module
3462 import re
3463 parser = pp.Regex(re.compile(r'[0-9]'))
3464 """
3466 def __init__(
3467 self,
3468 pattern: Any,
3469 flags: Union[re.RegexFlag, int] = 0,
3470 as_group_list: bool = False,
3471 as_match: bool = False,
3472 **kwargs,
3473 ) -> None:
3474 super().__init__()
3475 asGroupList: bool = deprecate_argument(kwargs, "asGroupList", False)
3476 asMatch: bool = deprecate_argument(kwargs, "asMatch", False)
3478 asGroupList = asGroupList or as_group_list
3479 asMatch = asMatch or as_match
3481 if isinstance(pattern, str_type):
3482 if not pattern:
3483 raise ValueError("null string passed to Regex; use Empty() instead")
3485 self._re = None
3486 self._may_return_empty = None # type: ignore [assignment]
3487 self.reString = self.pattern = pattern
3489 elif hasattr(pattern, "pattern") and hasattr(pattern, "match"):
3490 self._re = pattern
3491 self._may_return_empty = None # type: ignore [assignment]
3492 self.pattern = self.reString = pattern.pattern
3494 elif callable(pattern):
3495 # defer creating this pattern until we really need it
3496 self.pattern = pattern
3497 self._may_return_empty = None # type: ignore [assignment]
3498 self._re = None
3500 else:
3501 raise TypeError(
3502 "Regex may only be constructed with a string or a compiled RE object,"
3503 " or a callable that takes no arguments and returns a string or a"
3504 " compiled RE object"
3505 )
3507 self.flags = flags
3508 self.errmsg = f"Expected {self.name}"
3509 self.mayIndexError = False
3510 self.asGroupList = asGroupList
3511 self.asMatch = asMatch
3512 if self.asGroupList:
3513 self.parseImpl = self.parseImplAsGroupList # type: ignore [method-assign]
3514 if self.asMatch:
3515 self.parseImpl = self.parseImplAsMatch # type: ignore [method-assign]
3517 def copy(self) -> Regex:
3518 """
3519 Returns a copy of this expression.
3521 Generally only used internally by pyparsing.
3522 """
3523 ret: Regex = cast(Regex, super().copy())
3524 if self.asGroupList:
3525 ret.parseImpl = ret.parseImplAsGroupList # type: ignore [method-assign]
3526 if self.asMatch:
3527 ret.parseImpl = ret.parseImplAsMatch # type: ignore [method-assign]
3528 return ret
3530 @cached_property
3531 def re(self) -> re.Pattern:
3532 """
3533 Property returning the compiled regular expression for this Regex.
3535 Generally only used internally by pyparsing.
3536 """
3537 if self._re:
3538 return self._re
3540 if callable(self.pattern):
3541 # replace self.pattern with the string returned by calling self.pattern()
3542 self.pattern = cast(Callable[[], str], self.pattern)()
3544 # see if we got a compiled RE back instead of a str - if so, we're done
3545 if hasattr(self.pattern, "pattern") and hasattr(self.pattern, "match"):
3546 self._re = cast(re.Pattern[str], self.pattern)
3547 self.pattern = self.reString = self._re.pattern
3548 return self._re
3550 try:
3551 self._re = re.compile(self.pattern, self.flags)
3552 except re.error:
3553 raise ValueError(f"invalid pattern ({self.pattern!r}) passed to Regex")
3554 else:
3555 self._may_return_empty = self.re.match("", pos=0) is not None
3556 return self._re
3558 @cached_property
3559 def re_match(self) -> Callable[[str, int], Any]:
3560 return self.re.match
3562 @property
3563 def mayReturnEmpty(self):
3564 if self._may_return_empty is None:
3565 # force compile of regex pattern, to set may_return_empty flag
3566 self.re # noqa
3567 return self._may_return_empty
3569 @mayReturnEmpty.setter
3570 def mayReturnEmpty(self, value):
3571 self._may_return_empty = value
3573 def _generateDefaultName(self) -> str:
3574 unescaped = repr(self.pattern).replace("\\\\", "\\")
3575 return f"Re:({unescaped})"
3577 def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType:
3578 # explicit check for matching past the length of the string;
3579 # this is done because the re module will not complain about
3580 # a match with `pos > len(instring)`, it will just return ""
3581 if loc > len(instring) and self.mayReturnEmpty:
3582 raise ParseException(instring, loc, self.errmsg, self)
3584 result = self.re_match(instring, loc)
3585 if not result:
3586 raise ParseException(instring, loc, self.errmsg, self)
3588 loc = result.end()
3589 ret = ParseResults(result[0])
3590 d = result.groupdict()
3592 for k, v in d.items():
3593 ret[k] = v
3595 return loc, ret
3597 def parseImplAsGroupList(self, instring, loc, do_actions=True):
3598 if loc > len(instring) and self.mayReturnEmpty:
3599 raise ParseException(instring, loc, self.errmsg, self)
3601 result = self.re_match(instring, loc)
3602 if not result:
3603 raise ParseException(instring, loc, self.errmsg, self)
3605 loc = result.end()
3606 ret = result.groups()
3607 return loc, ret
3609 def parseImplAsMatch(self, instring, loc, do_actions=True):
3610 if loc > len(instring) and self.mayReturnEmpty:
3611 raise ParseException(instring, loc, self.errmsg, self)
3613 result = self.re_match(instring, loc)
3614 if not result:
3615 raise ParseException(instring, loc, self.errmsg, self)
3617 loc = result.end()
3618 ret = result
3619 return loc, ret
3621 def sub(self, repl: str) -> ParserElement:
3622 r"""
3623 Return :class:`Regex` with an attached parse action to transform the parsed
3624 result as if called using `re.sub(expr, repl, string) <https://docs.python.org/3/library/re.html#re.sub>`_.
3626 Example:
3628 .. testcode::
3630 make_html = Regex(r"(\w+):(.*?):").sub(r"<\1>\2</\1>")
3631 print(make_html.transform_string("h1:main title:"))
3633 .. testoutput::
3635 <h1>main title</h1>
3636 """
3637 if self.asGroupList:
3638 raise TypeError("cannot use sub() with Regex(as_group_list=True)")
3640 if self.asMatch and callable(repl):
3641 raise TypeError(
3642 "cannot use sub() with a callable with Regex(as_match=True)"
3643 )
3645 if self.asMatch:
3647 def pa(tokens):
3648 return tokens[0].expand(repl)
3650 else:
3652 def pa(tokens):
3653 return self.re.sub(repl, tokens[0])
3655 return self.add_parse_action(pa)
3658class QuotedString(Token):
3659 r"""
3660 Token for matching strings that are delimited by quoting characters.
3662 Defined with the following parameters:
3664 - ``quote_char`` - string of one or more characters defining the
3665 quote delimiting string
3666 - ``esc_char`` - character to re_escape quotes, typically backslash
3667 (default= ``None``)
3668 - ``esc_quote`` - special quote sequence to re_escape an embedded quote
3669 string (such as SQL's ``""`` to re_escape an embedded ``"``)
3670 (default= ``None``)
3671 - ``multiline`` - boolean indicating whether quotes can span
3672 multiple lines (default= ``False``)
3673 - ``unquote_results`` - boolean indicating whether the matched text
3674 should be unquoted (default= ``True``)
3675 - ``end_quote_char`` - string of one or more characters defining the
3676 end of the quote delimited string (default= ``None`` => same as
3677 quote_char)
3678 - ``convert_whitespace_escapes`` - convert escaped whitespace
3679 (``'\t'``, ``'\n'``, etc.) to actual whitespace
3680 (default= ``True``)
3682 .. caution:: ``convert_whitespace_escapes`` has no effect if
3683 ``unquote_results`` is ``False``.
3685 Example:
3687 .. doctest::
3689 >>> qs = QuotedString('"')
3690 >>> print(qs.search_string('lsjdf "This is the quote" sldjf'))
3691 [['This is the quote']]
3692 >>> complex_qs = QuotedString('{{', end_quote_char='}}')
3693 >>> print(complex_qs.search_string(
3694 ... 'lsjdf {{This is the "quote"}} sldjf'))
3695 [['This is the "quote"']]
3696 >>> sql_qs = QuotedString('"', esc_quote='""')
3697 >>> print(sql_qs.search_string(
3698 ... 'lsjdf "This is the quote with ""embedded"" quotes" sldjf'))
3699 [['This is the quote with "embedded" quotes']]
3700 """
3702 ws_map = dict(((r"\t", "\t"), (r"\n", "\n"), (r"\f", "\f"), (r"\r", "\r")))
3704 def __init__(
3705 self,
3706 quote_char: str = "",
3707 esc_char: typing.Optional[str] = None,
3708 esc_quote: typing.Optional[str] = None,
3709 multiline: bool = False,
3710 unquote_results: bool = True,
3711 end_quote_char: typing.Optional[str] = None,
3712 convert_whitespace_escapes: bool = True,
3713 **kwargs,
3714 ) -> None:
3715 super().__init__()
3716 quoteChar: str = deprecate_argument(kwargs, "quoteChar", "")
3717 escChar: str = deprecate_argument(kwargs, "escChar", None)
3718 escQuote: str = deprecate_argument(kwargs, "escQuote", None)
3719 unquoteResults: bool = deprecate_argument(kwargs, "unquoteResults", True)
3720 endQuoteChar: typing.Optional[str] = deprecate_argument(
3721 kwargs, "endQuoteChar", None
3722 )
3723 convertWhitespaceEscapes: bool = deprecate_argument(
3724 kwargs, "convertWhitespaceEscapes", True
3725 )
3727 esc_char = escChar or esc_char
3728 esc_quote = escQuote or esc_quote
3729 unquote_results = unquoteResults and unquote_results
3730 end_quote_char = endQuoteChar or end_quote_char
3731 convert_whitespace_escapes = (
3732 convertWhitespaceEscapes and convert_whitespace_escapes
3733 )
3734 quote_char = quoteChar or quote_char
3736 # reject empty or whitespace-only quote chars, but preserve any
3737 # whitespace that is part of a valid quote delimiter (e.g. a leading
3738 # newline in a multiline quote such as "\n;")
3739 if not quote_char.strip():
3740 raise ValueError("quote_char cannot be the empty string")
3742 if end_quote_char is None:
3743 end_quote_char = quote_char
3744 elif not end_quote_char.strip():
3745 raise ValueError("end_quote_char cannot be the empty string")
3747 self.quote_char: str = quote_char
3748 self.quote_char_len: int = len(quote_char)
3749 self.first_quote_char: str = quote_char[0]
3750 self.end_quote_char: str = end_quote_char
3751 self.end_quote_char_len: int = len(end_quote_char)
3752 self.esc_char: str = esc_char or ""
3753 self.has_esc_char: bool = esc_char is not None
3754 self.esc_quote: str = esc_quote or ""
3755 self.unquote_results: bool = unquote_results
3756 self.convert_whitespace_escapes: bool = convert_whitespace_escapes
3757 self.multiline = multiline
3758 self.re_flags = re.RegexFlag(0)
3760 # fmt: off
3761 # build up re pattern for the content between the quote delimiters
3762 inner_pattern: list[str] = []
3764 if esc_quote:
3765 inner_pattern.append(rf"(?:{re.escape(esc_quote)})")
3767 if esc_char:
3768 inner_pattern.append(rf"(?:{re.escape(esc_char)}.)")
3770 if len(self.end_quote_char) > 1:
3771 inner_pattern.append(
3772 "(?:"
3773 + "|".join(
3774 f"(?:{re.escape(self.end_quote_char[:i])}(?!{re.escape(self.end_quote_char[i:])}))"
3775 for i in range(len(self.end_quote_char) - 1, 0, -1)
3776 )
3777 + ")"
3778 )
3780 if self.multiline:
3781 self.re_flags |= re.MULTILINE | re.DOTALL
3782 inner_pattern.append(
3783 rf"(?:[^{_escape_regex_range_chars(self.end_quote_char[0])}"
3784 rf"{(_escape_regex_range_chars(self.esc_char) if self.has_esc_char else '')}])"
3785 )
3786 else:
3787 inner_pattern.append(
3788 rf"(?:[^{_escape_regex_range_chars(self.end_quote_char[0])}\n\r"
3789 rf"{(_escape_regex_range_chars(self.esc_char) if self.has_esc_char else '')}])"
3790 )
3792 self.pattern = "".join(
3793 [
3794 re.escape(self.quote_char),
3795 "(?:",
3796 '|'.join(inner_pattern),
3797 ")*",
3798 re.escape(self.end_quote_char),
3799 ]
3800 )
3802 if self.unquote_results:
3803 if self.convert_whitespace_escapes:
3804 self.unquote_scan_re = re.compile(
3805 rf"({'|'.join(re.escape(k) for k in self.ws_map)})"
3806 rf"|(\\[0-7]{3}|\\0|\\x[0-9a-fA-F]{2}|\\u[0-9a-fA-F]{4})"
3807 rf"|({re.escape(self.esc_char)}.)"
3808 rf"|(\n|.)",
3809 flags=self.re_flags,
3810 )
3811 else:
3812 self.unquote_scan_re = re.compile(
3813 rf"({re.escape(self.esc_char)}.)"
3814 rf"|(\n|.)",
3815 flags=self.re_flags
3816 )
3817 # fmt: on
3819 try:
3820 self.re = re.compile(self.pattern, self.re_flags)
3821 self.reString = self.pattern
3822 self.re_match = self.re.match
3823 except re.error:
3824 raise ValueError(f"invalid pattern {self.pattern!r} passed to Regex")
3826 self.errmsg = f"Expected {self.name}"
3827 self.mayIndexError = False
3828 self._may_return_empty = True
3830 def _generateDefaultName(self) -> str:
3831 if self.quote_char == self.end_quote_char and isinstance(
3832 self.quote_char, str_type
3833 ):
3834 return f"string enclosed in {self.quote_char!r}"
3836 return f"quoted string, starting with {self.quote_char} ending with {self.end_quote_char}"
3838 def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType:
3839 # check first character of opening quote to see if that is a match
3840 # before doing the more complicated regex match
3841 result = (
3842 instring[loc] == self.first_quote_char
3843 and self.re_match(instring, loc)
3844 or None
3845 )
3846 if not result:
3847 raise ParseException(instring, loc, self.errmsg, self)
3849 # get ending loc and matched string from regex matching result
3850 loc = result.end()
3851 ret = result[0]
3853 if self.unquote_results:
3854 # strip off quotes
3855 ret = ret[self.quote_char_len : -self.end_quote_char_len]
3857 if isinstance(ret, str_type):
3858 # fmt: off
3859 if self.convert_whitespace_escapes:
3860 # as we iterate over matches in the input string,
3861 # collect from whichever match group of the unquote_scan_re
3862 # regex matches (only 1 group will match at any given time)
3863 ret = "".join(
3864 # match group 1 matches \t, \n, etc.
3865 self.ws_map[g] if (g := match[1])
3866 # match group 2 matches escaped octal, null, hex, and Unicode
3867 # sequences
3868 else _convert_escaped_numerics_to_char(g[1:]) if (g := match[2])
3869 # match group 3 matches escaped characters
3870 else g[-1] if (g := match[3])
3871 # match group 4 matches any character
3872 else match[4]
3873 for match in self.unquote_scan_re.finditer(ret)
3874 )
3875 else:
3876 ret = "".join(
3877 # match group 1 matches escaped characters
3878 g[-1] if (g := match[1])
3879 # match group 2 matches any character
3880 else match[2]
3881 for match in self.unquote_scan_re.finditer(ret)
3882 )
3883 # fmt: on
3885 # replace escaped quotes
3886 if self.esc_quote:
3887 ret = ret.replace(self.esc_quote, self.end_quote_char)
3889 return loc, ret
3892class CharsNotIn(Token):
3893 """Token for matching words composed of characters *not* in a given
3894 set (will include whitespace in matched characters if not listed in
3895 the provided exclusion set - see example). Defined with string
3896 containing all disallowed characters, and an optional minimum,
3897 maximum, and/or exact length. The default value for ``min`` is
3898 1 (a minimum value < 1 is not valid); the default values for
3899 ``max`` and ``exact`` are 0, meaning no maximum or exact
3900 length restriction.
3902 Example:
3904 .. testcode::
3906 # define a comma-separated-value as anything that is not a ','
3907 csv_value = CharsNotIn(',')
3908 print(
3909 DelimitedList(csv_value).parse_string(
3910 "dkls,lsdkjf,s12 34,@!#,213"
3911 )
3912 )
3914 prints:
3916 .. testoutput::
3918 ['dkls', 'lsdkjf', 's12 34', '@!#', '213']
3919 """
3921 def __init__(
3922 self, not_chars: str = "", min: int = 1, max: int = 0, exact: int = 0, **kwargs
3923 ) -> None:
3924 super().__init__()
3925 notChars: str = deprecate_argument(kwargs, "notChars", "")
3927 self.skipWhitespace = False
3928 self.notChars = not_chars or notChars
3929 self.notCharsSet = set(self.notChars)
3931 if min < 1:
3932 raise ValueError(
3933 "cannot specify a minimum length < 1; use"
3934 " Opt(CharsNotIn()) if zero-length char group is permitted"
3935 )
3937 self.minLen = min
3939 if max > 0:
3940 self.maxLen = max
3941 else:
3942 self.maxLen = _MAX_INT
3944 if exact > 0:
3945 self.maxLen = exact
3946 self.minLen = exact
3948 self.errmsg = f"Expected {self.name}"
3949 self._may_return_empty = self.minLen == 0
3950 self.mayIndexError = False
3952 def _generateDefaultName(self) -> str:
3953 not_chars_str = _collapse_string_to_ranges(self.notChars)
3954 if len(not_chars_str) > 16:
3955 return f"!W:({self.notChars[: 16 - 3]}...)"
3956 else:
3957 return f"!W:({self.notChars})"
3959 def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType:
3960 notchars = self.notCharsSet
3961 if instring[loc] in notchars:
3962 raise ParseException(instring, loc, self.errmsg, self)
3964 start = loc
3965 loc += 1
3966 maxlen = min(start + self.maxLen, len(instring))
3967 while loc < maxlen and instring[loc] not in notchars:
3968 loc += 1
3970 if loc - start < self.minLen:
3971 raise ParseException(instring, loc, self.errmsg, self)
3973 return loc, instring[start:loc]
3976class White(Token):
3977 """Special matching class for matching whitespace. Normally,
3978 whitespace is ignored by pyparsing grammars. This class is included
3979 when some whitespace structures are significant. Define with
3980 a string containing the whitespace characters to be matched; default
3981 is ``" \\t\\r\\n"``. Also takes optional ``min``,
3982 ``max``, and ``exact`` arguments, as defined for the
3983 :class:`Word` class.
3984 """
3986 whiteStrs = {
3987 " ": "<SP>",
3988 "\t": "<TAB>",
3989 "\n": "<LF>",
3990 "\r": "<CR>",
3991 "\f": "<FF>",
3992 "\u00a0": "<NBSP>",
3993 "\u1680": "<OGHAM_SPACE_MARK>",
3994 "\u180e": "<MONGOLIAN_VOWEL_SEPARATOR>",
3995 "\u2000": "<EN_QUAD>",
3996 "\u2001": "<EM_QUAD>",
3997 "\u2002": "<EN_SPACE>",
3998 "\u2003": "<EM_SPACE>",
3999 "\u2004": "<THREE-PER-EM_SPACE>",
4000 "\u2005": "<FOUR-PER-EM_SPACE>",
4001 "\u2006": "<SIX-PER-EM_SPACE>",
4002 "\u2007": "<FIGURE_SPACE>",
4003 "\u2008": "<PUNCTUATION_SPACE>",
4004 "\u2009": "<THIN_SPACE>",
4005 "\u200a": "<HAIR_SPACE>",
4006 "\u200b": "<ZERO_WIDTH_SPACE>",
4007 "\u202f": "<NNBSP>",
4008 "\u205f": "<MMSP>",
4009 "\u3000": "<IDEOGRAPHIC_SPACE>",
4010 }
4012 def __init__(
4013 self, ws: str = " \t\r\n", min: int = 1, max: int = 0, exact: int = 0
4014 ) -> None:
4015 super().__init__()
4016 self.matchWhite = ws
4017 self.set_whitespace_chars(
4018 "".join(c for c in self.whiteStrs if c not in self.matchWhite),
4019 copy_defaults=True,
4020 )
4021 # self.leave_whitespace()
4022 self._may_return_empty = True
4023 self.errmsg = f"Expected {self.name}"
4025 self.minLen = min
4027 if max > 0:
4028 self.maxLen = max
4029 else:
4030 self.maxLen = _MAX_INT
4032 if exact > 0:
4033 self.maxLen = exact
4034 self.minLen = exact
4036 def _generateDefaultName(self) -> str:
4037 return "".join(White.whiteStrs[c] for c in self.matchWhite)
4039 def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType:
4040 if instring[loc] not in self.matchWhite:
4041 raise ParseException(instring, loc, self.errmsg, self)
4042 start = loc
4043 loc += 1
4044 maxloc = start + self.maxLen
4045 maxloc = min(maxloc, len(instring))
4046 while loc < maxloc and instring[loc] in self.matchWhite:
4047 loc += 1
4049 if loc - start < self.minLen:
4050 raise ParseException(instring, loc, self.errmsg, self)
4052 return loc, instring[start:loc]
4055class PositionToken(Token):
4056 def __init__(self) -> None:
4057 super().__init__()
4058 self._may_return_empty = True
4059 self.mayIndexError = False
4062class GoToColumn(PositionToken):
4063 """Token to advance to a specific column of input text; useful for
4064 tabular report scraping.
4065 """
4067 def __init__(self, colno: int) -> None:
4068 super().__init__()
4069 self.col = colno
4071 def preParse(self, instring: str, loc: int) -> int:
4072 if col(loc, instring) == self.col:
4073 return loc
4075 instrlen = len(instring)
4076 if self.ignoreExprs:
4077 loc = self._skipIgnorables(instring, loc)
4078 while (
4079 loc < instrlen
4080 and instring[loc].isspace()
4081 and col(loc, instring) != self.col
4082 ):
4083 loc += 1
4085 return loc
4087 def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType:
4088 thiscol = col(loc, instring)
4089 if thiscol > self.col:
4090 raise ParseException(instring, loc, "Text not in expected column", self)
4091 newloc = loc + self.col - thiscol
4092 ret = instring[loc:newloc]
4093 return newloc, ret
4096class LineStart(PositionToken):
4097 r"""Matches if current position is at the logical beginning of a line (after skipping whitespace)
4098 within the parse string
4100 Example:
4102 .. testcode::
4104 test = '''\
4105 AAA this line
4106 AAA and this line
4107 AAA and even this line
4108 B AAA but definitely not this line
4109 '''
4111 for t in (LineStart() + 'AAA' + rest_of_line).search_string(test):
4112 print(t)
4114 prints:
4116 .. testoutput::
4118 ['AAA', ' this line']
4119 ['AAA', ' and this line']
4120 ['AAA', ' and even this line']
4122 """
4124 def __init__(self) -> None:
4125 super().__init__()
4126 self.leave_whitespace()
4127 self.orig_whiteChars = set() | self.whiteChars
4128 self.whiteChars.discard("\n")
4129 self.skipper = Empty().set_whitespace_chars(self.whiteChars)
4130 self.set_name("start of line")
4132 def preParse(self, instring: str, loc: int) -> int:
4133 if loc == 0:
4134 return loc
4136 ret = self.skipper.preParse(instring, loc)
4138 if "\n" in self.orig_whiteChars:
4139 while instring[ret : ret + 1] == "\n":
4140 ret = self.skipper.preParse(instring, ret + 1)
4142 return ret
4144 def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType:
4145 if col(loc, instring) == 1:
4146 return loc, []
4147 raise ParseException(instring, loc, self.errmsg, self)
4150class LineEnd(PositionToken):
4151 """Matches if current position is at the end of a line within the
4152 parse string
4153 """
4155 def __init__(self) -> None:
4156 super().__init__()
4157 self.whiteChars.discard("\n")
4158 self.set_whitespace_chars(self.whiteChars, copy_defaults=False)
4159 self.set_name("end of line")
4161 def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType:
4162 if loc < len(instring):
4163 if instring[loc] == "\n":
4164 return loc + 1, "\n"
4165 else:
4166 raise ParseException(instring, loc, self.errmsg, self)
4167 elif loc == len(instring):
4168 return loc + 1, []
4169 else:
4170 raise ParseException(instring, loc, self.errmsg, self)
4173class StringStart(PositionToken):
4174 """Matches if current position is at the beginning of the parse
4175 string
4176 """
4178 def __init__(self) -> None:
4179 super().__init__()
4180 self.set_name("start of text")
4182 def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType:
4183 # see if entire string up to here is just whitespace and ignoreables
4184 if loc != 0 and loc != self.preParse(instring, 0):
4185 raise ParseException(instring, loc, self.errmsg, self)
4187 return loc, []
4190class StringEnd(PositionToken):
4191 """
4192 Matches if current position is at the end of the parse string
4193 """
4195 def __init__(self) -> None:
4196 super().__init__()
4197 self.set_name("end of text")
4199 def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType:
4200 if loc < len(instring):
4201 raise ParseException(instring, loc, self.errmsg, self)
4202 if loc == len(instring):
4203 return loc + 1, []
4204 if loc > len(instring):
4205 return loc, []
4207 raise ParseException(instring, loc, self.errmsg, self)
4210class WordStart(PositionToken):
4211 """Matches if the current position is at the beginning of a
4212 :class:`Word`, and is not preceded by any character in a given
4213 set of ``word_chars`` (default= ``printables``). To emulate the
4214 ``\b`` behavior of regular expressions, use
4215 ``WordStart(alphanums)``. ``WordStart`` will also match at
4216 the beginning of the string being parsed, or at the beginning of
4217 a line.
4218 """
4220 def __init__(self, word_chars: str = printables, **kwargs) -> None:
4221 wordChars: str = deprecate_argument(kwargs, "wordChars", printables)
4223 wordChars = word_chars if wordChars == printables else wordChars
4224 super().__init__()
4225 self.wordChars = set(wordChars)
4226 self.set_name("start of a word")
4228 def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType:
4229 if loc != 0:
4230 if (
4231 instring[loc - 1] in self.wordChars
4232 or instring[loc] not in self.wordChars
4233 ):
4234 raise ParseException(instring, loc, self.errmsg, self)
4235 return loc, []
4238class WordEnd(PositionToken):
4239 """Matches if the current position is at the end of a :class:`Word`,
4240 and is not followed by any character in a given set of ``word_chars``
4241 (default= ``printables``). To emulate the ``\b`` behavior of
4242 regular expressions, use ``WordEnd(alphanums)``. ``WordEnd``
4243 will also match at the end of the string being parsed, or at the end
4244 of a line.
4245 """
4247 def __init__(self, word_chars: str = printables, **kwargs) -> None:
4248 wordChars: str = deprecate_argument(kwargs, "wordChars", printables)
4250 wordChars = word_chars if wordChars == printables else wordChars
4251 super().__init__()
4252 self.wordChars = set(wordChars)
4253 self.skipWhitespace = False
4254 self.set_name("end of a word")
4256 def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType:
4257 instrlen = len(instring)
4258 if instrlen > 0 and loc < instrlen:
4259 if (
4260 instring[loc] in self.wordChars
4261 or instring[loc - 1] not in self.wordChars
4262 ):
4263 raise ParseException(instring, loc, self.errmsg, self)
4264 return loc, []
4267class Tag(Token):
4268 """
4269 A meta-element for inserting a named result into the parsed
4270 tokens that may be checked later in a parse action or while
4271 processing the parsed results. Accepts an optional tag value,
4272 defaulting to `True`.
4274 Example:
4276 .. doctest::
4278 >>> end_punc = "." | ("!" + Tag("enthusiastic"))
4279 >>> greeting = "Hello," + Word(alphas) + end_punc
4281 >>> result = greeting.parse_string("Hello, World.")
4282 >>> print(result.dump())
4283 ['Hello,', 'World', '.']
4285 >>> result = greeting.parse_string("Hello, World!")
4286 >>> print(result.dump())
4287 ['Hello,', 'World', '!']
4288 - enthusiastic: True
4290 .. versionadded:: 3.1.0
4291 """
4293 def __init__(self, tag_name: str, value: Any = True) -> None:
4294 super().__init__()
4295 self._may_return_empty = True
4296 self.mayIndexError = False
4297 self.leave_whitespace()
4298 self.tag_name = tag_name
4299 self.tag_value = value
4300 self.add_parse_action(self._add_tag)
4301 self.show_in_diagram = False
4303 def _add_tag(self, tokens: ParseResults):
4304 tokens[self.tag_name] = self.tag_value
4306 def _generateDefaultName(self) -> str:
4307 return f"{type(self).__name__}:{self.tag_name}={self.tag_value!r}"
4310class ParseExpression(ParserElement):
4311 """Abstract subclass of ParserElement, for combining and
4312 post-processing parsed tokens.
4313 """
4315 def __init__(
4316 self, exprs: typing.Iterable[ParserElement], savelist: bool = False
4317 ) -> None:
4318 super().__init__(savelist)
4319 self.exprs: list[ParserElement]
4320 if isinstance(exprs, _generatorType):
4321 exprs = list(exprs)
4323 if isinstance(exprs, str_type):
4324 self.exprs = [self._literalStringClass(exprs)]
4325 elif isinstance(exprs, ParserElement):
4326 self.exprs = [exprs]
4327 elif isinstance(exprs, Iterable):
4328 exprs = list(exprs)
4329 # if sequence of strings provided, wrap with Literal
4330 if any(isinstance(expr, str_type) for expr in exprs):
4331 exprs = (
4332 self._literalStringClass(e) if isinstance(e, str_type) else e
4333 for e in exprs
4334 )
4335 self.exprs = list(exprs)
4336 else:
4337 try:
4338 self.exprs = list(exprs)
4339 except TypeError:
4340 self.exprs = [exprs]
4341 self.callPreparse = False
4343 def recurse(self) -> list[ParserElement]:
4344 return self.exprs[:]
4346 def append(self, other) -> ParserElement:
4347 """
4348 Add an expression to the list of expressions related to this ParseExpression instance.
4349 """
4350 self.exprs.append(other)
4351 self._defaultName = None
4352 return self
4354 def leave_whitespace(self, recursive: bool = True) -> ParserElement:
4355 """
4356 Extends ``leave_whitespace`` defined in base class, and also invokes ``leave_whitespace`` on
4357 all contained expressions.
4358 """
4359 super().leave_whitespace(recursive)
4361 if recursive:
4362 self.exprs = [e.copy() for e in self.exprs]
4363 for e in self.exprs:
4364 e.leave_whitespace(recursive)
4365 return self
4367 def ignore_whitespace(self, recursive: bool = True) -> ParserElement:
4368 """
4369 Extends ``ignore_whitespace`` defined in base class, and also invokes ``ignore_whitespace`` on
4370 all contained expressions.
4371 """
4372 super().ignore_whitespace(recursive)
4373 if recursive:
4374 self.exprs = [e.copy() for e in self.exprs]
4375 for e in self.exprs:
4376 e.ignore_whitespace(recursive)
4377 return self
4379 def ignore(self, other) -> ParserElement:
4380 """
4381 Define expression to be ignored (e.g., comments) while doing pattern
4382 matching; may be called repeatedly, to define multiple comment or other
4383 ignorable patterns.
4384 """
4385 if isinstance(other, Suppress):
4386 if other not in self.ignoreExprs:
4387 super().ignore(other)
4388 for e in self.exprs:
4389 e.ignore(self.ignoreExprs[-1])
4390 else:
4391 super().ignore(other)
4392 for e in self.exprs:
4393 e.ignore(self.ignoreExprs[-1])
4394 return self
4396 def _generateDefaultName(self) -> str:
4397 return f"{type(self).__name__}:({self.exprs})"
4399 def streamline(self) -> ParserElement:
4400 if self.streamlined:
4401 return self
4403 super().streamline()
4405 for e in self.exprs:
4406 e.streamline()
4408 # collapse nested :class:`And`'s of the form ``And(And(And(a, b), c), d)`` to ``And(a, b, c, d)``
4409 # but only if there are no parse actions or resultsNames on the nested And's
4410 # (likewise for :class:`Or`'s and :class:`MatchFirst`'s)
4411 if len(self.exprs) == 2:
4412 first, second = self.exprs
4413 if (
4414 isinstance(first, self.__class__)
4415 and not first.parseAction
4416 and first.resultsName is None
4417 and not first.debug
4418 ):
4419 self.exprs[:] = (*first.exprs, second)
4420 self._defaultName = None
4421 self._may_return_empty |= first.mayReturnEmpty
4422 self.mayIndexError |= first.mayIndexError
4424 last = self.exprs[-1]
4425 if (
4426 isinstance(last, self.__class__)
4427 and not last.parseAction
4428 and last.resultsName is None
4429 and not last.debug
4430 ):
4431 self.exprs[-1:] = last.exprs
4432 self._defaultName = None
4433 self._may_return_empty |= last.mayReturnEmpty
4434 self.mayIndexError |= last.mayIndexError
4436 self.errmsg = f"Expected {self}"
4438 return self
4440 def validate(self, validateTrace=None) -> None:
4441 warnings.warn(
4442 "ParserElement.validate() is deprecated, and should not be used to check for left recursion",
4443 PyparsingDeprecationWarning,
4444 stacklevel=2,
4445 )
4446 tmp = (validateTrace if validateTrace is not None else [])[:] + [self]
4447 for e in self.exprs:
4448 e.validate(tmp)
4449 self._checkRecursion([])
4451 def copy(self) -> ParserElement:
4452 """
4453 Returns a copy of this expression.
4455 Generally only used internally by pyparsing.
4456 """
4457 ret = super().copy()
4458 ret = typing.cast(ParseExpression, ret)
4459 ret.exprs = [e.copy() for e in self.exprs]
4460 return ret
4462 def _setResultsName(self, name, list_all_matches=False) -> ParserElement:
4463 if not (
4464 __diag__.warn_ungrouped_named_tokens_in_collection
4465 and Diagnostics.warn_ungrouped_named_tokens_in_collection
4466 not in self.suppress_warnings_
4467 ):
4468 return super()._setResultsName(name, list_all_matches)
4470 for e in self.exprs:
4471 if (
4472 isinstance(e, ParserElement)
4473 and e.resultsName
4474 and (
4475 Diagnostics.warn_ungrouped_named_tokens_in_collection
4476 not in e.suppress_warnings_
4477 )
4478 ):
4479 warning = (
4480 "warn_ungrouped_named_tokens_in_collection:"
4481 f" setting results name {name!r} on {type(self).__name__} expression"
4482 f" collides with {e.resultsName!r} on contained expression"
4483 )
4484 warnings.warn(warning, PyparsingDiagnosticWarning, stacklevel=3)
4485 break
4487 return super()._setResultsName(name, list_all_matches)
4489 # Compatibility synonyms
4490 # fmt: off
4491 leaveWhitespace = replaced_by_pep8("leaveWhitespace", leave_whitespace)
4492 ignoreWhitespace = replaced_by_pep8("ignoreWhitespace", ignore_whitespace)
4493 # fmt: on
4496class And(ParseExpression):
4497 """
4498 Requires all given :class:`ParserElement` s to be found in the given order.
4499 Expressions may be separated by whitespace.
4500 May be constructed using the ``'+'`` operator.
4501 May also be constructed using the ``'-'`` operator, which will
4502 suppress backtracking.
4504 Example:
4506 .. testcode::
4508 integer = Word(nums)
4509 name_expr = Word(alphas)[1, ...]
4511 expr = And([integer("id"), name_expr("name"), integer("age")])
4512 # more easily written as:
4513 expr = integer("id") + name_expr("name") + integer("age")
4514 """
4516 class _ErrorStop(Empty):
4517 def __init__(self, *args, **kwargs) -> None:
4518 super().__init__(*args, **kwargs)
4519 self.leave_whitespace()
4521 def _generateDefaultName(self) -> str:
4522 return "-"
4524 def __init__(
4525 self,
4526 exprs_arg: typing.Iterable[Union[ParserElement, str]],
4527 savelist: bool = True,
4528 ) -> None:
4529 # instantiate exprs as a list, converting strs to ParserElements
4530 exprs: list[ParserElement] = [
4531 self._literalStringClass(e) if isinstance(e, str) else e for e in exprs_arg
4532 ]
4534 # convert any Ellipsis elements to SkipTo
4535 if Ellipsis in exprs:
4537 # Ellipsis cannot be the last element
4538 if exprs[-1] is Ellipsis:
4539 raise Exception("cannot construct And with sequence ending in ...")
4541 tmp: list[ParserElement] = []
4542 for cur_expr, next_expr in zip(exprs, exprs[1:]):
4543 if cur_expr is Ellipsis:
4544 tmp.append(SkipTo(next_expr)("_skipped*"))
4545 else:
4546 tmp.append(cur_expr)
4548 exprs[:-1] = tmp
4550 super().__init__(exprs, savelist)
4551 if self.exprs:
4552 self._may_return_empty = all(e.mayReturnEmpty for e in self.exprs)
4553 if not isinstance(self.exprs[0], White):
4554 self.set_whitespace_chars(
4555 self.exprs[0].whiteChars,
4556 copy_defaults=self.exprs[0].copyDefaultWhiteChars,
4557 )
4558 self.skipWhitespace = self.exprs[0].skipWhitespace
4559 else:
4560 self.skipWhitespace = False
4561 else:
4562 self._may_return_empty = True
4563 self.callPreparse = True
4565 def streamline(self) -> ParserElement:
4566 """
4567 Collapse `And` expressions like `And(And(And(A, B), C), D)`
4568 to `And(A, B, C, D)`.
4570 .. doctest::
4572 >>> expr = Word("A") + Word("B") + Word("C") + Word("D")
4573 >>> # Using '+' operator creates nested And expression
4574 >>> expr
4575 {{{W:(A) W:(B)} W:(C)} W:(D)}
4576 >>> # streamline simplifies to a single And with multiple expressions
4577 >>> expr.streamline()
4578 {W:(A) W:(B) W:(C) W:(D)}
4580 Guards against collapsing out expressions that have special features,
4581 such as results names or parse actions.
4583 Resolves pending Skip commands defined using `...` terms.
4584 """
4585 # collapse any _PendingSkip's
4586 if self.exprs and any(
4587 isinstance(e, ParseExpression)
4588 and e.exprs
4589 and isinstance(e.exprs[-1], _PendingSkip)
4590 for e in self.exprs[:-1]
4591 ):
4592 deleted_expr_marker = NoMatch()
4593 for i, e in enumerate(self.exprs[:-1]):
4594 if e is deleted_expr_marker:
4595 continue
4596 if (
4597 isinstance(e, ParseExpression)
4598 and e.exprs
4599 and isinstance(e.exprs[-1], _PendingSkip)
4600 ):
4601 e.exprs[-1] = e.exprs[-1] + self.exprs[i + 1]
4602 self.exprs[i + 1] = deleted_expr_marker
4603 self.exprs = [e for e in self.exprs if e is not deleted_expr_marker]
4605 super().streamline()
4607 # link any IndentedBlocks to the prior expression
4608 prev: ParserElement
4609 cur: ParserElement
4610 for prev, cur in zip(self.exprs, self.exprs[1:]):
4611 # traverse cur or any first embedded expr of cur looking for an IndentedBlock
4612 # (but watch out for recursive grammar)
4613 seen = set()
4614 while True:
4615 if id(cur) in seen:
4616 break
4617 seen.add(id(cur))
4618 if isinstance(cur, IndentedBlock):
4619 prev.add_parse_action(
4620 lambda s, l, t, cur_=cur: setattr(
4621 cur_, "parent_anchor", col(l, s)
4622 )
4623 )
4624 break
4625 subs = cur.recurse()
4626 next_first = next(iter(subs), None)
4627 if next_first is None:
4628 break
4629 cur = typing.cast(ParserElement, next_first)
4631 self._may_return_empty = all(e.mayReturnEmpty for e in self.exprs)
4632 return self
4634 def parseImpl(self, instring, loc, do_actions=True):
4636 # if no exprs defined, assume we contain a single Empty
4637 # (consistent with behavior of `all([])` returning True)
4638 exprs = iter(self.exprs or (Empty(),))
4640 # pass False as callPreParse arg to _parse for first element, since we already
4641 # pre-parsed the string as part of our And pre-parsing
4642 loc, resultlist = next(exprs)._parse(
4643 instring, loc, do_actions, callPreParse=False
4644 )
4646 # iterate over remaining expressions
4647 raise_syntax_error_immediately = False
4648 for e in exprs:
4649 # if isinstance(e, And._ErrorStop):
4650 if type(e) is And._ErrorStop:
4651 raise_syntax_error_immediately = True
4652 continue
4654 if raise_syntax_error_immediately:
4655 try:
4656 loc, exprtokens = e._parse(instring, loc, do_actions)
4657 except ParseSyntaxException:
4658 raise
4659 except ParseBaseException as pe:
4660 pe.__traceback__ = None
4661 raise ParseSyntaxException._from_exception(pe)
4662 except IndexError:
4663 raise ParseSyntaxException(
4664 instring, len(instring), self.errmsg, self
4665 )
4666 else:
4667 loc, exprtokens = e._parse(instring, loc, do_actions)
4668 resultlist += exprtokens
4669 return loc, resultlist
4671 def __iadd__(self, other):
4672 if isinstance(other, str_type):
4673 other = self._literalStringClass(other)
4674 if not isinstance(other, ParserElement):
4675 return NotImplemented
4676 return self.append(other) # And([self, other])
4678 def _checkRecursion(self, parseElementList):
4679 subRecCheckList = parseElementList[:] + [self]
4680 for e in self.exprs:
4681 e._checkRecursion(subRecCheckList)
4682 if not e.mayReturnEmpty:
4683 break
4685 def _generateDefaultName(self) -> str:
4686 inner = " ".join(str(e) for e in self.exprs)
4687 # strip off redundant inner {}'s
4688 while len(inner) > 1 and inner[0 :: len(inner) - 1] == "{}":
4689 inner = inner[1:-1]
4690 return f"{{{inner}}}"
4693class Or(ParseExpression):
4694 """Requires that at least one :class:`ParserElement` is found. If
4695 two expressions match, the expression that matches the longest
4696 string will be used. May be constructed using the ``'^'``
4697 operator.
4699 Example:
4701 .. testcode::
4703 # construct Or using '^' operator
4705 number = Word(nums) ^ Combine(Word(nums) + '.' + Word(nums))
4706 print(number.search_string("123 3.1416 789"))
4708 prints:
4710 .. testoutput::
4712 [['123'], ['3.1416'], ['789']]
4713 """
4715 def __init__(
4716 self, exprs: typing.Iterable[ParserElement], savelist: bool = False
4717 ) -> None:
4718 super().__init__(exprs, savelist)
4719 if self.exprs:
4720 self._may_return_empty = any(e.mayReturnEmpty for e in self.exprs)
4721 self.skipWhitespace = all(e.skipWhitespace for e in self.exprs)
4722 else:
4723 self._may_return_empty = True
4725 def streamline(self) -> ParserElement:
4726 super().streamline()
4727 if self.exprs:
4728 self._may_return_empty = any(e.mayReturnEmpty for e in self.exprs)
4729 self.saveAsList = any(e.saveAsList for e in self.exprs)
4730 self.skipWhitespace = all(
4731 e.skipWhitespace and not isinstance(e, White) for e in self.exprs
4732 )
4733 else:
4734 self.saveAsList = False
4735 return self
4737 def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType:
4738 maxExcLoc = -1
4739 maxException = None
4740 matches: list[tuple[int, ParserElement]] = []
4741 fatals: list[ParseFatalException] = []
4742 if all(e.callPreparse for e in self.exprs):
4743 loc = self.preParse(instring, loc)
4744 for e in self.exprs:
4745 try:
4746 loc2 = e.try_parse(instring, loc, raise_fatal=True)
4747 except ParseFatalException as pfe:
4748 pfe.__traceback__ = None
4749 pfe.parser_element = e
4750 fatals.append(pfe)
4751 maxException = None
4752 maxExcLoc = -1
4753 except ParseException as err:
4754 if not fatals:
4755 err.__traceback__ = None
4756 if err.loc > maxExcLoc:
4757 maxException = err
4758 maxExcLoc = err.loc
4759 except IndexError:
4760 if len(instring) > maxExcLoc:
4761 maxException = ParseException(
4762 instring, len(instring), e.errmsg, self
4763 )
4764 maxExcLoc = len(instring)
4765 else:
4766 # save match among all matches, to retry longest to shortest
4767 matches.append((loc2, e))
4769 if matches:
4770 # re-evaluate all matches in descending order of length of match, in case attached actions
4771 # might change whether or how much they match of the input.
4772 matches.sort(key=itemgetter(0), reverse=True)
4774 if not do_actions:
4775 # no further conditions or parse actions to change the selection of
4776 # alternative, so the first match will be the best match
4777 best_expr = matches[0][1]
4778 return best_expr._parse(instring, loc, do_actions)
4780 longest: tuple[int, typing.Optional[ParseResults]] = -1, None
4781 for loc1, expr1 in matches:
4782 if loc1 <= longest[0]:
4783 # already have a longer match than this one will deliver, we are done
4784 return longest
4786 try:
4787 loc2, toks = expr1._parse(instring, loc, do_actions)
4788 except ParseException as err:
4789 err.__traceback__ = None
4790 if err.loc > maxExcLoc:
4791 maxException = err
4792 maxExcLoc = err.loc
4793 else:
4794 if loc2 >= loc1:
4795 return loc2, toks
4796 # didn't match as much as before
4797 elif loc2 > longest[0]:
4798 longest = loc2, toks
4800 if longest != (-1, None):
4801 return longest
4803 if fatals:
4804 if len(fatals) > 1:
4805 fatals.sort(key=lambda e: -e.loc)
4806 if fatals[0].loc == fatals[1].loc:
4807 fatals.sort(key=lambda e: (-e.loc, -len(str(e.parser_element))))
4808 max_fatal = fatals[0]
4809 raise max_fatal
4811 if maxException is not None:
4812 # infer from this check that all alternatives failed at the current position
4813 # so emit this collective error message instead of any single error message
4814 parse_start_loc = self.preParse(instring, loc)
4815 if maxExcLoc == parse_start_loc:
4816 maxException.msg = self.errmsg or ""
4817 raise maxException
4819 raise ParseException(instring, loc, "no defined alternatives to match", self)
4821 def __ixor__(self, other):
4822 if isinstance(other, str_type):
4823 other = self._literalStringClass(other)
4824 if not isinstance(other, ParserElement):
4825 return NotImplemented
4826 return self.append(other) # Or([self, other])
4828 def _generateDefaultName(self) -> str:
4829 return f"{{{' ^ '.join(str(e) for e in self.exprs)}}}"
4831 def _setResultsName(self, name, list_all_matches=False) -> ParserElement:
4832 if (
4833 __diag__.warn_multiple_tokens_in_named_alternation
4834 and Diagnostics.warn_multiple_tokens_in_named_alternation
4835 not in self.suppress_warnings_
4836 ):
4837 if any(
4838 isinstance(e, And)
4839 and Diagnostics.warn_multiple_tokens_in_named_alternation
4840 not in e.suppress_warnings_
4841 for e in self.exprs
4842 ):
4843 warning = (
4844 "warn_multiple_tokens_in_named_alternation:"
4845 f" setting results name {name!r} on {type(self).__name__} expression"
4846 " will return a list of all parsed tokens in an And alternative,"
4847 " in prior versions only the first token was returned; enclose"
4848 " contained argument in Group"
4849 )
4850 warnings.warn(warning, PyparsingDiagnosticWarning, stacklevel=3)
4852 return super()._setResultsName(name, list_all_matches)
4855class MatchFirst(ParseExpression):
4856 """Requires that at least one :class:`ParserElement` is found. If
4857 more than one expression matches, the first one listed is the one that will
4858 match. May be constructed using the ``'|'`` operator.
4860 Example: Construct MatchFirst using '|' operator
4862 .. doctest::
4864 # watch the order of expressions to match
4865 >>> number = Word(nums) | Combine(Word(nums) + '.' + Word(nums))
4866 >>> print(number.search_string("123 3.1416 789")) # Fail!
4867 [['123'], ['3'], ['1416'], ['789']]
4869 # put more selective expression first
4870 >>> number = Combine(Word(nums) + '.' + Word(nums)) | Word(nums)
4871 >>> print(number.search_string("123 3.1416 789")) # Better
4872 [['123'], ['3.1416'], ['789']]
4873 """
4875 def __init__(
4876 self, exprs: typing.Iterable[ParserElement], savelist: bool = False
4877 ) -> None:
4878 super().__init__(exprs, savelist)
4879 if self.exprs:
4880 self._may_return_empty = any(e.mayReturnEmpty for e in self.exprs)
4881 self.skipWhitespace = all(e.skipWhitespace for e in self.exprs)
4882 else:
4883 self._may_return_empty = True
4885 def streamline(self) -> ParserElement:
4886 if self.streamlined:
4887 return self
4889 super().streamline()
4890 if self.exprs:
4891 self.saveAsList = any(e.saveAsList for e in self.exprs)
4892 self._may_return_empty = any(e.mayReturnEmpty for e in self.exprs)
4893 self.skipWhitespace = all(
4894 e.skipWhitespace and not isinstance(e, White) for e in self.exprs
4895 )
4896 else:
4897 self.saveAsList = False
4898 self._may_return_empty = True
4899 return self
4901 def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType:
4902 maxExcLoc = -1
4903 maxException = None
4905 for e in self.exprs:
4906 try:
4907 return e._parse(instring, loc, do_actions)
4908 except ParseFatalException as pfe:
4909 pfe.__traceback__ = None
4910 pfe.parser_element = e
4911 raise
4912 except ParseException as err:
4913 if err.loc > maxExcLoc:
4914 maxException = err
4915 maxExcLoc = err.loc
4916 except IndexError:
4917 if len(instring) > maxExcLoc:
4918 maxException = ParseException(
4919 instring, len(instring), e.errmsg, self
4920 )
4921 maxExcLoc = len(instring)
4923 if maxException is not None:
4924 # infer from this check that all alternatives failed at the current position
4925 # so emit this collective error message instead of any individual error message
4926 parse_start_loc = self.preParse(instring, loc)
4927 if maxExcLoc == parse_start_loc:
4928 maxException.msg = self.errmsg or ""
4929 raise maxException
4931 raise ParseException(instring, loc, "no defined alternatives to match", self)
4933 def __ior__(self, other):
4934 if isinstance(other, str_type):
4935 other = self._literalStringClass(other)
4936 if not isinstance(other, ParserElement):
4937 return NotImplemented
4938 return self.append(other) # MatchFirst([self, other])
4940 def _generateDefaultName(self) -> str:
4941 return f"{{{' | '.join(str(e) for e in self.exprs)}}}"
4943 def _setResultsName(self, name, list_all_matches=False) -> ParserElement:
4944 if (
4945 __diag__.warn_multiple_tokens_in_named_alternation
4946 and Diagnostics.warn_multiple_tokens_in_named_alternation
4947 not in self.suppress_warnings_
4948 ):
4949 if any(
4950 isinstance(e, And)
4951 and Diagnostics.warn_multiple_tokens_in_named_alternation
4952 not in e.suppress_warnings_
4953 for e in self.exprs
4954 ):
4955 warning = (
4956 "warn_multiple_tokens_in_named_alternation:"
4957 f" setting results name {name!r} on {type(self).__name__} expression"
4958 " will return a list of all parsed tokens in an And alternative,"
4959 " in prior versions only the first token was returned; enclose"
4960 " contained argument in Group"
4961 )
4962 warnings.warn(warning, PyparsingDiagnosticWarning, stacklevel=3)
4964 return super()._setResultsName(name, list_all_matches)
4967class Each(ParseExpression):
4968 """Requires all given :class:`ParserElement` s to be found, but in
4969 any order. Expressions may be separated by whitespace.
4971 May be constructed using the ``'&'`` operator.
4973 Example:
4975 .. testcode::
4977 color = one_of("RED ORANGE YELLOW GREEN BLUE PURPLE BLACK WHITE BROWN")
4978 shape_type = one_of("SQUARE CIRCLE TRIANGLE STAR HEXAGON OCTAGON")
4979 integer = Word(nums)
4980 shape_attr = "shape:" + shape_type("shape")
4981 posn_attr = "posn:" + Group(integer("x") + ',' + integer("y"))("posn")
4982 color_attr = "color:" + color("color")
4983 size_attr = "size:" + integer("size")
4985 # use Each (using operator '&') to accept attributes in any order
4986 # (shape and posn are required, color and size are optional)
4987 shape_spec = shape_attr & posn_attr & Opt(color_attr) & Opt(size_attr)
4989 shape_spec.run_tests('''
4990 shape: SQUARE color: BLACK posn: 100, 120
4991 shape: CIRCLE size: 50 color: BLUE posn: 50,80
4992 color:GREEN size:20 shape:TRIANGLE posn:20,40
4993 '''
4994 )
4996 prints:
4998 .. testoutput::
4999 :options: +NORMALIZE_WHITESPACE
5002 shape: SQUARE color: BLACK posn: 100, 120
5003 ['shape:', 'SQUARE', 'color:', 'BLACK', 'posn:', ['100', ',', '120']]
5004 - color: 'BLACK'
5005 - posn: ['100', ',', '120']
5006 - x: '100'
5007 - y: '120'
5008 - shape: 'SQUARE'
5009 ...
5011 shape: CIRCLE size: 50 color: BLUE posn: 50,80
5012 ['shape:', 'CIRCLE', 'size:', '50', 'color:', 'BLUE',
5013 'posn:', ['50', ',', '80']]
5014 - color: 'BLUE'
5015 - posn: ['50', ',', '80']
5016 - x: '50'
5017 - y: '80'
5018 - shape: 'CIRCLE'
5019 - size: '50'
5020 ...
5022 color:GREEN size:20 shape:TRIANGLE posn:20,40
5023 ['color:', 'GREEN', 'size:', '20', 'shape:', 'TRIANGLE',
5024 'posn:', ['20', ',', '40']]
5025 - color: 'GREEN'
5026 - posn: ['20', ',', '40']
5027 - x: '20'
5028 - y: '40'
5029 - shape: 'TRIANGLE'
5030 - size: '20'
5031 ...
5032 """
5034 def __init__(
5035 self, exprs: typing.Iterable[ParserElement], savelist: bool = True
5036 ) -> None:
5037 super().__init__(exprs, savelist)
5038 if self.exprs:
5039 self._may_return_empty = all(e.mayReturnEmpty for e in self.exprs)
5040 else:
5041 self._may_return_empty = True
5042 self.skipWhitespace = True
5043 self.initExprGroups = True
5044 self.saveAsList = True
5046 def __iand__(self, other):
5047 if isinstance(other, str_type):
5048 other = self._literalStringClass(other)
5049 if not isinstance(other, ParserElement):
5050 return NotImplemented
5051 return self.append(other) # Each([self, other])
5053 def streamline(self) -> ParserElement:
5054 super().streamline()
5055 if self.exprs:
5056 self._may_return_empty = all(e.mayReturnEmpty for e in self.exprs)
5057 else:
5058 self._may_return_empty = True
5059 return self
5061 def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType:
5062 if self.initExprGroups:
5063 self.opt1map = dict(
5064 (id(e.expr), e) for e in self.exprs if isinstance(e, Opt)
5065 )
5066 opt1 = [e.expr for e in self.exprs if isinstance(e, Opt)]
5067 opt2 = [
5068 e
5069 for e in self.exprs
5070 if e.mayReturnEmpty and not isinstance(e, (Opt, Regex, ZeroOrMore))
5071 ]
5072 self.optionals = opt1 + opt2
5073 self.multioptionals = [
5074 e.expr.set_results_name(e.resultsName, list_all_matches=True)
5075 for e in self.exprs
5076 if isinstance(e, _MultipleMatch)
5077 ]
5078 self.multirequired = [
5079 e.expr.set_results_name(e.resultsName, list_all_matches=True)
5080 for e in self.exprs
5081 if isinstance(e, OneOrMore)
5082 ]
5083 self.required = [
5084 e for e in self.exprs if not isinstance(e, (Opt, ZeroOrMore, OneOrMore))
5085 ]
5086 self.required += self.multirequired
5087 self.initExprGroups = False
5089 tmpLoc = loc
5090 tmpReqd = self.required[:]
5091 tmpOpt = self.optionals[:]
5092 multis = self.multioptionals[:]
5093 matchOrder: list[ParserElement] = []
5095 keepMatching = True
5096 failed: list[ParserElement] = []
5097 fatals: list[ParseFatalException] = []
5098 while keepMatching:
5099 tmpExprs = tmpReqd + tmpOpt + multis
5100 failed.clear()
5101 fatals.clear()
5102 for e in tmpExprs:
5103 try:
5104 tmpLoc = e.try_parse(instring, tmpLoc, raise_fatal=True)
5105 except ParseFatalException as pfe:
5106 pfe.__traceback__ = None
5107 pfe.parser_element = e
5108 fatals.append(pfe)
5109 failed.append(e)
5110 except ParseException:
5111 failed.append(e)
5112 else:
5113 matchOrder.append(self.opt1map.get(id(e), e))
5114 if e in tmpReqd:
5115 tmpReqd.remove(e)
5116 elif e in tmpOpt:
5117 tmpOpt.remove(e)
5118 if len(failed) == len(tmpExprs):
5119 keepMatching = False
5121 # look for any ParseFatalExceptions
5122 if fatals:
5123 if len(fatals) > 1:
5124 fatals.sort(key=lambda e: -e.loc)
5125 if fatals[0].loc == fatals[1].loc:
5126 fatals.sort(key=lambda e: (-e.loc, -len(str(e.parser_element))))
5127 max_fatal = fatals[0]
5128 raise max_fatal
5130 if tmpReqd:
5131 missing = ", ".join([str(e) for e in tmpReqd])
5132 raise ParseException(
5133 instring,
5134 loc,
5135 f"Missing one or more required elements ({missing})",
5136 )
5138 # add any unmatched Opts, in case they have default values defined
5139 matchOrder += [e for e in self.exprs if isinstance(e, Opt) and e.expr in tmpOpt]
5141 total_results = ParseResults([])
5142 for e in matchOrder:
5143 loc, results = e._parse(instring, loc, do_actions)
5144 total_results += results
5146 return loc, total_results
5148 def _generateDefaultName(self) -> str:
5149 return f"{{{' & '.join(str(e) for e in self.exprs)}}}"
5152class ParseElementEnhance(ParserElement):
5153 """Abstract subclass of :class:`ParserElement`, for combining and
5154 post-processing parsed tokens.
5155 """
5157 def __init__(self, expr: Union[ParserElement, str], savelist: bool = False) -> None:
5158 super().__init__(savelist)
5159 if isinstance(expr, str_type):
5160 expr_str = typing.cast(str, expr)
5161 if issubclass(self._literalStringClass, Token):
5162 expr = self._literalStringClass(expr_str) # type: ignore[call-arg]
5163 elif issubclass(type(self), self._literalStringClass):
5164 expr = Literal(expr_str)
5165 else:
5166 expr = self._literalStringClass(Literal(expr_str)) # type: ignore[assignment, call-arg]
5167 expr = typing.cast(ParserElement, expr)
5168 self.expr = expr
5169 if expr is not None:
5170 self.mayIndexError = expr.mayIndexError
5171 self._may_return_empty = expr.mayReturnEmpty
5172 self.set_whitespace_chars(
5173 expr.whiteChars, copy_defaults=expr.copyDefaultWhiteChars
5174 )
5175 self.skipWhitespace = expr.skipWhitespace
5176 self.saveAsList = expr.saveAsList
5177 self.callPreparse = expr.callPreparse
5178 self.ignoreExprs.extend(expr.ignoreExprs)
5180 def recurse(self) -> list[ParserElement]:
5181 return [self.expr] if self.expr is not None else []
5183 def parseImpl(self, instring, loc, do_actions=True):
5184 if self.expr is None:
5185 raise ParseException(instring, loc, "No expression defined", self)
5187 try:
5188 return self.expr._parse(instring, loc, do_actions, callPreParse=False)
5189 except ParseSyntaxException:
5190 raise
5191 except ParseBaseException as pbe:
5192 pbe.pstr = pbe.pstr or instring
5193 pbe.loc = pbe.loc or loc
5194 pbe.parser_element = pbe.parser_element or self
5195 if not isinstance(self, Forward) and self.customName is not None:
5196 if self.errmsg:
5197 pbe.msg = self.errmsg
5198 raise
5200 def leave_whitespace(self, recursive: bool = True) -> ParserElement:
5201 """
5202 Extends ``leave_whitespace`` defined in base class, and also invokes ``leave_whitespace`` on
5203 the contained expression.
5204 """
5205 super().leave_whitespace(recursive)
5207 if recursive:
5208 if self.expr is not None:
5209 self.expr = self.expr.copy()
5210 self.expr.leave_whitespace(recursive)
5211 return self
5213 def ignore_whitespace(self, recursive: bool = True) -> ParserElement:
5214 """
5215 Extends ``ignore_whitespace`` defined in base class, and also invokes ``ignore_whitespace`` on
5216 the contained expression.
5217 """
5218 super().ignore_whitespace(recursive)
5220 if recursive:
5221 if self.expr is not None:
5222 self.expr = self.expr.copy()
5223 self.expr.ignore_whitespace(recursive)
5224 return self
5226 def ignore(self, other) -> ParserElement:
5227 """
5228 Define expression to be ignored (e.g., comments) while doing pattern
5229 matching; may be called repeatedly, to define multiple comment or other
5230 ignorable patterns.
5231 """
5232 if not isinstance(other, Suppress) or other not in self.ignoreExprs:
5233 super().ignore(other)
5234 if self.expr is not None:
5235 self.expr.ignore(self.ignoreExprs[-1])
5237 return self
5239 def streamline(self) -> ParserElement:
5240 super().streamline()
5241 if self.expr is not None:
5242 self.expr.streamline()
5243 return self
5245 def _checkRecursion(self, parseElementList):
5246 if self in parseElementList:
5247 raise RecursiveGrammarException(parseElementList + [self])
5248 subRecCheckList = parseElementList[:] + [self]
5249 if self.expr is not None:
5250 self.expr._checkRecursion(subRecCheckList)
5252 def validate(self, validateTrace=None) -> None:
5253 warnings.warn(
5254 "ParserElement.validate() is deprecated, and should not be used to check for left recursion",
5255 PyparsingDeprecationWarning,
5256 stacklevel=2,
5257 )
5258 if validateTrace is None:
5259 validateTrace = []
5260 tmp = validateTrace[:] + [self]
5261 if self.expr is not None:
5262 self.expr.validate(tmp)
5263 self._checkRecursion([])
5265 def _generateDefaultName(self) -> str:
5266 return f"{type(self).__name__}:({self.expr})"
5268 # Compatibility synonyms
5269 # fmt: off
5270 leaveWhitespace = replaced_by_pep8("leaveWhitespace", leave_whitespace)
5271 ignoreWhitespace = replaced_by_pep8("ignoreWhitespace", ignore_whitespace)
5272 # fmt: on
5275class IndentedBlock(ParseElementEnhance):
5276 """
5277 Expression to match one or more expressions at a given indentation level.
5278 Useful for parsing text where structure is implied by indentation (like Python source code).
5280 Example:
5282 .. testcode::
5284 '''
5285 BNF:
5286 statement ::= assignment_stmt | if_stmt
5287 assignment_stmt ::= identifier '=' rvalue
5288 rvalue ::= identifier | integer
5289 if_stmt ::= 'if' bool_condition block
5290 block ::= ([indent] statement)...
5291 identifier ::= [A..Za..z]
5292 integer ::= [0..9]...
5293 bool_condition ::= 'TRUE' | 'FALSE'
5294 '''
5296 IF, TRUE, FALSE = Keyword.using_each("IF TRUE FALSE".split())
5298 statement = Forward()
5299 identifier = Char(alphas)
5300 integer = Word(nums).add_parse_action(lambda t: int(t[0]))
5301 rvalue = identifier | integer
5302 assignment_stmt = identifier + "=" + rvalue
5304 if_stmt = IF + (TRUE | FALSE) + IndentedBlock(statement)
5306 statement <<= Group(assignment_stmt | if_stmt)
5308 result = if_stmt.parse_string('''
5309 IF TRUE
5310 a = 1000
5311 b = 2000
5312 IF FALSE
5313 z = 100
5314 ''')
5315 print(result.dump())
5317 .. testoutput::
5319 ['IF', 'TRUE', [['a', '=', 1000], ['b', '=', 2000], ['IF', 'FALSE', [['z', '=', 100]]]]]
5320 [0]:
5321 IF
5322 [1]:
5323 TRUE
5324 [2]:
5325 [['a', '=', 1000], ['b', '=', 2000], ['IF', 'FALSE', [['z', '=', 100]]]]
5326 [0]:
5327 ['a', '=', 1000]
5328 [1]:
5329 ['b', '=', 2000]
5330 [2]:
5331 ['IF', 'FALSE', [['z', '=', 100]]]
5332 [0]:
5333 IF
5334 [1]:
5335 FALSE
5336 [2]:
5337 [['z', '=', 100]]
5338 [0]:
5339 ['z', '=', 100]
5340 """
5342 class _Indent(Empty):
5343 def __init__(self, ref_col: int) -> None:
5344 super().__init__()
5345 self.errmsg = f"expected indent at column {ref_col}"
5346 self.add_condition(lambda s, l, t: col(l, s) == ref_col)
5348 class _IndentGreater(Empty):
5349 def __init__(self, ref_col: int) -> None:
5350 super().__init__()
5351 self.errmsg = f"expected indent at column greater than {ref_col}"
5352 self.add_condition(lambda s, l, t: col(l, s) > ref_col)
5354 def __init__(
5355 self, expr: ParserElement, *, recursive: bool = False, grouped: bool = True
5356 ) -> None:
5357 super().__init__(expr, savelist=True)
5358 # if recursive:
5359 # raise NotImplementedError("IndentedBlock with recursive is not implemented")
5360 self._recursive = recursive
5361 self._grouped = grouped
5362 self.parent_anchor = 1
5364 def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType:
5365 # advance parse position to non-whitespace by using an Empty()
5366 # this should be the column to be used for all subsequent indented lines
5367 anchor_loc = Empty().preParse(instring, loc)
5369 # see if self.expr matches at the current location - if not it will raise an exception
5370 # and no further work is necessary
5371 self.expr.try_parse(instring, anchor_loc, do_actions=do_actions)
5373 indent_col = col(anchor_loc, instring)
5374 peer_detect_expr = self._Indent(indent_col)
5376 inner_expr = Empty() + peer_detect_expr + self.expr
5377 if self._recursive:
5378 sub_indent = self._IndentGreater(indent_col)
5379 nested_block = IndentedBlock(
5380 self.expr, recursive=self._recursive, grouped=self._grouped
5381 )
5382 nested_block.set_debug(self.debug)
5383 nested_block.parent_anchor = indent_col
5384 inner_expr += Opt(sub_indent + nested_block)
5386 inner_expr.set_name(f"inner {hex(id(inner_expr))[-4:].upper()}@{indent_col}")
5387 block = OneOrMore(inner_expr)
5389 trailing_undent = self._Indent(self.parent_anchor) | StringEnd()
5391 if self._grouped:
5392 wrapper = Group
5393 else:
5394 wrapper = lambda expr: expr # type: ignore[misc, assignment]
5395 return (wrapper(block) + Optional(trailing_undent)).parseImpl(
5396 instring, anchor_loc, do_actions
5397 )
5400class AtStringStart(ParseElementEnhance):
5401 """Matches if expression matches at the beginning of the parse
5402 string::
5404 AtStringStart(Word(nums)).parse_string("123")
5405 # prints ["123"]
5407 AtStringStart(Word(nums)).parse_string(" 123")
5408 # raises ParseException
5409 """
5411 def __init__(self, expr: Union[ParserElement, str]) -> None:
5412 super().__init__(expr)
5413 self.callPreparse = False
5415 def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType:
5416 if loc != 0:
5417 raise ParseException(instring, loc, "not found at string start")
5418 return super().parseImpl(instring, loc, do_actions)
5421class AtLineStart(ParseElementEnhance):
5422 r"""Matches if an expression matches at the beginning of a line within
5423 the parse string
5425 Example:
5427 .. testcode::
5429 test = '''\
5430 BBB this line
5431 BBB and this line
5432 BBB but not this one
5433 A BBB and definitely not this one
5434 '''
5436 for t in (AtLineStart('BBB') + rest_of_line).search_string(test):
5437 print(t)
5439 prints:
5441 .. testoutput::
5443 ['BBB', ' this line']
5444 ['BBB', ' and this line']
5445 """
5447 def __init__(self, expr: Union[ParserElement, str]) -> None:
5448 super().__init__(expr)
5449 self.callPreparse = False
5451 def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType:
5452 if col(loc, instring) != 1:
5453 raise ParseException(instring, loc, "not found at line start")
5454 return super().parseImpl(instring, loc, do_actions)
5457class FollowedBy(ParseElementEnhance):
5458 """Lookahead matching of the given parse expression.
5459 ``FollowedBy`` does *not* advance the parsing position within
5460 the input string, it only verifies that the specified parse
5461 expression matches at the current position. ``FollowedBy``
5462 always returns a null token list. If any results names are defined
5463 in the lookahead expression, those *will* be returned for access by
5464 name.
5466 Example:
5468 .. testcode::
5470 # use FollowedBy to match a label only if it is followed by a ':'
5471 data_word = Word(alphas)
5472 label = data_word + FollowedBy(':')
5473 attr_expr = Group(
5474 label + Suppress(':')
5475 + OneOrMore(data_word, stop_on=label).set_parse_action(' '.join)
5476 )
5478 attr_expr[1, ...].parse_string(
5479 "shape: SQUARE color: BLACK posn: upper left").pprint()
5481 prints:
5483 .. testoutput::
5485 [['shape', 'SQUARE'], ['color', 'BLACK'], ['posn', 'upper left']]
5486 """
5488 def __init__(self, expr: Union[ParserElement, str]) -> None:
5489 super().__init__(expr)
5490 self._may_return_empty = True
5492 def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType:
5493 # by using self._expr.parse and deleting the contents of the returned ParseResults list
5494 # we keep any named results that were defined in the FollowedBy expression
5495 _, ret = self.expr._parse(instring, loc, do_actions=do_actions)
5496 del ret[:]
5498 return loc, ret
5501class PrecededBy(ParseElementEnhance):
5502 """Lookbehind matching of the given parse expression.
5503 ``PrecededBy`` does not advance the parsing position within the
5504 input string, it only verifies that the specified parse expression
5505 matches prior to the current position. ``PrecededBy`` always
5506 returns a null token list, but if a results name is defined on the
5507 given expression, it is returned.
5509 Parameters:
5511 - ``expr`` - expression that must match prior to the current parse
5512 location
5513 - ``retreat`` - (default= ``None``) - (int) maximum number of characters
5514 to lookbehind prior to the current parse location
5516 If the lookbehind expression is a string, :class:`Literal`,
5517 :class:`Keyword`, or a :class:`Word` or :class:`CharsNotIn`
5518 with a specified exact or maximum length, then the retreat
5519 parameter is not required. Otherwise, retreat must be specified to
5520 give a maximum number of characters to look back from
5521 the current parse position for a lookbehind match.
5523 Example:
5525 .. testcode::
5527 # VB-style variable names with type prefixes
5528 int_var = PrecededBy("#") + pyparsing_common.identifier
5529 str_var = PrecededBy("$") + pyparsing_common.identifier
5530 """
5532 def __init__(self, expr: Union[ParserElement, str], retreat: int = 0) -> None:
5533 super().__init__(expr)
5534 self.expr = self.expr().leave_whitespace()
5535 self._may_return_empty = True
5536 self.mayIndexError = False
5537 self.exact = False
5538 if isinstance(expr, str_type):
5539 expr = typing.cast(str, expr)
5540 retreat = len(expr)
5541 self.exact = True
5542 elif isinstance(expr, (Literal, Keyword)):
5543 retreat = expr.matchLen
5544 self.exact = True
5545 elif isinstance(expr, (Word, CharsNotIn)) and expr.maxLen != _MAX_INT:
5546 retreat = expr.maxLen
5547 self.exact = True
5548 elif isinstance(expr, PositionToken):
5549 retreat = 0
5550 self.exact = True
5551 self.retreat = retreat
5552 self.errmsg = f"not preceded by {expr}"
5553 self.skipWhitespace = False
5554 self.parseAction.append(lambda s, l, t: t.__delitem__(slice(None, None)))
5556 def parseImpl(self, instring, loc=0, do_actions=True) -> ParseImplReturnType:
5557 if self.exact:
5558 if loc < self.retreat:
5559 raise ParseException(instring, loc, self.errmsg, self)
5560 start = loc - self.retreat
5561 _, ret = self.expr._parse(instring, start)
5562 return loc, ret
5564 # retreat specified a maximum lookbehind window, iterate
5565 test_expr = self.expr + StringEnd()
5566 instring_slice = instring[max(0, loc - self.retreat) : loc]
5567 last_expr: ParseBaseException = ParseException(instring, loc, self.errmsg, self)
5569 for offset in range(1, min(loc, self.retreat + 1) + 1):
5570 try:
5571 # print('trying', offset, instring_slice, repr(instring_slice[loc - offset:]))
5572 _, ret = test_expr._parse(instring_slice, len(instring_slice) - offset)
5573 except ParseBaseException as pbe:
5574 last_expr = pbe
5575 else:
5576 break
5577 else:
5578 raise last_expr
5580 return loc, ret
5583class Located(ParseElementEnhance):
5584 """
5585 Decorates a returned token with its starting and ending
5586 locations in the input string.
5588 This helper adds the following results names:
5590 - ``locn_start`` - location where matched expression begins
5591 - ``locn_end`` - location where matched expression ends
5592 - ``value`` - the actual parsed results
5594 Be careful if the input text contains ``<TAB>`` characters, you
5595 may want to call :class:`ParserElement.parse_with_tabs`
5597 Example:
5599 .. testcode::
5601 wd = Word(alphas)
5602 for match in Located(wd).search_string("ljsdf123lksdjjf123lkkjj1222"):
5603 print(match)
5605 prints:
5607 .. testoutput::
5609 [0, ['ljsdf'], 5]
5610 [8, ['lksdjjf'], 15]
5611 [18, ['lkkjj'], 23]
5612 """
5614 def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType:
5615 # skip leading whitespace before capturing the start location, so
5616 # locn_start marks the start of the match and not preceding whitespace,
5617 # even when the wrapped expression delegates whitespace skipping to its
5618 # sub-expressions (e.g. And/MatchFirst). Issue #621.
5619 start = self.expr.preParse(instring, loc)
5620 loc, tokens = self.expr._parse(instring, start, do_actions, callPreParse=False)
5621 ret_tokens = ParseResults([start, tokens, loc])
5622 ret_tokens["locn_start"] = start
5623 ret_tokens["value"] = tokens
5624 ret_tokens["locn_end"] = loc
5625 if self.resultsName:
5626 # must return as a list, so that the name will be attached to the complete group
5627 return loc, [ret_tokens]
5628 else:
5629 return loc, ret_tokens
5632class NotAny(ParseElementEnhance):
5633 """
5634 Lookahead to disallow matching with the given parse expression.
5635 ``NotAny`` does *not* advance the parsing position within the
5636 input string, it only verifies that the specified parse expression
5637 does *not* match at the current position. Also, ``NotAny`` does
5638 *not* skip over leading whitespace. ``NotAny`` always returns
5639 a null token list. May be constructed using the ``'~'`` operator.
5641 Example:
5643 .. testcode::
5645 AND, OR, NOT = map(CaselessKeyword, "AND OR NOT".split())
5647 # take care not to mistake keywords for identifiers
5648 ident = ~(AND | OR | NOT) + Word(alphas)
5649 boolean_term = Opt(NOT) + ident
5651 # very crude boolean expression - to support parenthesis groups and
5652 # operation hierarchy, use infix_notation
5653 boolean_expr = boolean_term + ((AND | OR) + boolean_term)[...]
5655 # integers that are followed by "." are actually floats
5656 integer = Word(nums) + ~Char(".")
5657 """
5659 def __init__(self, expr: Union[ParserElement, str]) -> None:
5660 super().__init__(expr)
5661 # do NOT use self.leave_whitespace(), don't want to propagate to exprs
5662 # self.leave_whitespace()
5663 self.skipWhitespace = False
5665 self._may_return_empty = True
5666 self.errmsg = f"Found unwanted token, {self.expr}"
5668 def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType:
5669 if self.expr.can_parse_next(instring, loc, do_actions=do_actions):
5670 raise ParseException(instring, loc, self.errmsg, self)
5671 return loc, []
5673 def _generateDefaultName(self) -> str:
5674 return f"~{{{self.expr}}}"
5677class _MultipleMatch(ParseElementEnhance):
5678 def __init__(
5679 self,
5680 expr: Union[str, ParserElement],
5681 stop_on: typing.Optional[Union[ParserElement, str]] = None,
5682 max: typing.Optional[int] = None,
5683 **kwargs,
5684 ) -> None:
5685 stopOn: typing.Optional[Union[ParserElement, str]] = deprecate_argument(
5686 kwargs, "stopOn", None
5687 )
5689 super().__init__(expr)
5690 if max is not None and max <= 0:
5691 raise ValueError("max must be greater than 0")
5692 stopOn = stopOn or stop_on
5693 self.saveAsList = True
5694 self.max_count = max
5695 ender = stopOn
5696 if isinstance(ender, str_type):
5697 ender = self._literalStringClass(ender)
5698 self.stopOn(ender)
5700 def stop_on(self, ender) -> ParserElement:
5701 if isinstance(ender, str_type):
5702 ender = self._literalStringClass(ender)
5703 self.not_ender = ~ender if ender is not None else None
5704 return self
5706 stopOn = stop_on
5708 def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType:
5709 self_expr_parse = self.expr._parse
5710 self_skip_ignorables = self._skipIgnorables
5711 check_ender = False
5712 if self.not_ender is not None:
5713 try_not_ender = self.not_ender.try_parse
5714 check_ender = True
5716 # must be at least one (but first see if we are the stopOn sentinel;
5717 # if so, fail)
5718 if check_ender:
5719 try_not_ender(instring, loc)
5720 loc, tokens = self_expr_parse(instring, loc, do_actions)
5721 match_count = 1
5722 try:
5723 hasIgnoreExprs = not not self.ignoreExprs
5724 while self.max_count is None or match_count < self.max_count:
5725 if check_ender:
5726 try_not_ender(instring, loc)
5727 if hasIgnoreExprs:
5728 preloc = self_skip_ignorables(instring, loc)
5729 else:
5730 preloc = loc
5731 loc, tmptokens = self_expr_parse(instring, preloc, do_actions)
5732 tokens += tmptokens
5733 match_count += 1
5734 except (ParseException, IndexError):
5735 pass
5737 return loc, tokens
5739 def _setResultsName(self, name, list_all_matches=False) -> ParserElement:
5740 if (
5741 __diag__.warn_ungrouped_named_tokens_in_collection
5742 and Diagnostics.warn_ungrouped_named_tokens_in_collection
5743 not in self.suppress_warnings_
5744 ):
5745 for e in [self.expr] + self.expr.recurse():
5746 if (
5747 isinstance(e, ParserElement)
5748 and e.resultsName
5749 and (
5750 Diagnostics.warn_ungrouped_named_tokens_in_collection
5751 not in e.suppress_warnings_
5752 )
5753 ):
5754 warning = (
5755 "warn_ungrouped_named_tokens_in_collection:"
5756 f" setting results name {name!r} on {type(self).__name__} expression"
5757 f" collides with {e.resultsName!r} on contained expression"
5758 )
5759 warnings.warn(warning, PyparsingDiagnosticWarning, stacklevel=3)
5760 break
5762 return super()._setResultsName(name, list_all_matches)
5765class OneOrMore(_MultipleMatch):
5766 """
5767 Repetition of one or more of the given expression.
5769 Parameters:
5771 - ``expr`` - expression that must match one or more times
5772 - ``stop_on`` - (default= ``None``) - expression for a terminating sentinel
5773 (only required if the sentinel would ordinarily match the repetition
5774 expression)
5776 Example:
5778 .. doctest::
5780 >>> data_word = Word(alphas)
5781 >>> label = data_word + FollowedBy(':')
5782 >>> attr_expr = Group(
5783 ... label + Suppress(':')
5784 ... + OneOrMore(data_word).set_parse_action(' '.join))
5786 >>> text = "shape: SQUARE posn: upper left color: BLACK"
5788 # Fail! read 'posn' as data instead of next label
5789 >>> attr_expr[1, ...].parse_string(text).pprint()
5790 [['shape', 'SQUARE posn']]
5792 # use stop_on attribute for OneOrMore
5793 # to avoid reading label string as part of the data
5794 >>> attr_expr = Group(
5795 ... label + Suppress(':')
5796 ... + OneOrMore(
5797 ... data_word, stop_on=label).set_parse_action(' '.join))
5798 >>> OneOrMore(attr_expr).parse_string(text).pprint() # Better
5799 [['shape', 'SQUARE'], ['posn', 'upper left'], ['color', 'BLACK']]
5801 # could also be written as
5802 >>> (attr_expr * (1,)).parse_string(text).pprint()
5803 [['shape', 'SQUARE'], ['posn', 'upper left'], ['color', 'BLACK']]
5804 """
5806 def _generateDefaultName(self) -> str:
5807 return f"{{{self.expr}}}..."
5810class ZeroOrMore(_MultipleMatch):
5811 """
5812 Optional repetition of zero or more of the given expression.
5814 Parameters:
5816 - ``expr`` - expression that must match zero or more times
5817 - ``stop_on`` - expression for a terminating sentinel
5818 (only required if the sentinel would ordinarily match the repetition
5819 expression) - (default= ``None``)
5821 Example: similar to :class:`OneOrMore`
5822 """
5824 def __init__(
5825 self,
5826 expr: Union[str, ParserElement],
5827 stop_on: typing.Optional[Union[ParserElement, str]] = None,
5828 max: typing.Optional[int] = None,
5829 **kwargs,
5830 ) -> None:
5831 stopOn: Union[ParserElement, str] = deprecate_argument(kwargs, "stopOn", None)
5833 super().__init__(expr, stop_on=stopOn or stop_on, max=max)
5834 self._may_return_empty = True
5836 def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType:
5837 try:
5838 return super().parseImpl(instring, loc, do_actions)
5839 except (ParseException, IndexError):
5840 return loc, ParseResults([], name=self.resultsName)
5842 def _generateDefaultName(self) -> str:
5843 return f"[{self.expr}]..."
5846class DelimitedList(ParseElementEnhance):
5847 """Helper to define a delimited list of expressions - the delimiter
5848 defaults to ','. By default, the list elements and delimiters can
5849 have intervening whitespace, and comments, but this can be
5850 overridden by passing ``combine=True`` in the constructor. If
5851 ``combine`` is set to ``True``, the matching tokens are
5852 returned as a single token string, with the delimiters included;
5853 otherwise, the matching tokens are returned as a list of tokens,
5854 with the delimiters suppressed.
5856 If ``allow_trailing_delim`` is set to True, then the list may end with
5857 a delimiter.
5859 Example:
5861 .. doctest::
5863 >>> DelimitedList(Word(alphas)).parse_string("aa,bb,cc")
5864 ParseResults(['aa', 'bb', 'cc'], {})
5865 >>> DelimitedList(Word(hexnums), delim=':', combine=True
5866 ... ).parse_string("AA:BB:CC:DD:EE")
5867 ParseResults(['AA:BB:CC:DD:EE'], {})
5869 .. versionadded:: 3.1.0
5870 """
5872 def __init__(
5873 self,
5874 expr: Union[str, ParserElement],
5875 delim: Union[str, ParserElement] = ",",
5876 combine: bool = False,
5877 min: typing.Optional[int] = None,
5878 max: typing.Optional[int] = None,
5879 *,
5880 allow_trailing_delim: bool = False,
5881 ) -> None:
5882 if isinstance(expr, str_type):
5883 expr = ParserElement._literalStringClass(expr)
5884 expr = typing.cast(ParserElement, expr)
5886 if min is not None and min < 1:
5887 raise ValueError("min must be greater than 0")
5889 if max is not None and min is not None and max < min:
5890 raise ValueError("max must be greater than, or equal to min")
5892 self.content = expr
5893 self.raw_delim = str(delim)
5894 self.delim = delim
5895 self.combine = combine
5896 if not combine:
5897 self.delim = Suppress(delim) if not isinstance(delim, Suppress) else delim
5898 self.min = min or 1
5899 self.max = max
5900 self.allow_trailing_delim = allow_trailing_delim
5902 delim_list_expr = self.content + (self.delim + self.content) * (
5903 self.min - 1,
5904 None if self.max is None else self.max - 1,
5905 )
5906 if self.allow_trailing_delim:
5907 delim_list_expr += Opt(self.delim)
5909 if self.combine:
5910 delim_list_expr = Combine(delim_list_expr)
5912 super().__init__(delim_list_expr, savelist=True)
5914 def _generateDefaultName(self) -> str:
5915 content_expr = self.content.streamline()
5916 return f"{content_expr} [{self.raw_delim} {content_expr}]..."
5919class _NullToken:
5920 def __bool__(self):
5921 return False
5923 def __str__(self):
5924 return ""
5927class Opt(ParseElementEnhance):
5928 """
5929 Optional matching of the given expression.
5931 :param expr: expression that must match zero or more times
5932 :param default: (optional) - value to be returned
5933 if the optional expression is not found.
5935 Example:
5937 .. testcode::
5939 # US postal code can be a 5-digit zip, plus optional 4-digit qualifier
5940 zip = Combine(Word(nums, exact=5) + Opt('-' + Word(nums, exact=4)))
5941 zip.run_tests('''
5942 # traditional ZIP code
5943 12345
5945 # ZIP+4 form
5946 12101-0001
5948 # invalid ZIP
5949 98765-
5950 ''')
5952 prints:
5954 .. testoutput::
5955 :options: +NORMALIZE_WHITESPACE
5958 # traditional ZIP code
5959 12345
5960 ['12345']
5962 # ZIP+4 form
5963 12101-0001
5964 ['12101-0001']
5966 # invalid ZIP
5967 98765-
5968 98765-
5969 ^
5970 ParseException: Expected end of text, found '-' (at char 5), (line:1, col:6)
5971 FAIL: Expected end of text, found '-' (at char 5), (line:1, col:6)
5972 """
5974 __optionalNotMatched = _NullToken()
5976 def __init__(
5977 self, expr: Union[ParserElement, str], default: Any = __optionalNotMatched
5978 ) -> None:
5979 super().__init__(expr, savelist=False)
5980 self.saveAsList = self.expr.saveAsList
5981 self.defaultValue = default
5982 self._may_return_empty = True
5984 def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType:
5985 self_expr = self.expr
5986 try:
5987 loc, tokens = self_expr._parse(
5988 instring, loc, do_actions, callPreParse=False
5989 )
5990 except (ParseException, IndexError):
5991 default_value = self.defaultValue
5992 if default_value is not self.__optionalNotMatched:
5993 if self_expr.resultsName:
5994 tokens = ParseResults([default_value])
5995 tokens[self_expr.resultsName] = default_value
5996 else:
5997 tokens = [default_value] # type: ignore[assignment]
5998 else:
5999 tokens = [] # type: ignore[assignment]
6000 return loc, tokens
6002 def _generateDefaultName(self) -> str:
6003 inner = str(self.expr)
6004 # strip off redundant inner {}'s
6005 while len(inner) > 1 and inner[0 :: len(inner) - 1] == "{}":
6006 inner = inner[1:-1]
6007 return f"[{inner}]"
6010Optional = Opt
6013class SkipTo(ParseElementEnhance):
6014 """
6015 Token for skipping over all undefined text until the matched
6016 expression is found.
6018 :param expr: target expression marking the end of the data to be skipped
6019 :param include: if ``True``, the target expression is also parsed
6020 (the skipped text and target expression are returned
6021 as a 2-element list) (default= ``False``).
6023 :param ignore: (default= ``None``) used to define grammars
6024 (typically quoted strings and comments)
6025 that might contain false matches to the target expression
6027 :param fail_on: (default= ``None``) define expressions that
6028 are not allowed to be included in the skipped test;
6029 if found before the target expression is found,
6030 the :class:`SkipTo` is not a match
6032 Example:
6034 .. testcode::
6036 report = '''
6037 Outstanding Issues Report - 1 Jan 2000
6039 # | Severity | Description | Days Open
6040 -----+----------+-------------------------------------------+-----------
6041 101 | Critical | Intermittent system crash | 6
6042 94 | Cosmetic | Spelling error on Login ('log|n') | 14
6043 79 | Minor | System slow when running too many reports | 47
6044 '''
6045 integer = Word(nums)
6046 SEP = Suppress('|')
6047 # use SkipTo to simply match everything up until the next SEP
6048 # - ignore quoted strings, so that a '|' character inside a quoted string does not match
6049 # - parse action will call token.strip() for each matched token, i.e., the description body
6050 string_data = SkipTo(SEP, ignore=quoted_string)
6051 string_data.set_parse_action(token_map(str.strip))
6052 ticket_expr = (integer("issue_num") + SEP
6053 + string_data("sev") + SEP
6054 + string_data("desc") + SEP
6055 + integer("days_open"))
6057 for tkt in ticket_expr.search_string(report):
6058 print(tkt.dump())
6060 prints:
6062 .. testoutput::
6064 ['101', 'Critical', 'Intermittent system crash', '6']
6065 - days_open: '6'
6066 - desc: 'Intermittent system crash'
6067 - issue_num: '101'
6068 - sev: 'Critical'
6069 ['94', 'Cosmetic', "Spelling error on Login ('log|n')", '14']
6070 - days_open: '14'
6071 - desc: "Spelling error on Login ('log|n')"
6072 - issue_num: '94'
6073 - sev: 'Cosmetic'
6074 ['79', 'Minor', 'System slow when running too many reports', '47']
6075 - days_open: '47'
6076 - desc: 'System slow when running too many reports'
6077 - issue_num: '79'
6078 - sev: 'Minor'
6079 """
6081 def __init__(
6082 self,
6083 other: Union[ParserElement, str],
6084 include: bool = False,
6085 ignore: typing.Optional[Union[ParserElement, str]] = None,
6086 fail_on: typing.Optional[Union[ParserElement, str]] = None,
6087 **kwargs,
6088 ) -> None:
6089 failOn: typing.Optional[Union[ParserElement, str]] = deprecate_argument(
6090 kwargs, "failOn", None
6091 )
6093 super().__init__(other)
6094 failOn = failOn or fail_on
6095 self.ignoreExpr = ignore
6096 self._may_return_empty = True
6097 self.mayIndexError = False
6098 self.includeMatch = include
6099 self.saveAsList = False
6100 if isinstance(failOn, str_type):
6101 self.failOn = self._literalStringClass(failOn)
6102 else:
6103 self.failOn = failOn
6104 self.errmsg = f"No match found for {self.expr}"
6105 self.ignorer = Empty().leave_whitespace()
6106 self._update_ignorer()
6108 def _update_ignorer(self):
6109 # rebuild internal ignore expr from current ignore exprs and assigned ignoreExpr
6110 self.ignorer.ignoreExprs.clear()
6111 for e in self.expr.ignoreExprs:
6112 self.ignorer.ignore(e)
6113 if self.ignoreExpr:
6114 self.ignorer.ignore(self.ignoreExpr)
6116 def ignore(self, expr):
6117 """
6118 Define expression to be ignored (e.g., comments) while doing pattern
6119 matching; may be called repeatedly, to define multiple comment or other
6120 ignorable patterns.
6121 """
6122 super().ignore(expr)
6123 self._update_ignorer()
6125 def parseImpl(self, instring, loc, do_actions=True):
6126 startloc = loc
6127 instrlen = len(instring)
6128 self_expr_parse = self.expr._parse
6129 self_failOn_canParseNext = (
6130 self.failOn.can_parse_next if self.failOn is not None else None
6131 )
6132 ignorer_try_parse = self.ignorer.try_parse if self.ignorer.ignoreExprs else None
6134 tmploc = loc
6135 while tmploc <= instrlen:
6136 if self_failOn_canParseNext is not None:
6137 # break if failOn expression matches
6138 if self_failOn_canParseNext(instring, tmploc):
6139 break
6141 if ignorer_try_parse is not None:
6142 # advance past ignore expressions
6143 prev_tmploc = tmploc
6144 while 1:
6145 try:
6146 tmploc = ignorer_try_parse(instring, tmploc)
6147 except ParseBaseException:
6148 break
6149 # see if all ignorers matched, but didn't actually ignore anything
6150 if tmploc == prev_tmploc:
6151 break
6152 prev_tmploc = tmploc
6154 try:
6155 self_expr_parse(instring, tmploc, do_actions=False, callPreParse=False)
6156 except (ParseException, IndexError):
6157 # no match, advance loc in string
6158 tmploc += 1
6159 else:
6160 # matched skipto expr, done
6161 break
6163 else:
6164 # ran off the end of the input string without matching skipto expr, fail
6165 raise ParseException(instring, loc, self.errmsg, self)
6167 # build up return values
6168 loc = tmploc
6169 skiptext = instring[startloc:loc]
6170 skipresult = ParseResults(skiptext)
6172 if self.includeMatch:
6173 loc, mat = self_expr_parse(instring, loc, do_actions, callPreParse=False)
6174 skipresult += mat
6176 return loc, skipresult
6179class Forward(ParseElementEnhance):
6180 """
6181 Forward declaration of an expression to be defined later -
6182 used for recursive grammars, such as algebraic infix notation.
6183 When the expression is known, it is assigned to the ``Forward``
6184 instance using the ``'<<'`` operator.
6186 .. Note::
6188 Take care when assigning to ``Forward`` not to overlook
6189 precedence of operators.
6191 Specifically, ``'|'`` has a lower precedence than ``'<<'``, so that::
6193 fwd_expr << a | b | c
6195 will actually be evaluated as::
6197 (fwd_expr << a) | b | c
6199 thereby leaving b and c out as parseable alternatives.
6200 It is recommended that you explicitly group the values
6201 inserted into the :class:`Forward`::
6203 fwd_expr << (a | b | c)
6205 Converting to use the ``'<<='`` operator instead will avoid this problem.
6207 See :meth:`ParseResults.pprint` for an example of a recursive
6208 parser created using :class:`Forward`.
6209 """
6211 def __init__(
6212 self, other: typing.Optional[Union[ParserElement, str]] = None
6213 ) -> None:
6214 self.caller_frame = traceback.extract_stack(limit=2)[0]
6215 super().__init__(other, savelist=False) # type: ignore[arg-type]
6216 self.lshift_line = None
6218 def __lshift__(self, other) -> Forward:
6219 if hasattr(self, "caller_frame"):
6220 del self.caller_frame
6221 if isinstance(other, str_type):
6222 other = self._literalStringClass(other)
6224 if not isinstance(other, ParserElement):
6225 return NotImplemented
6227 self.expr = other
6228 self.streamlined = other.streamlined
6229 self.mayIndexError = self.expr.mayIndexError
6230 self._may_return_empty = self.expr.mayReturnEmpty
6231 self.set_whitespace_chars(
6232 self.expr.whiteChars, copy_defaults=self.expr.copyDefaultWhiteChars
6233 )
6234 self.skipWhitespace = self.expr.skipWhitespace
6235 self.saveAsList = self.expr.saveAsList
6236 self.ignoreExprs.extend(self.expr.ignoreExprs)
6237 self.lshift_line = traceback.extract_stack(limit=2)[-2] # type: ignore[assignment]
6238 return self
6240 def __ilshift__(self, other) -> Forward:
6241 if not isinstance(other, ParserElement):
6242 return NotImplemented
6244 return self << other
6246 def __or__(self, other) -> ParserElement:
6247 caller_line = traceback.extract_stack(limit=2)[-2]
6248 if (
6249 __diag__.warn_on_match_first_with_lshift_operator
6250 and caller_line == self.lshift_line
6251 and Diagnostics.warn_on_match_first_with_lshift_operator
6252 not in self.suppress_warnings_
6253 ):
6254 warnings.warn(
6255 "warn_on_match_first_with_lshift_operator:"
6256 " using '<<' operator with '|' is probably an error, use '<<='",
6257 PyparsingDiagnosticWarning,
6258 stacklevel=2,
6259 )
6260 ret = super().__or__(other)
6261 return ret
6263 def __del__(self):
6264 # see if we are getting dropped because of '=' reassignment of var instead of '<<=' or '<<'
6265 if (
6266 self.expr is None
6267 and __diag__.warn_on_assignment_to_Forward
6268 and Diagnostics.warn_on_assignment_to_Forward not in self.suppress_warnings_
6269 ):
6270 warnings.warn_explicit(
6271 "warn_on_assignment_to_Forward:"
6272 " Forward defined here but no expression attached later using '<<=' or '<<'",
6273 UserWarning,
6274 filename=self.caller_frame.filename,
6275 lineno=self.caller_frame.lineno,
6276 )
6278 def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType:
6279 if (
6280 self.expr is None
6281 and __diag__.warn_on_parse_using_empty_Forward
6282 and Diagnostics.warn_on_parse_using_empty_Forward
6283 not in self.suppress_warnings_
6284 ):
6285 # walk stack until parse_string, scan_string, search_string, or transform_string is found
6286 parse_fns = (
6287 "parse_string",
6288 "scan_string",
6289 "search_string",
6290 "transform_string",
6291 )
6292 tb = traceback.extract_stack(limit=200)
6293 for i, frm in enumerate(reversed(tb), start=1):
6294 if frm.name in parse_fns:
6295 stacklevel = i + 1
6296 break
6297 else:
6298 stacklevel = 2
6299 warnings.warn(
6300 "warn_on_parse_using_empty_Forward:"
6301 " Forward expression was never assigned a value, will not parse any input",
6302 PyparsingDiagnosticWarning,
6303 stacklevel=stacklevel,
6304 )
6305 if not ParserElement._left_recursion_enabled:
6306 return super().parseImpl(instring, loc, do_actions)
6307 # ## Bounded Recursion algorithm ##
6308 # Recursion only needs to be processed at ``Forward`` elements, since they are
6309 # the only ones that can actually refer to themselves. The general idea is
6310 # to handle recursion stepwise: We start at no recursion, then recurse once,
6311 # recurse twice, ..., until more recursion offers no benefit (we hit the bound).
6312 #
6313 # The "trick" here is that each ``Forward`` gets evaluated in two contexts
6314 # - to *match* a specific recursion level, and
6315 # - to *search* the bounded recursion level
6316 # and the two run concurrently. The *search* must *match* each recursion level
6317 # to find the best possible match. This is handled by a memo table, which
6318 # provides the previous match to the next level match attempt.
6319 #
6320 # See also "Left Recursion in Parsing Expression Grammars", Medeiros et al.
6321 #
6322 # There is a complication since we not only *parse* but also *transform* via
6323 # actions: We do not want to run the actions too often while expanding. Thus,
6324 # we expand using `do_actions=False` and only run `do_actions=True` if the next
6325 # recursion level is acceptable.
6326 with ParserElement.recursion_lock:
6327 memo = ParserElement.recursion_memos
6328 try:
6329 # we are parsing at a specific recursion expansion - use it as-is
6330 prev_loc, prev_result = memo[loc, self, do_actions]
6331 if isinstance(prev_result, Exception):
6332 raise prev_result
6333 return prev_loc, prev_result.copy()
6334 except KeyError:
6335 act_key = (loc, self, True)
6336 peek_key = (loc, self, False)
6337 # we are searching for the best recursion expansion - keep on improving
6338 # both `do_actions` cases must be tracked separately here!
6339 prev_loc, prev_peek = memo[peek_key] = (
6340 loc - 1,
6341 ParseException(
6342 instring, loc, "Forward recursion without base case", self
6343 ),
6344 )
6345 if do_actions:
6346 memo[act_key] = memo[peek_key]
6347 while True:
6348 try:
6349 new_loc, new_peek = super().parseImpl(instring, loc, False)
6350 except ParseException:
6351 # we failed before getting any match - do not hide the error
6352 if isinstance(prev_peek, Exception):
6353 raise
6354 new_loc, new_peek = prev_loc, prev_peek
6355 # the match did not get better: we are done
6356 if new_loc <= prev_loc:
6357 if do_actions:
6358 # replace the match for do_actions=False as well,
6359 # in case the action did backtrack
6360 prev_loc, prev_result = memo[peek_key] = memo[act_key]
6361 del memo[peek_key], memo[act_key]
6362 return prev_loc, copy.copy(prev_result)
6363 del memo[peek_key]
6364 return prev_loc, copy.copy(prev_peek)
6365 # the match did get better: see if we can improve further
6366 if do_actions:
6367 try:
6368 memo[act_key] = super().parseImpl(instring, loc, True)
6369 except ParseException as e:
6370 memo[peek_key] = memo[act_key] = (new_loc, e)
6371 raise
6372 prev_loc, prev_peek = memo[peek_key] = new_loc, new_peek
6374 def leave_whitespace(self, recursive: bool = True) -> ParserElement:
6375 """
6376 Extends ``leave_whitespace`` defined in base class.
6377 """
6378 self.skipWhitespace = False
6379 return self
6381 def ignore_whitespace(self, recursive: bool = True) -> ParserElement:
6382 """
6383 Extends ``ignore_whitespace`` defined in base class.
6384 """
6385 self.skipWhitespace = True
6386 return self
6388 def streamline(self) -> ParserElement:
6389 if not self.streamlined:
6390 self.streamlined = True
6391 if self.expr is not None:
6392 self.expr.streamline()
6393 return self
6395 def validate(self, validateTrace=None) -> None:
6396 warnings.warn(
6397 "ParserElement.validate() is deprecated, and should not be used to check for left recursion",
6398 PyparsingDeprecationWarning,
6399 stacklevel=2,
6400 )
6401 if validateTrace is None:
6402 validateTrace = []
6404 if self not in validateTrace:
6405 tmp = validateTrace[:] + [self]
6406 if self.expr is not None:
6407 self.expr.validate(tmp)
6408 self._checkRecursion([])
6410 def _generateDefaultName(self) -> str:
6411 # Avoid infinite recursion by setting a temporary _defaultName
6412 save_default_name = self._defaultName
6413 self._defaultName = ": ..."
6415 # Use the string representation of main expression.
6416 try:
6417 if self.expr is not None:
6418 ret_string = str(self.expr)[:1000]
6419 else:
6420 ret_string = "None"
6421 except Exception:
6422 ret_string = "..."
6424 self._defaultName = save_default_name
6425 return f"{type(self).__name__}: {ret_string}"
6427 def copy(self) -> ParserElement:
6428 """
6429 Returns a copy of this expression.
6431 Generally only used internally by pyparsing.
6432 """
6433 if self.expr is not None:
6434 return super().copy()
6435 else:
6436 ret = Forward()
6437 ret <<= self
6438 return ret
6440 def _setResultsName(self, name, list_all_matches=False) -> ParserElement:
6441 # fmt: off
6442 if (
6443 __diag__.warn_name_set_on_empty_Forward
6444 and Diagnostics.warn_name_set_on_empty_Forward not in self.suppress_warnings_
6445 and self.expr is None
6446 ):
6447 warning = (
6448 "warn_name_set_on_empty_Forward:"
6449 f" setting results name {name!r} on {type(self).__name__} expression"
6450 " that has no contained expression"
6451 )
6452 warnings.warn(warning, PyparsingDiagnosticWarning, stacklevel=3)
6453 # fmt: on
6455 return super()._setResultsName(name, list_all_matches)
6457 # Compatibility synonyms
6458 # fmt: off
6459 leaveWhitespace = replaced_by_pep8("leaveWhitespace", leave_whitespace)
6460 ignoreWhitespace = replaced_by_pep8("ignoreWhitespace", ignore_whitespace)
6461 # fmt: on
6464class TokenConverter(ParseElementEnhance):
6465 """
6466 Abstract subclass of :class:`ParseElementEnhance`, for converting parsed results.
6467 """
6469 def __init__(self, expr: Union[ParserElement, str], savelist=False) -> None:
6470 super().__init__(expr) # , savelist)
6471 self.saveAsList = False
6474class Combine(TokenConverter):
6475 """Converter to concatenate all matching tokens to a single string.
6476 By default, the matching patterns must also be contiguous in the
6477 input string; this can be disabled by specifying
6478 ``'adjacent=False'`` in the constructor.
6480 Example:
6482 .. doctest::
6484 >>> real = Word(nums) + '.' + Word(nums)
6485 >>> print(real.parse_string('3.1416'))
6486 ['3', '.', '1416']
6488 >>> # will also erroneously match the following
6489 >>> print(real.parse_string('3. 1416'))
6490 ['3', '.', '1416']
6492 >>> real = Combine(Word(nums) + '.' + Word(nums))
6493 >>> print(real.parse_string('3.1416'))
6494 ['3.1416']
6496 >>> # no match when there are internal spaces
6497 >>> print(real.parse_string('3. 1416'))
6498 Traceback (most recent call last):
6499 ParseException: Expected W:(0123...)
6500 """
6502 def __init__(
6503 self,
6504 expr: ParserElement,
6505 join_string: str = "",
6506 adjacent: bool = True,
6507 *,
6508 joinString: typing.Optional[str] = None,
6509 ) -> None:
6510 super().__init__(expr)
6511 joinString = joinString if joinString is not None else join_string
6512 # suppress whitespace-stripping in contained parse expressions, but re-enable it on the Combine itself
6513 if adjacent:
6514 self.leave_whitespace()
6515 self.adjacent = adjacent
6516 self.skipWhitespace = True
6517 self.joinString = joinString
6518 self.callPreparse = True
6520 def ignore(self, other) -> ParserElement:
6521 """
6522 Define expression to be ignored (e.g., comments) while doing pattern
6523 matching; may be called repeatedly, to define multiple comment or other
6524 ignorable patterns.
6525 """
6526 if self.adjacent:
6527 ParserElement.ignore(self, other)
6528 else:
6529 super().ignore(other)
6530 return self
6532 def postParse(self, instring, loc, tokenlist):
6533 retToks = tokenlist.copy()
6534 del retToks[:]
6535 retToks += ParseResults(
6536 ["".join(tokenlist._asStringList(self.joinString))], modal=self.modalResults
6537 )
6539 if self.resultsName and retToks.haskeys():
6540 return [retToks]
6541 else:
6542 return retToks
6545class Group(TokenConverter):
6546 """Converter to return the matched tokens as a list - useful for
6547 returning tokens of :class:`ZeroOrMore` and :class:`OneOrMore` expressions.
6549 The optional ``aslist`` argument when set to True will return the
6550 parsed tokens as a Python list instead of a pyparsing ParseResults.
6552 Example:
6554 .. doctest::
6556 >>> ident = Word(alphas)
6557 >>> num = Word(nums)
6558 >>> term = ident | num
6559 >>> func = ident + Opt(DelimitedList(term))
6560 >>> print(func.parse_string("fn a, b, 100"))
6561 ['fn', 'a', 'b', '100']
6563 >>> func = ident + Group(Opt(DelimitedList(term)))
6564 >>> print(func.parse_string("fn a, b, 100"))
6565 ['fn', ['a', 'b', '100']]
6566 """
6568 def __init__(self, expr: ParserElement, aslist: bool = False) -> None:
6569 super().__init__(expr)
6570 self.saveAsList = True
6571 self._asPythonList = aslist
6573 def postParse(self, instring, loc, tokenlist):
6574 if self._asPythonList:
6575 return ParseResults.List(
6576 tokenlist.as_list()
6577 if isinstance(tokenlist, ParseResults)
6578 else list(tokenlist)
6579 )
6581 return [tokenlist]
6584class Dict(TokenConverter):
6585 """Converter to return a repetitive expression as a list, but also
6586 as a dictionary. Each element can also be referenced using the first
6587 token in the expression as its key. Useful for tabular report
6588 scraping when the first column can be used as a item key.
6590 The optional ``asdict`` argument when set to True will return the
6591 parsed tokens as a Python dict instead of a pyparsing ParseResults.
6593 Example:
6595 .. doctest::
6597 >>> data_word = Word(alphas)
6598 >>> label = data_word + FollowedBy(':')
6600 >>> attr_expr = (
6601 ... label + Suppress(':')
6602 ... + OneOrMore(data_word, stop_on=label)
6603 ... .set_parse_action(' '.join)
6604 ... )
6606 >>> text = "shape: SQUARE posn: upper left color: light blue texture: burlap"
6608 >>> # print attributes as plain groups
6609 >>> print(attr_expr[1, ...].parse_string(text).dump())
6610 ['shape', 'SQUARE', 'posn', 'upper left', 'color', 'light blue', 'texture', 'burlap']
6612 # instead of OneOrMore(expr), parse using Dict(Group(expr)[1, ...])
6613 # Dict will auto-assign names.
6614 >>> result = Dict(Group(attr_expr)[1, ...]).parse_string(text)
6615 >>> print(result.dump())
6616 [['shape', 'SQUARE'], ['posn', 'upper left'], ['color', 'light blue'], ['texture', 'burlap']]
6617 - color: 'light blue'
6618 - posn: 'upper left'
6619 - shape: 'SQUARE'
6620 - texture: 'burlap'
6621 [0]:
6622 ['shape', 'SQUARE']
6623 [1]:
6624 ['posn', 'upper left']
6625 [2]:
6626 ['color', 'light blue']
6627 [3]:
6628 ['texture', 'burlap']
6630 # access named fields as dict entries, or output as dict
6631 >>> print(result['shape'])
6632 SQUARE
6633 >>> print(result.as_dict())
6634 {'shape': 'SQUARE', 'posn': 'upper left', 'color': 'light blue', 'texture': 'burlap'}
6636 See more examples at :class:`ParseResults` of accessing fields by results name.
6637 """
6639 def __init__(self, expr: ParserElement, asdict: bool = False) -> None:
6640 super().__init__(expr)
6641 self.saveAsList = True
6642 self._asPythonDict = asdict
6644 def postParse(self, instring, loc, tokenlist):
6645 for i, tok in enumerate(tokenlist):
6646 if len(tok) == 0:
6647 continue
6649 ikey = tok[0]
6650 if isinstance(ikey, int):
6651 ikey = str(ikey).strip()
6653 # ParseResults are not hashable, cannot be used as a dict key
6654 if isinstance(ikey, ParseResults):
6655 continue
6657 if len(tok) == 1:
6658 tokenlist[ikey] = _ParseResultsWithOffset("", i)
6660 elif len(tok) == 2 and not isinstance(tok[1], ParseResults):
6661 tokenlist[ikey] = _ParseResultsWithOffset(tok[1], i)
6663 else:
6664 try:
6665 dictvalue = tok.copy() # ParseResults(i)
6666 except Exception:
6667 exc = TypeError(
6668 "could not extract dict values from parsed results"
6669 " - Dict expression must contain Grouped expressions"
6670 )
6671 raise exc from None
6673 del dictvalue[0]
6675 if len(dictvalue) != 1 or (
6676 isinstance(dictvalue, ParseResults) and dictvalue.haskeys()
6677 ):
6678 tokenlist[ikey] = _ParseResultsWithOffset(dictvalue, i)
6679 else:
6680 tokenlist[ikey] = _ParseResultsWithOffset(dictvalue[0], i)
6682 # Mark the tokenlist so that as_dict() knows to serialize an empty
6683 # result as {} rather than [], preserving dict semantics even when
6684 # there are no matched entries.
6685 tokenlist._is_dict_context = True
6687 if self._asPythonDict:
6688 return [tokenlist.as_dict()] if self.resultsName else tokenlist.as_dict()
6690 return [tokenlist] if self.resultsName else tokenlist
6693class Suppress(TokenConverter):
6694 """Converter for ignoring the results of a parsed expression.
6696 Example:
6698 .. doctest::
6700 >>> source = "a, b, c,d"
6701 >>> wd = Word(alphas)
6702 >>> wd_list1 = wd + (',' + wd)[...]
6703 >>> print(wd_list1.parse_string(source))
6704 ['a', ',', 'b', ',', 'c', ',', 'd']
6706 # often, delimiters that are useful during parsing are just in the
6707 # way afterward - use Suppress to keep them out of the parsed output
6708 >>> wd_list2 = wd + (Suppress(',') + wd)[...]
6709 >>> print(wd_list2.parse_string(source))
6710 ['a', 'b', 'c', 'd']
6712 # Skipped text (using '...') can be suppressed as well
6713 >>> source = "lead in START relevant text END trailing text"
6714 >>> start_marker = Keyword("START")
6715 >>> end_marker = Keyword("END")
6716 >>> find_body = Suppress(...) + start_marker + ... + end_marker
6717 >>> print(find_body.parse_string(source))
6718 ['START', 'relevant text ', 'END']
6720 (See also :class:`DelimitedList`.)
6721 """
6723 def __init__(self, expr: Union[ParserElement, str], savelist: bool = False) -> None:
6724 if expr is ...:
6725 expr = _PendingSkip(NoMatch())
6726 super().__init__(expr)
6728 def __add__(self, other) -> ParserElement:
6729 if isinstance(self.expr, _PendingSkip):
6730 return Suppress(SkipTo(other)) + other
6732 return super().__add__(other)
6734 def __sub__(self, other) -> ParserElement:
6735 if isinstance(self.expr, _PendingSkip):
6736 return Suppress(SkipTo(other)) - other
6738 return super().__sub__(other)
6740 def postParse(self, instring, loc, tokenlist):
6741 return []
6743 def suppress(self) -> ParserElement:
6744 return self
6747# XXX: Example needs to be re-done for updated output
6748def trace_parse_action(f: ParseAction) -> ParseAction:
6749 """Decorator for debugging parse actions.
6751 When the parse action is called, this decorator will print
6752 ``">> entering method-name(line:<current_source_line>, <parse_location>, <matched_tokens>)"``.
6753 When the parse action completes, the decorator will print
6754 ``"<<"`` followed by the returned value, or any exception that the parse action raised.
6756 Example:
6758 .. testsetup:: stderr
6760 import sys
6761 sys.stderr = sys.stdout
6763 .. testcleanup:: stderr
6765 sys.stderr = sys.__stderr__
6767 .. testcode:: stderr
6769 wd = Word(alphas)
6771 @trace_parse_action
6772 def remove_duplicate_chars(tokens):
6773 return ''.join(sorted(set(''.join(tokens))))
6775 wds = wd[1, ...].set_parse_action(remove_duplicate_chars)
6776 print(wds.parse_string("slkdjs sld sldd sdlf sdljf"))
6778 prints:
6780 .. testoutput:: stderr
6781 :options: +NORMALIZE_WHITESPACE
6783 >>entering remove_duplicate_chars(line: 'slkdjs sld sldd sdlf sdljf',
6784 0, ParseResults(['slkdjs', 'sld', 'sldd', 'sdlf', 'sdljf'], {}))
6785 <<leaving remove_duplicate_chars (ret: 'dfjkls')
6786 ['dfjkls']
6788 .. versionchanged:: 3.1.0
6789 Exception type added to output
6790 """
6791 f = _trim_arity(f)
6793 def z(*paArgs):
6794 thisFunc = f.__name__
6795 s, l, t = paArgs[-3:]
6796 if len(paArgs) > 3:
6797 thisFunc = f"{type(paArgs[0]).__name__}.{thisFunc}"
6798 sys.stderr.write(f">>entering {thisFunc}(line: {line(l, s)!r}, {l}, {t!r})\n")
6799 try:
6800 ret = f(*paArgs)
6801 except Exception as exc:
6802 sys.stderr.write(
6803 f"<<leaving {thisFunc} (exception: {type(exc).__name__}: {exc})\n"
6804 )
6805 raise
6806 sys.stderr.write(f"<<leaving {thisFunc} (ret: {ret!r})\n")
6807 return ret
6809 z.__name__ = f.__name__
6810 return z
6813# convenience constants for positional expressions
6814empty = Empty().set_name("empty")
6815line_start = LineStart().set_name("line_start")
6816line_end = LineEnd().set_name("line_end")
6817string_start = StringStart().set_name("string_start")
6818string_end = StringEnd().set_name("string_end")
6820_escapedPunc = Regex(r"\\[\\[\]\/\-\*\.\$\+\^\?()~ ]").set_parse_action(
6821 lambda s, l, t: t[0][1]
6822)
6823_escapedHexChar = Regex(r"\\0?[xX][0-9a-fA-F]+").set_parse_action(
6824 lambda s, l, t: chr(int(t[0].lstrip(r"\0x"), 16))
6825)
6826_escapedOctChar = Regex(r"\\0[0-7]+").set_parse_action(
6827 lambda s, l, t: chr(int(t[0][1:], 8))
6828)
6829_singleChar = (
6830 _escapedPunc | _escapedHexChar | _escapedOctChar | CharsNotIn(r"\]", exact=1)
6831)
6832_charRange = Group(_singleChar + Suppress("-") + _singleChar)
6833_reBracketExpr = (
6834 Literal("[")
6835 + Opt("^").set_results_name("negate")
6836 + Group(OneOrMore(_charRange | _singleChar)).set_results_name("body")
6837 + Literal("]")
6838)
6841def srange(s: str) -> str:
6842 r"""Helper to easily define string ranges for use in :class:`Word`
6843 construction. Borrows syntax from regexp ``'[]'`` string range
6844 definitions::
6846 srange("[0-9]") -> "0123456789"
6847 srange("[a-z]") -> "abcdefghijklmnopqrstuvwxyz"
6848 srange("[a-z$_]") -> "abcdefghijklmnopqrstuvwxyz$_"
6850 The input string must be enclosed in []'s, and the returned string
6851 is the expanded character set joined into a single string. The
6852 values enclosed in the []'s may be:
6854 - a single character
6855 - an escaped character with a leading backslash (such as ``\-``
6856 or ``\]``)
6857 - an escaped hex character with a leading ``'\x'``
6858 (``\x21``, which is a ``'!'`` character) (``\0x##``
6859 is also supported for backwards compatibility)
6860 - an escaped octal character with a leading ``'\0'``
6861 (``\041``, which is a ``'!'`` character)
6862 - a range of any of the above, separated by a dash (``'a-z'``,
6863 etc.)
6864 - any combination of the above (``'aeiouy'``,
6865 ``'a-zA-Z0-9_$'``, etc.)
6866 """
6868 def _expanded(p):
6869 if isinstance(p, ParseResults):
6870 yield from (chr(c) for c in range(ord(p[0]), ord(p[1]) + 1))
6871 else:
6872 yield p
6874 try:
6875 return "".join(
6876 [c for part in _reBracketExpr.parse_string(s).body for c in _expanded(part)]
6877 )
6878 except Exception as e:
6879 return ""
6882def token_map(func, *args) -> ParseAction:
6883 """Helper to define a parse action by mapping a function to all
6884 elements of a :class:`ParseResults` list. If any additional args are passed,
6885 they are forwarded to the given function as additional arguments
6886 after the token, as in
6887 ``hex_integer = Word(hexnums).set_parse_action(token_map(int, 16))``,
6888 which will convert the parsed data to an integer using base 16.
6890 Example (compare the last to example in :class:`ParserElement.transform_string`::
6892 hex_ints = Word(hexnums)[1, ...].set_parse_action(token_map(int, 16))
6893 hex_ints.run_tests('''
6894 00 11 22 aa FF 0a 0d 1a
6895 ''')
6897 upperword = Word(alphas).set_parse_action(token_map(str.upper))
6898 upperword[1, ...].run_tests('''
6899 my kingdom for a horse
6900 ''')
6902 wd = Word(alphas).set_parse_action(token_map(str.title))
6903 wd[1, ...].set_parse_action(' '.join).run_tests('''
6904 now is the winter of our discontent made glorious summer by this sun of york
6905 ''')
6907 prints::
6909 00 11 22 aa FF 0a 0d 1a
6910 [0, 17, 34, 170, 255, 10, 13, 26]
6912 my kingdom for a horse
6913 ['MY', 'KINGDOM', 'FOR', 'A', 'HORSE']
6915 now is the winter of our discontent made glorious summer by this sun of york
6916 ['Now Is The Winter Of Our Discontent Made Glorious Summer By This Sun Of York']
6917 """
6919 def pa(s, l, t):
6920 return [func(tokn, *args) for tokn in t]
6922 func_name = getattr(func, "__name__", getattr(func, "__class__").__name__)
6923 pa.__name__ = func_name
6925 return pa
6928def autoname_elements() -> None:
6929 """
6930 Utility to simplify mass-naming of parser elements, for
6931 generating railroad diagram with named subdiagrams.
6932 """
6934 # guard against _getframe not being implemented in the current Python
6935 getframe_fn = getattr(sys, "_getframe", lambda _: None)
6936 calling_frame = getframe_fn(1)
6937 if calling_frame is None:
6938 return
6940 # find all locals in the calling frame that are ParserElements
6941 calling_frame = typing.cast(types.FrameType, calling_frame)
6942 for name, var in calling_frame.f_locals.items():
6943 # if no custom name defined, set the name to the var name
6944 if isinstance(var, ParserElement) and not var.customName:
6945 var.set_name(name)
6948dbl_quoted_string = (
6949 Regex(r'"(?:[^"\n\r\\]|(?:"")|(?:\\(?:[^x]|x[0-9a-fA-F]+)))*"')
6950).set_name("string enclosed in double quotes")
6952sgl_quoted_string = Regex(
6953 r"'(?:[^'\n\r\\]|(?:'')|(?:\\(?:[^x]|x[0-9a-fA-F]+)))*'"
6954).set_name("string enclosed in single quotes")
6956quoted_string = Combine(
6957 Regex(r'"(?:[^"\n\r\\]|(?:"")|(?:\\(?:[^x]|x[0-9a-fA-F]+)))*"').set_name(
6958 "double quoted string"
6959 )
6960 | Regex(r"'(?:[^'\n\r\\]|(?:'')|(?:\\(?:[^x]|x[0-9a-fA-F]+)))*'").set_name(
6961 "single quoted string"
6962 )
6963).set_name("quoted string using single or double quotes")
6965# XXX: Is there some way to make this show up in API docs?
6966# .. versionadded:: 3.1.0
6967python_quoted_string = Combine(
6968 Regex(r'"""(?:[^"\\]|""(?!")|"(?!"")|\\.)*"""', flags=re.MULTILINE).set_name(
6969 "multiline double quoted string"
6970 )
6971 ^ Regex(r"'''(?:[^'\\]|''(?!')|'(?!'')|\\.)*'''", flags=re.MULTILINE).set_name(
6972 "multiline single quoted string"
6973 )
6974 ^ Regex(r'"(?:[^"\n\r\\]|(?:\\")|(?:\\(?:[^x]|x[0-9a-fA-F]+)))*"').set_name(
6975 "double quoted string"
6976 )
6977 ^ Regex(r"'(?:[^'\n\r\\]|(?:\\')|(?:\\(?:[^x]|x[0-9a-fA-F]+)))*'").set_name(
6978 "single quoted string"
6979 )
6980).set_name("Python quoted string")
6982unicode_string = Combine("u" + quoted_string.copy()).set_name("unicode string literal")
6985alphas8bit = srange(r"[\0xc0-\0xd6\0xd8-\0xf6\0xf8-\0xff]")
6986punc8bit = srange(r"[\0xa1-\0xbf\0xd7\0xf7]")
6988# build list of built-in expressions, for future reference if a global default value
6989# gets updated
6990_builtin_exprs: list[ParserElement] = [
6991 v for v in vars().values() if isinstance(v, ParserElement)
6992]
6994# Compatibility synonyms
6995# fmt: off
6996sglQuotedString = sgl_quoted_string
6997dblQuotedString = dbl_quoted_string
6998quotedString = quoted_string
6999unicodeString = unicode_string
7000lineStart = line_start
7001lineEnd = line_end
7002stringStart = string_start
7003stringEnd = string_end
7004nullDebugAction = replaced_by_pep8("nullDebugAction", null_debug_action)
7005traceParseAction = replaced_by_pep8("traceParseAction", trace_parse_action)
7006conditionAsParseAction = replaced_by_pep8("conditionAsParseAction", condition_as_parse_action)
7007tokenMap = replaced_by_pep8("tokenMap", token_map)
7008# fmt: on