Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/pyparsing/core.py: 44%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#
2# core.py
3#
4from __future__ import annotations
6import collections.abc
7from collections import deque
8import os
9import typing
10from typing import (
11 Any,
12 Callable,
13 Generator,
14 NamedTuple,
15 Sequence,
16 TextIO,
17 Union,
18 cast,
19)
20from abc import ABC, abstractmethod
21from enum import Enum
22import string
23import copy
24import warnings
25import re
26import sys
27from collections.abc import Iterable
28import traceback
29import types
30from operator import itemgetter
31from functools import wraps
32from threading import RLock
33from pathlib import Path
35from .warnings import PyparsingDeprecationWarning, PyparsingDiagnosticWarning
36from .util import (
37 _FifoCache,
38 _UnboundedCache,
39 __config_flags,
40 _collapse_string_to_ranges,
41 _convert_escaped_numerics_to_char,
42 _escape_regex_range_chars,
43 _flatten,
44 LRUMemo as _LRUMemo,
45 UnboundedMemo as _UnboundedMemo,
46 deprecate_argument,
47 replaced_by_pep8,
48)
49from .exceptions import *
50from .actions import *
51from .results import ParseResults, _ParseResultsWithOffset
52from .unicode import pyparsing_unicode
54_MAX_INT = sys.maxsize
55str_type: tuple[type, ...] = (str, bytes)
57#
58# Copyright (c) 2003-2022 Paul T. McGuire
59#
60# Permission is hereby granted, free of charge, to any person obtaining
61# a copy of this software and associated documentation files (the
62# "Software"), to deal in the Software without restriction, including
63# without limitation the rights to use, copy, modify, merge, publish,
64# distribute, sublicense, and/or sell copies of the Software, and to
65# permit persons to whom the Software is furnished to do so, subject to
66# the following conditions:
67#
68# The above copyright notice and this permission notice shall be
69# included in all copies or substantial portions of the Software.
70#
71# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
72# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
73# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
74# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
75# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
76# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
77# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
78#
80from functools import cached_property
83class __compat__(__config_flags):
84 """
85 A cross-version compatibility configuration for pyparsing features that will be
86 released in a future version. By setting values in this configuration to True,
87 those features can be enabled in prior versions for compatibility development
88 and testing.
90 - ``collect_all_And_tokens`` - flag to enable fix for Issue #63 that fixes erroneous grouping
91 of results names when an :class:`And` expression is nested within an :class:`Or` or :class:`MatchFirst`;
92 maintained for compatibility, but setting to ``False`` no longer restores pre-2.3.1
93 behavior
94 """
96 _type_desc = "compatibility"
98 collect_all_And_tokens = True
100 _all_names = [__ for __ in locals() if not __.startswith("_")]
101 _fixed_names = """
102 collect_all_And_tokens
103 """.split()
106class __diag__(__config_flags):
107 _type_desc = "diagnostic"
109 warn_multiple_tokens_in_named_alternation = False
110 warn_ungrouped_named_tokens_in_collection = False
111 warn_name_set_on_empty_Forward = False
112 warn_on_parse_using_empty_Forward = False
113 warn_on_assignment_to_Forward = False
114 warn_on_multiple_string_args_to_oneof = False
115 warn_on_match_first_with_lshift_operator = False
116 enable_debug_on_named_expressions = False
118 _all_names = [__ for __ in locals() if not __.startswith("_")]
119 _warning_names = [name for name in _all_names if name.startswith("warn")]
120 _debug_names = [name for name in _all_names if name.startswith("enable_debug")]
122 @classmethod
123 def enable_all_warnings(cls) -> None:
124 for name in cls._warning_names:
125 cls.enable(name)
128class Diagnostics(Enum):
129 """
130 Diagnostic configuration (all default to disabled)
132 - ``warn_multiple_tokens_in_named_alternation`` - flag to enable warnings when a results
133 name is defined on a :class:`MatchFirst` or :class:`Or` expression with one or more :class:`And` subexpressions
134 - ``warn_ungrouped_named_tokens_in_collection`` - flag to enable warnings when a results
135 name is defined on a containing expression with ungrouped subexpressions that also
136 have results names
137 - ``warn_name_set_on_empty_Forward`` - flag to enable warnings when a :class:`Forward` is defined
138 with a results name, but has no contents defined
139 - ``warn_on_parse_using_empty_Forward`` - flag to enable warnings when a :class:`Forward` is
140 defined in a grammar but has never had an expression attached to it
141 - ``warn_on_assignment_to_Forward`` - flag to enable warnings when a :class:`Forward` is defined
142 but is overwritten by assigning using ``'='`` instead of ``'<<='`` or ``'<<'``
143 - ``warn_on_multiple_string_args_to_oneof`` - flag to enable warnings when :class:`one_of` is
144 incorrectly called with multiple str arguments
145 - ``enable_debug_on_named_expressions`` - flag to auto-enable debug on all subsequent
146 calls to :class:`ParserElement.set_name`
148 Diagnostics are enabled/disabled by calling :class:`enable_diag` and :class:`disable_diag`.
149 All warnings can be enabled by calling :class:`enable_all_warnings`.
150 """
152 warn_multiple_tokens_in_named_alternation = 0
153 warn_ungrouped_named_tokens_in_collection = 1
154 warn_name_set_on_empty_Forward = 2
155 warn_on_parse_using_empty_Forward = 3
156 warn_on_assignment_to_Forward = 4
157 warn_on_multiple_string_args_to_oneof = 5
158 warn_on_match_first_with_lshift_operator = 6
159 enable_debug_on_named_expressions = 7
162def enable_diag(diag_enum: Diagnostics) -> None:
163 """
164 Enable a global pyparsing diagnostic flag (see :class:`Diagnostics`).
165 """
166 __diag__.enable(diag_enum.name)
169def disable_diag(diag_enum: Diagnostics) -> None:
170 """
171 Disable a global pyparsing diagnostic flag (see :class:`Diagnostics`).
172 """
173 __diag__.disable(diag_enum.name)
176def enable_all_warnings() -> None:
177 """
178 Enable all global pyparsing diagnostic warnings (see :class:`Diagnostics`).
179 """
180 __diag__.enable_all_warnings()
183# hide abstract class
184del __config_flags
187def _should_enable_warnings(
188 cmd_line_warn_options: typing.Iterable[str], warn_env_var: typing.Optional[str]
189) -> bool:
190 enable = bool(warn_env_var)
191 for warn_opt in cmd_line_warn_options:
192 w_action, w_message, w_category, w_module, w_line = (warn_opt + "::::").split(
193 ":"
194 )[:5]
195 if not w_action.lower().startswith("i") and (
196 not (w_message or w_category or w_module) or w_module == "pyparsing"
197 ):
198 enable = True
199 elif w_action.lower().startswith("i") and w_module in ("pyparsing", ""):
200 enable = False
201 return enable
204if _should_enable_warnings(
205 sys.warnoptions, os.environ.get("PYPARSINGENABLEALLWARNINGS")
206):
207 enable_all_warnings()
210# build list of single arg builtins, that can be used as parse actions
211# fmt: off
212_single_arg_builtins = {
213 sum, len, sorted, reversed, list, tuple, set, any, all, min, max
214}
215# fmt: on
217_generatorType = types.GeneratorType
218ParseImplReturnType = tuple[int, Any]
219PostParseReturnType = Union[ParseResults, Sequence[ParseResults]]
221ParseCondition = Union[
222 Callable[[], bool],
223 Callable[[ParseResults], bool],
224 Callable[[int, ParseResults], bool],
225 Callable[[str, int, ParseResults], bool],
226]
227ParseFailAction = Callable[[str, int, "ParserElement", Exception], None]
228DebugStartAction = Callable[[str, int, "ParserElement", bool], None]
229DebugSuccessAction = Callable[
230 [str, int, int, "ParserElement", ParseResults, bool], None
231]
232DebugExceptionAction = Callable[[str, int, "ParserElement", Exception, bool], None]
235alphas: str = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
236identchars: str = pyparsing_unicode.Latin1.identchars
237identbodychars: str = pyparsing_unicode.Latin1.identbodychars
238nums: str = "0123456789"
239hexnums: str = "0123456789ABCDEFabcdef"
240alphanums: str = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
241printables: str = (
242 '!"'
243 "#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ"
244 "[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~"
245)
248class _ParseActionIndexError(Exception):
249 """
250 Internal wrapper around IndexError so that IndexErrors raised inside
251 parse actions aren't misinterpreted as IndexErrors raised inside
252 ParserElement parseImpl methods.
253 """
255 def __init__(self, msg: str, exc: BaseException) -> None:
256 self.msg: str = msg
257 self.exc: BaseException = exc
260_trim_arity_call_line: traceback.StackSummary = None # type: ignore[assignment]
261pa_call_line_synth = ()
264def _trim_arity(func, max_limit=3):
265 """decorator to trim function calls to match the arity of the target"""
266 global _trim_arity_call_line, pa_call_line_synth
268 if func in _single_arg_builtins:
269 return lambda s, l, t: func(t)
271 limit = 0
272 found_arity = False
274 # synthesize what would be returned by traceback.extract_stack at the call to
275 # user's parse action 'func', so that we don't incur call penalty at parse time
277 # fmt: off
278 LINE_DIFF = 9
279 # IF ANY CODE CHANGES, EVEN JUST COMMENTS OR BLANK LINES, BETWEEN THE NEXT LINE AND
280 # THE CALL TO FUNC INSIDE WRAPPER, LINE_DIFF MUST BE MODIFIED!!!!
281 _trim_arity_call_line = _trim_arity_call_line or traceback.extract_stack(limit=2)[-1]
282 pa_call_line_synth = pa_call_line_synth or (_trim_arity_call_line[0], _trim_arity_call_line[1] + LINE_DIFF)
284 def wrapper(*args):
285 nonlocal found_arity, limit
286 if found_arity:
287 return func(*args[limit:])
288 while 1:
289 try:
290 ret = func(*args[limit:])
291 found_arity = True
292 return ret
293 except TypeError as te:
294 # re-raise TypeErrors if they did not come from our arity testing
295 if found_arity:
296 raise
297 else:
298 tb = te.__traceback__
299 frames = traceback.extract_tb(tb, limit=2)
300 frame_summary = frames[-1]
301 trim_arity_type_error = (
302 [frame_summary[:2]][-1][:2] == pa_call_line_synth
303 )
304 del tb
306 if trim_arity_type_error:
307 if limit < max_limit:
308 limit += 1
309 continue
311 raise
312 except IndexError as ie:
313 # wrap IndexErrors inside a _ParseActionIndexError
314 raise _ParseActionIndexError(
315 "IndexError raised in parse action", ie
316 ).with_traceback(None)
317 # fmt: on
319 # copy func name to wrapper for sensible debug output
320 # (can't use functools.wraps, since that messes with function signature)
321 func_name = getattr(func, "__name__", getattr(func, "__class__").__name__)
322 wrapper.__name__ = func_name
323 wrapper.__doc__ = func.__doc__
325 return wrapper
328def condition_as_parse_action(
329 fn: ParseCondition, message: typing.Optional[str] = None, fatal: bool = False
330) -> ParseAction:
331 """
332 Function to convert a simple predicate function that returns ``True`` or ``False``
333 into a parse action. Can be used in places when a parse action is required
334 and :meth:`ParserElement.add_condition` cannot be used (such as when adding a condition
335 to an operator level in :class:`infix_notation`).
337 Optional keyword arguments:
339 :param message: define a custom message to be used in the raised exception
340 :param fatal: if ``True``, will raise :class:`ParseFatalException`
341 to stop parsing immediately;
342 otherwise will raise :class:`ParseException`
344 """
345 msg = message if message is not None else "failed user-defined condition"
346 exc_type = ParseFatalException if fatal else ParseException
347 fn = _trim_arity(fn)
349 @wraps(fn)
350 def pa(s, l, t):
351 if not bool(fn(s, l, t)):
352 raise exc_type(s, l, msg)
354 return pa
357def _default_start_debug_action(
358 instring: str, loc: int, expr: ParserElement, cache_hit: bool = False
359):
360 cache_hit_str = "*" if cache_hit else ""
361 print(
362 (
363 f"{cache_hit_str}Match {expr} at loc {loc}({lineno(loc, instring)},{col(loc, instring)})\n"
364 f" {line(loc, instring)}\n"
365 f" {'^':>{col(loc, instring)}}"
366 )
367 )
370def _default_success_debug_action(
371 instring: str,
372 startloc: int,
373 endloc: int,
374 expr: ParserElement,
375 toks: ParseResults,
376 cache_hit: bool = False,
377):
378 cache_hit_str = "*" if cache_hit else ""
379 print(f"{cache_hit_str}Matched {expr} -> {toks.as_list()}")
382def _default_exception_debug_action(
383 instring: str,
384 loc: int,
385 expr: ParserElement,
386 exc: Exception,
387 cache_hit: bool = False,
388):
389 cache_hit_str = "*" if cache_hit else ""
390 print(f"{cache_hit_str}Match {expr} failed, {type(exc).__name__} raised: {exc}")
393def null_debug_action(*args):
394 """'Do-nothing' debug action, to suppress debugging output during parsing."""
397class ParserElement(ABC):
398 """Abstract base level parser element class."""
400 DEFAULT_WHITE_CHARS: str = " \n\t\r"
401 verbose_stacktrace: bool = False
402 _literalStringClass: type = None # type: ignore[assignment]
404 @staticmethod
405 def set_default_whitespace_chars(chars: str) -> None:
406 r"""
407 Overrides the default whitespace chars
409 Example:
411 .. doctest::
413 # default whitespace chars are space, <TAB> and newline
414 >>> Word(alphas)[1, ...].parse_string("abc def\nghi jkl")
415 ParseResults(['abc', 'def', 'ghi', 'jkl'], {})
417 # change to just treat newline as significant
418 >>> ParserElement.set_default_whitespace_chars(" \t")
419 >>> Word(alphas)[1, ...].parse_string("abc def\nghi jkl")
420 ParseResults(['abc', 'def'], {})
422 # Reset to default
423 >>> ParserElement.set_default_whitespace_chars(" \n\t\r")
424 """
425 ParserElement.DEFAULT_WHITE_CHARS = chars
427 # update whitespace all parse expressions defined in this module
428 for expr in _builtin_exprs:
429 if expr.copyDefaultWhiteChars:
430 expr.whiteChars = set(chars)
432 @staticmethod
433 def inline_literals_using(cls: type) -> None:
434 """
435 Set class to be used for inclusion of string literals into a parser.
437 Example:
439 .. doctest::
440 :options: +NORMALIZE_WHITESPACE
442 # default literal class used is Literal
443 >>> integer = Word(nums)
444 >>> date_str = (
445 ... integer("year") + '/'
446 ... + integer("month") + '/'
447 ... + integer("day")
448 ... )
450 >>> date_str.parse_string("1999/12/31")
451 ParseResults(['1999', '/', '12', '/', '31'],
452 {'year': '1999', 'month': '12', 'day': '31'})
454 # change to Suppress
455 >>> ParserElement.inline_literals_using(Suppress)
456 >>> date_str = (
457 ... integer("year") + '/'
458 ... + integer("month") + '/'
459 ... + integer("day")
460 ... )
462 >>> date_str.parse_string("1999/12/31")
463 ParseResults(['1999', '12', '31'],
464 {'year': '1999', 'month': '12', 'day': '31'})
466 # Reset
467 >>> ParserElement.inline_literals_using(Literal)
468 """
469 ParserElement._literalStringClass = cls
471 @classmethod
472 def using_each(cls, seq, **class_kwargs):
473 """
474 Yields a sequence of ``class(obj, **class_kwargs)`` for obj in seq.
476 Example:
478 .. testcode::
480 LPAR, RPAR, LBRACE, RBRACE, SEMI = Suppress.using_each("(){};")
482 .. versionadded:: 3.1.0
483 """
484 yield from (cls(obj, **class_kwargs) for obj in seq)
486 class DebugActions(NamedTuple):
487 debug_try: typing.Optional[DebugStartAction]
488 debug_match: typing.Optional[DebugSuccessAction]
489 debug_fail: typing.Optional[DebugExceptionAction]
491 def __init__(self, savelist: bool = False) -> None:
492 self.parseAction: list[ParseAction] = list()
493 self.failAction: typing.Optional[ParseFailAction] = None
494 self.customName: str = None # type: ignore[assignment]
495 self._defaultName: typing.Optional[str] = None
496 self.resultsName: str = None # type: ignore[assignment]
497 self.saveAsList: bool = savelist
498 self.skipWhitespace: bool = True
499 self.whiteChars: set[str] = set(ParserElement.DEFAULT_WHITE_CHARS)
500 self.copyDefaultWhiteChars: bool = True
501 # used when checking for left-recursion
502 self._may_return_empty: bool = False
503 self.keepTabs: bool = False
504 self.ignoreExprs: list[ParserElement] = list()
505 self.debug: bool = False
506 self.streamlined: bool = False
507 # optimize exception handling for subclasses that don't advance parse index
508 self.mayIndexError: bool = True
509 self.errmsg: Union[str, None] = ""
510 # mark results names as modal (report only last) or cumulative (list all)
511 self.modalResults: bool = True
512 # custom debug actions
513 self.debugActions = self.DebugActions(None, None, None)
514 # avoid redundant calls to preParse
515 self.callPreparse: bool = True
516 self.callDuringTry: bool = False
517 self.suppress_warnings_: list[Diagnostics] = []
518 self.show_in_diagram: bool = True
520 @property
521 def mayReturnEmpty(self) -> bool:
522 """
523 .. deprecated:: 3.3.0
524 use _may_return_empty instead.
525 """
526 return self._may_return_empty
528 @mayReturnEmpty.setter
529 def mayReturnEmpty(self, value) -> None:
530 """
531 .. deprecated:: 3.3.0
532 use _may_return_empty instead.
533 """
534 self._may_return_empty = value
536 def suppress_warning(self, warning_type: Diagnostics) -> ParserElement:
537 """
538 Suppress warnings emitted for a particular diagnostic on this expression.
540 Example:
542 .. doctest::
544 >>> label = pp.Word(pp.alphas)
546 # Normally using an empty Forward in a grammar
547 # would print a warning, but we can suppress that
548 >>> base = pp.Forward().suppress_warning(
549 ... pp.Diagnostics.warn_on_parse_using_empty_Forward)
551 >>> grammar = base | label
552 >>> print(grammar.parse_string("x"))
553 ['x']
554 """
555 self.suppress_warnings_.append(warning_type)
556 return self
558 def visit_all(self):
559 """General-purpose method to yield all expressions and sub-expressions
560 in a grammar. Typically just for internal use.
561 """
562 to_visit = deque([self])
563 seen = set()
564 while to_visit:
565 cur = to_visit.popleft()
567 # guard against looping forever through recursive grammars
568 if cur in seen:
569 continue
570 seen.add(cur)
572 to_visit.extend(cur.recurse())
573 yield cur
575 def copy(self) -> ParserElement:
576 """
577 Make a copy of this :class:`ParserElement`. Useful for defining
578 different parse actions for the same parsing pattern, using copies of
579 the original parse element.
581 Example:
583 .. testcode::
585 integer = Word(nums).set_parse_action(
586 lambda toks: int(toks[0]))
587 integerK = integer.copy().add_parse_action(
588 lambda toks: toks[0] * 1024) + Suppress("K")
589 integerM = integer.copy().add_parse_action(
590 lambda toks: toks[0] * 1024 * 1024) + Suppress("M")
592 print(
593 (integerK | integerM | integer)[1, ...].parse_string(
594 "5K 100 640K 256M")
595 )
597 prints:
599 .. testoutput::
601 [5120, 100, 655360, 268435456]
603 Equivalent form of ``expr.copy()`` is just ``expr()``:
605 .. testcode::
607 integerM = integer().add_parse_action(
608 lambda toks: toks[0] * 1024 * 1024) + Suppress("M")
609 """
610 cpy = copy.copy(self)
611 cpy.parseAction = self.parseAction[:]
612 cpy.ignoreExprs = self.ignoreExprs[:]
613 if self.copyDefaultWhiteChars:
614 cpy.whiteChars = set(ParserElement.DEFAULT_WHITE_CHARS)
615 return cpy
617 def set_results_name(
618 self, name: str, list_all_matches: bool = False, **kwargs
619 ) -> ParserElement:
620 """
621 Define name for referencing matching tokens as a nested attribute
622 of the returned parse results.
624 Normally, results names are assigned as you would assign keys in a dict:
625 any existing value is overwritten by later values. If it is necessary to
626 keep all values captured for a particular results name, call ``set_results_name``
627 with ``list_all_matches`` = True.
629 NOTE: ``set_results_name`` returns a *copy* of the original :class:`ParserElement` object;
630 this is so that the client can define a basic element, such as an
631 integer, and reference it in multiple places with different names.
633 You can also set results names using the abbreviated syntax,
634 ``expr("name")`` in place of ``expr.set_results_name("name")``
635 - see :meth:`__call__`. If ``list_all_matches`` is required, use
636 ``expr("name*")``.
638 Example:
640 .. testcode::
642 integer = Word(nums)
643 date_str = (integer.set_results_name("year") + '/'
644 + integer.set_results_name("month") + '/'
645 + integer.set_results_name("day"))
647 # equivalent form:
648 date_str = integer("year") + '/' + integer("month") + '/' + integer("day")
649 """
650 listAllMatches: bool = deprecate_argument(kwargs, "listAllMatches", False)
652 list_all_matches = listAllMatches or list_all_matches
653 return self._setResultsName(name, list_all_matches)
655 def _setResultsName(self, name, list_all_matches=False) -> ParserElement:
656 if name is None:
657 return self
658 newself = self.copy()
659 if name.endswith("*"):
660 name = name[:-1]
661 list_all_matches = True
662 newself.resultsName = name
663 newself.modalResults = not list_all_matches
664 return newself
666 def set_break(self, break_flag: bool = True) -> ParserElement:
667 """
668 Method to invoke the Python pdb debugger when this element is
669 about to be parsed. Set ``break_flag`` to ``True`` to enable, ``False`` to
670 disable.
671 """
672 if break_flag:
673 _parseMethod = self._parse
675 def breaker(instring, loc, do_actions=True, callPreParse=True):
676 # this call to breakpoint() is intentional, not a checkin error
677 breakpoint()
678 return _parseMethod(instring, loc, do_actions, callPreParse)
680 breaker._originalParseMethod = _parseMethod # type: ignore [attr-defined]
681 self._parse = breaker # type: ignore [method-assign]
682 elif hasattr(self._parse, "_originalParseMethod"):
683 self._parse = self._parse._originalParseMethod # type: ignore [method-assign]
684 return self
686 def set_parse_action(
687 self, *fns: ParseAction, call_during_try: bool = False, **kwargs: Any
688 ) -> ParserElement:
689 """
690 Define one or more actions to perform when successfully matching parse element definition.
692 Parse actions can be called to perform data conversions, do extra validation,
693 update external data structures, or enhance or replace the parsed tokens.
694 Each parse action ``fn`` is a callable method with 0-3 arguments, called as
695 ``fn(s, loc, toks)`` , ``fn(loc, toks)`` , ``fn(toks)`` , or just ``fn()`` , where:
697 - ``s`` = the original string being parsed (see note below)
698 - ``loc`` = the location of the matching substring
699 - ``toks`` = a list of the matched tokens, packaged as a :class:`ParseResults` object
701 The parsed tokens are passed to the parse action as ParseResults. They can be
702 modified in place using list-style append, extend, and pop operations to update
703 the parsed list elements; and with dictionary-style item set and del operations
704 to add, update, or remove any named results. If the tokens are modified in place,
705 it is not necessary to return them with a return statement.
707 Parse actions can also completely replace the given tokens, with another ``ParseResults``
708 object, or with some entirely different object (common for parse actions that perform data
709 conversions). A convenient way to build a new parse result is to define the values
710 using a dict, and then create the return value using :class:`ParseResults.from_dict`.
712 If None is passed as the ``fn`` parse action, all previously added parse actions for this
713 expression are cleared.
715 Optional keyword arguments:
717 :param call_during_try: (default= ``False``) indicate if parse action
718 should be run during lookaheads and alternate
719 testing. For parse actions that have side
720 effects, it is important to only call the parse
721 action once it is determined that it is being
722 called as part of a successful parse.
723 For parse actions that perform additional
724 validation, then ``call_during_try`` should
725 be passed as True, so that the validation code
726 is included in the preliminary "try" parses.
728 .. Note::
729 The default parsing behavior is to expand tabs in the input string
730 before starting the parsing process.
731 See :meth:`parse_string` for more information on parsing strings
732 containing ``<TAB>`` s, and suggested methods to maintain a
733 consistent view of the parsed string, the parse location, and
734 line and column positions within the parsed string.
736 Example: Parse dates in the form ``YYYY/MM/DD``
737 -----------------------------------------------
739 Setup code:
741 .. testcode::
743 def convert_to_int(toks):
744 '''a parse action to convert toks from str to int
745 at parse time'''
746 return int(toks[0])
748 def is_valid_date(instring, loc, toks):
749 '''a parse action to verify that the date is a valid date'''
750 from datetime import date
751 year, month, day = toks[::2]
752 try:
753 date(year, month, day)
754 except ValueError:
755 raise ParseException(instring, loc, "invalid date given")
757 integer = Word(nums)
758 date_str = integer + '/' + integer + '/' + integer
760 # add parse actions
761 integer.set_parse_action(convert_to_int)
762 date_str.set_parse_action(is_valid_date)
764 Successful parse - note that integer fields are converted to ints:
766 .. testcode::
768 print(date_str.parse_string("1999/12/31"))
770 prints:
772 .. testoutput::
774 [1999, '/', 12, '/', 31]
776 Failure - invalid date:
778 .. testcode::
780 date_str.parse_string("1999/13/31")
782 prints:
784 .. testoutput::
786 Traceback (most recent call last):
787 ParseException: invalid date given, found '1999' ...
788 """
789 callDuringTry: bool = deprecate_argument(kwargs, "callDuringTry", False)
791 if list(fns) == [None]:
792 self.parseAction.clear()
793 return self
795 if not all(callable(fn) for fn in fns):
796 raise TypeError("parse actions must be callable")
797 self.parseAction[:] = [_trim_arity(fn) for fn in fns]
798 self.callDuringTry = self.callDuringTry or call_during_try or callDuringTry
800 return self
802 def add_parse_action(
803 self, *fns: ParseAction, call_during_try: bool = False, **kwargs: Any
804 ) -> ParserElement:
805 """
806 Add one or more parse actions to expression's list of parse actions. See :class:`set_parse_action`.
808 See examples in :class:`copy`.
809 """
810 callDuringTry: bool = deprecate_argument(kwargs, "callDuringTry", False)
812 self.parseAction += [_trim_arity(fn) for fn in fns]
813 self.callDuringTry = self.callDuringTry or callDuringTry or call_during_try
814 return self
816 def add_condition(
817 self, *fns: ParseCondition, call_during_try: bool = False, **kwargs: Any
818 ) -> ParserElement:
819 """Add a boolean predicate function to expression's list of parse actions. See
820 :class:`set_parse_action` for function call signatures. Unlike ``set_parse_action``,
821 functions passed to ``add_condition`` need to return boolean success/fail of the condition.
823 Optional keyword arguments:
825 - ``message`` = define a custom message to be used in the raised exception
826 - ``fatal`` = if True, will raise ParseFatalException to stop parsing immediately; otherwise will raise
827 ParseException
828 - ``call_during_try`` = boolean to indicate if this method should be called during internal tryParse calls,
829 default=False
831 Example:
833 .. doctest::
834 :options: +NORMALIZE_WHITESPACE
836 >>> integer = Word(nums).set_parse_action(lambda toks: int(toks[0]))
837 >>> year_int = integer.copy().add_condition(
838 ... lambda toks: toks[0] >= 2000,
839 ... message="Only support years 2000 and later")
840 >>> date_str = year_int + '/' + integer + '/' + integer
842 >>> result = date_str.parse_string("1999/12/31")
843 Traceback (most recent call last):
844 ParseException: Only support years 2000 and later...
845 """
846 callDuringTry: bool = deprecate_argument(kwargs, "callDuringTry", False)
848 for fn in fns:
849 self.parseAction.append(
850 condition_as_parse_action(
851 fn,
852 message=str(kwargs.get("message")),
853 fatal=bool(kwargs.get("fatal", False)),
854 )
855 )
857 self.callDuringTry = self.callDuringTry or call_during_try or callDuringTry
858 return self
860 def set_fail_action(self, fn: ParseFailAction) -> ParserElement:
861 """
862 Define action to perform if parsing fails at this expression.
863 Fail acton fn is a callable function that takes the arguments
864 ``fn(s, loc, expr, err)`` where:
866 - ``s`` = string being parsed
867 - ``loc`` = location where expression match was attempted and failed
868 - ``expr`` = the parse expression that failed
869 - ``err`` = the exception thrown
871 The function returns no value. It may throw :class:`ParseFatalException`
872 if it is desired to stop parsing immediately."""
873 self.failAction = fn
874 return self
876 def _skipIgnorables(self, instring: str, loc: int) -> int:
877 if not self.ignoreExprs:
878 return loc
879 exprsFound = True
880 ignore_expr_fns = [e._parse for e in self.ignoreExprs]
881 last_loc = loc
882 while exprsFound:
883 exprsFound = False
884 for ignore_fn in ignore_expr_fns:
885 try:
886 while 1:
887 loc, dummy = ignore_fn(instring, loc)
888 exprsFound = True
889 except ParseException:
890 pass
891 # check if all ignore exprs matched but didn't actually advance the parse location
892 if loc == last_loc:
893 break
894 last_loc = loc
895 return loc
897 def preParse(self, instring: str, loc: int) -> int:
898 if self.ignoreExprs:
899 loc = self._skipIgnorables(instring, loc)
901 if self.skipWhitespace:
902 instrlen = len(instring)
903 white_chars = self.whiteChars
904 while loc < instrlen and instring[loc] in white_chars:
905 loc += 1
907 return loc
909 def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType:
910 return loc, []
912 def postParse(self, instring, loc, tokenlist):
913 return tokenlist
915 # @profile
916 def _parseNoCache(
917 self, instring, loc, do_actions=True, callPreParse=True
918 ) -> tuple[int, ParseResults]:
919 debugging = self.debug # and do_actions)
920 len_instring = len(instring)
922 if debugging or self.failAction:
923 # print("Match {} at loc {}({}, {})".format(self, loc, lineno(loc, instring), col(loc, instring)))
924 try:
925 if callPreParse and self.callPreparse:
926 pre_loc = self.preParse(instring, loc)
927 else:
928 pre_loc = loc
929 tokens_start = pre_loc
930 if self.debugActions.debug_try:
931 self.debugActions.debug_try(instring, tokens_start, self, False)
932 if self.mayIndexError or pre_loc >= len_instring:
933 try:
934 loc, tokens = self.parseImpl(instring, pre_loc, do_actions)
935 except IndexError:
936 raise ParseException(instring, len_instring, self.errmsg, self)
937 else:
938 loc, tokens = self.parseImpl(instring, pre_loc, do_actions)
939 except Exception as err:
940 # print("Exception raised:", err)
941 if self.debugActions.debug_fail:
942 self.debugActions.debug_fail(
943 instring, tokens_start, self, err, False
944 )
945 if self.failAction:
946 self.failAction(instring, tokens_start, self, err)
947 raise
948 else:
949 if callPreParse and self.callPreparse:
950 pre_loc = self.preParse(instring, loc)
951 else:
952 pre_loc = loc
953 tokens_start = pre_loc
954 if self.mayIndexError or pre_loc >= len_instring:
955 try:
956 loc, tokens = self.parseImpl(instring, pre_loc, do_actions)
957 except IndexError:
958 raise ParseException(instring, len_instring, self.errmsg, self)
959 else:
960 loc, tokens = self.parseImpl(instring, pre_loc, do_actions)
962 tokens = self.postParse(instring, loc, tokens)
964 ret_tokens = ParseResults(
965 tokens, self.resultsName, aslist=self.saveAsList, modal=self.modalResults
966 )
967 if self.parseAction and (do_actions or self.callDuringTry):
968 if debugging:
969 try:
970 for fn in self.parseAction:
971 try:
972 tokens = fn(instring, tokens_start, ret_tokens) # type: ignore [call-arg, arg-type]
973 except IndexError as parse_action_exc:
974 exc = ParseException("exception raised in parse action")
975 raise exc from parse_action_exc
977 if tokens is not None and tokens is not ret_tokens:
978 ret_tokens = ParseResults(
979 tokens,
980 self.resultsName,
981 aslist=self.saveAsList
982 and isinstance(tokens, (ParseResults, list)),
983 modal=self.modalResults,
984 )
985 except Exception as err:
986 # print "Exception raised in user parse action:", err
987 if self.debugActions.debug_fail:
988 self.debugActions.debug_fail(
989 instring, tokens_start, self, err, False
990 )
991 raise
992 else:
993 for fn in self.parseAction:
994 try:
995 tokens = fn(instring, tokens_start, ret_tokens) # type: ignore [call-arg, arg-type]
996 except IndexError as parse_action_exc:
997 exc = ParseException("exception raised in parse action")
998 raise exc from parse_action_exc
1000 if tokens is not None and tokens is not ret_tokens:
1001 ret_tokens = ParseResults(
1002 tokens,
1003 self.resultsName,
1004 aslist=self.saveAsList
1005 and isinstance(tokens, (ParseResults, list)),
1006 modal=self.modalResults,
1007 )
1008 if debugging:
1009 # print("Matched", self, "->", ret_tokens.as_list())
1010 if self.debugActions.debug_match:
1011 self.debugActions.debug_match(
1012 instring, tokens_start, loc, self, ret_tokens, False
1013 )
1015 return loc, ret_tokens
1017 def try_parse(
1018 self,
1019 instring: str,
1020 loc: int,
1021 *,
1022 raise_fatal: bool = False,
1023 do_actions: bool = False,
1024 ) -> int:
1025 try:
1026 return self._parse(instring, loc, do_actions=do_actions)[0]
1027 except ParseFatalException:
1028 if raise_fatal:
1029 raise
1030 raise ParseException(instring, loc, self.errmsg, self)
1032 def can_parse_next(self, instring: str, loc: int, do_actions: bool = False) -> bool:
1033 try:
1034 self.try_parse(instring, loc, do_actions=do_actions)
1035 except (ParseException, IndexError):
1036 return False
1037 else:
1038 return True
1040 # cache for left-recursion in Forward references
1041 recursion_lock = RLock()
1042 recursion_memos: collections.abc.MutableMapping[
1043 tuple[int, Forward, bool], tuple[int, Union[ParseResults, Exception]]
1044 ] = {}
1046 class _CacheType(typing.Protocol):
1047 """
1048 Class to be used for packrat and left-recursion cacheing of results
1049 and exceptions.
1050 """
1052 not_in_cache: bool
1054 def get(self, *args) -> typing.Any: ...
1056 def set(self, *args) -> None: ...
1058 def clear(self) -> None: ...
1060 class NullCache(dict):
1061 """
1062 A null cache type for initialization of the packrat_cache class variable.
1063 If/when enable_packrat() is called, this null cache will be replaced by a
1064 proper _CacheType class instance.
1065 """
1067 not_in_cache: bool = True
1069 def get(self, *args) -> typing.Any: ...
1071 def set(self, *args) -> None: ...
1073 def clear(self) -> None: ...
1075 # class-level argument cache for optimizing repeated calls when backtracking
1076 # through recursive expressions
1077 packrat_cache: _CacheType = NullCache()
1078 packrat_cache_lock = RLock()
1079 packrat_cache_stats = [0, 0]
1081 # this method gets repeatedly called during backtracking with the same arguments -
1082 # we can cache these arguments and save ourselves the trouble of re-parsing the contained expression
1083 def _parseCache(
1084 self, instring, loc, do_actions=True, callPreParse=True
1085 ) -> tuple[int, ParseResults]:
1086 HIT, MISS = 0, 1
1087 lookup = (self, instring, loc, callPreParse, do_actions)
1088 with ParserElement.packrat_cache_lock:
1089 cache = ParserElement.packrat_cache
1090 value = cache.get(lookup)
1091 if value is cache.not_in_cache:
1092 ParserElement.packrat_cache_stats[MISS] += 1
1093 try:
1094 value = self._parseNoCache(instring, loc, do_actions, callPreParse)
1095 except ParseBaseException as pe:
1096 # cache a copy of the exception, without the traceback
1097 cache.set(lookup, pe.__class__(*pe.args))
1098 raise
1099 else:
1100 cache.set(lookup, (value[0], value[1].copy(), loc))
1101 return value
1102 else:
1103 ParserElement.packrat_cache_stats[HIT] += 1
1104 if self.debug and self.debugActions.debug_try:
1105 try:
1106 self.debugActions.debug_try(instring, loc, self, cache_hit=True) # type: ignore [call-arg]
1107 except TypeError:
1108 pass
1109 if isinstance(value, Exception):
1110 if self.debug and self.debugActions.debug_fail:
1111 try:
1112 self.debugActions.debug_fail(
1113 instring, loc, self, value, cache_hit=True # type: ignore [call-arg]
1114 )
1115 except TypeError:
1116 pass
1117 raise value
1119 value = cast(tuple[int, ParseResults, int], value)
1120 loc_, result, endloc = value[0], value[1].copy(), value[2]
1121 if self.debug and self.debugActions.debug_match:
1122 try:
1123 self.debugActions.debug_match(
1124 instring, loc_, endloc, self, result, cache_hit=True # type: ignore [call-arg]
1125 )
1126 except TypeError:
1127 pass
1129 return loc_, result
1131 _parse = _parseNoCache
1133 @staticmethod
1134 def reset_cache() -> None:
1135 """
1136 Clears caches used by packrat and left-recursion.
1137 """
1138 with ParserElement.packrat_cache_lock:
1139 ParserElement.packrat_cache.clear()
1140 ParserElement.packrat_cache_stats[:] = [0] * len(
1141 ParserElement.packrat_cache_stats
1142 )
1143 ParserElement.recursion_memos.clear()
1145 # class attributes to keep caching status
1146 _packratEnabled = False
1147 _left_recursion_enabled = False
1149 @staticmethod
1150 def disable_memoization() -> None:
1151 """
1152 Disables active Packrat or Left Recursion parsing and their memoization
1154 This method also works if neither Packrat nor Left Recursion are enabled.
1155 This makes it safe to call before activating Packrat nor Left Recursion
1156 to clear any previous settings.
1157 """
1158 with ParserElement.packrat_cache_lock:
1159 ParserElement.reset_cache()
1160 ParserElement._left_recursion_enabled = False
1161 ParserElement._packratEnabled = False
1162 ParserElement._parse = ParserElement._parseNoCache
1164 @staticmethod
1165 def enable_left_recursion(
1166 cache_size_limit: typing.Optional[int] = None, *, force=False
1167 ) -> None:
1168 """
1169 Enables "bounded recursion" parsing, which allows for both direct and indirect
1170 left-recursion. During parsing, left-recursive :class:`Forward` elements are
1171 repeatedly matched with a fixed recursion depth that is gradually increased
1172 until finding the longest match.
1174 Example:
1176 .. testcode::
1178 import pyparsing as pp
1179 pp.ParserElement.enable_left_recursion()
1181 E = pp.Forward("E")
1182 num = pp.Word(pp.nums)
1184 # match `num`, or `num '+' num`, or `num '+' num '+' num`, ...
1185 E <<= E + '+' - num | num
1187 print(E.parse_string("1+2+3+4"))
1189 prints:
1191 .. testoutput::
1193 ['1', '+', '2', '+', '3', '+', '4']
1195 Recursion search naturally memoizes matches of ``Forward`` elements and may
1196 thus skip reevaluation of parse actions during backtracking. This may break
1197 programs with parse actions which rely on strict ordering of side-effects.
1199 Parameters:
1201 - ``cache_size_limit`` - (default=``None``) - memoize at most this many
1202 ``Forward`` elements during matching; if ``None`` (the default),
1203 memoize all ``Forward`` elements.
1205 Bounded Recursion parsing works similar but not identical to Packrat parsing,
1206 thus the two cannot be used together. Use ``force=True`` to disable any
1207 previous, conflicting settings.
1208 """
1209 with ParserElement.packrat_cache_lock:
1210 if force:
1211 ParserElement.disable_memoization()
1212 elif ParserElement._packratEnabled:
1213 raise RuntimeError("Packrat and Bounded Recursion are not compatible")
1214 if cache_size_limit is None:
1215 ParserElement.recursion_memos = _UnboundedMemo()
1216 elif cache_size_limit > 0:
1217 ParserElement.recursion_memos = _LRUMemo(capacity=cache_size_limit) # type: ignore[assignment]
1218 else:
1219 raise NotImplementedError(f"Memo size of {cache_size_limit}")
1220 ParserElement._left_recursion_enabled = True
1222 @staticmethod
1223 def enable_packrat(
1224 cache_size_limit: Union[int, None] = 128, *, force: bool = False
1225 ) -> None:
1226 """
1227 Enables "packrat" parsing, which adds memoizing to the parsing logic.
1228 Repeated parse attempts at the same string location (which happens
1229 often in many complex grammars) can immediately return a cached value,
1230 instead of re-executing parsing/validating code. Memoizing is done of
1231 both valid results and parsing exceptions.
1233 Parameters:
1235 - ``cache_size_limit`` - (default= ``128``) - if an integer value is provided
1236 will limit the size of the packrat cache; if None is passed, then
1237 the cache size will be unbounded; if 0 is passed, the cache will
1238 be effectively disabled.
1240 This speedup may break existing programs that use parse actions that
1241 have side-effects. For this reason, packrat parsing is disabled when
1242 you first import pyparsing. To activate the packrat feature, your
1243 program must call the class method :class:`ParserElement.enable_packrat`.
1244 For best results, call ``enable_packrat()`` immediately after
1245 importing pyparsing.
1247 .. Can't really be doctested, alas
1249 Example::
1251 import pyparsing
1252 pyparsing.ParserElement.enable_packrat()
1254 Packrat parsing works similar but not identical to Bounded Recursion parsing,
1255 thus the two cannot be used together. Use ``force=True`` to disable any
1256 previous, conflicting settings.
1257 """
1258 with ParserElement.packrat_cache_lock:
1259 if force:
1260 ParserElement.disable_memoization()
1261 elif ParserElement._left_recursion_enabled:
1262 raise RuntimeError("Packrat and Bounded Recursion are not compatible")
1264 if ParserElement._packratEnabled:
1265 return
1267 ParserElement._packratEnabled = True
1268 if cache_size_limit is None:
1269 ParserElement.packrat_cache = _UnboundedCache()
1270 else:
1271 ParserElement.packrat_cache = _FifoCache(cache_size_limit)
1272 ParserElement._parse = ParserElement._parseCache
1274 def parse_string(
1275 self, instring: str, parse_all: bool = False, **kwargs
1276 ) -> ParseResults:
1277 """
1278 Parse a string with respect to the parser definition. This function is intended as the primary interface to the
1279 client code.
1281 :param instring: The input string to be parsed.
1282 :param parse_all: If set, the entire input string must match the grammar.
1283 :param parseAll: retained for pre-PEP8 compatibility, will be removed in a future release.
1284 :raises ParseException: Raised if ``parse_all`` is set and the input string does not match the whole grammar.
1285 :returns: the parsed data as a :class:`ParseResults` object, which may be accessed as a `list`, a `dict`, or
1286 an object with attributes if the given parser includes results names.
1288 If the input string is required to match the entire grammar, ``parse_all`` flag must be set to ``True``. This
1289 is also equivalent to ending the grammar with :class:`StringEnd`\\ ().
1291 To report proper column numbers, ``parse_string`` operates on a copy of the input string where all tabs are
1292 converted to spaces (8 spaces per tab, as per the default in ``string.expandtabs``). If the input string
1293 contains tabs and the grammar uses parse actions that use the ``loc`` argument to index into the string
1294 being parsed, one can ensure a consistent view of the input string by doing one of the following:
1296 - calling ``parse_with_tabs`` on your grammar before calling ``parse_string`` (see :class:`parse_with_tabs`),
1297 - define your parse action using the full ``(s,loc,toks)`` signature, and reference the input string using the
1298 parse action's ``s`` argument, or
1299 - explicitly expand the tabs in your input string before calling ``parse_string``.
1301 Examples:
1303 By default, partial matches are OK.
1305 .. doctest::
1307 >>> res = Word('a').parse_string('aaaaabaaa')
1308 >>> print(res)
1309 ['aaaaa']
1311 The parsing behavior varies by the inheriting class of this abstract class. Please refer to the children
1312 directly to see more examples.
1314 It raises an exception if parse_all flag is set and instring does not match the whole grammar.
1316 .. doctest::
1318 >>> res = Word('a').parse_string('aaaaabaaa', parse_all=True)
1319 Traceback (most recent call last):
1320 ParseException: Expected end of text, found 'b' ...
1321 """
1322 parseAll: bool = deprecate_argument(kwargs, "parseAll", False)
1324 parse_all = parse_all or parseAll
1326 ParserElement.reset_cache()
1327 if not self.streamlined:
1328 self.streamline()
1329 for e in self.ignoreExprs:
1330 e.streamline()
1331 if not self.keepTabs:
1332 instring = instring.expandtabs()
1333 try:
1334 loc, tokens = self._parse(instring, 0)
1335 if parse_all:
1336 loc = self.preParse(instring, loc)
1337 se = Empty() + StringEnd().set_debug(False)
1338 se._parse(instring, loc)
1339 except _ParseActionIndexError as pa_exc:
1340 raise pa_exc.exc
1341 except ParseBaseException as exc:
1342 if ParserElement.verbose_stacktrace:
1343 raise
1345 # catch and re-raise exception from here, clearing out pyparsing internal stack trace
1346 raise exc.with_traceback(None)
1347 else:
1348 return tokens
1350 def scan_string(
1351 self,
1352 instring: str,
1353 max_matches: int = _MAX_INT,
1354 overlap: bool = False,
1355 always_skip_whitespace=True,
1356 *,
1357 debug: bool = False,
1358 **kwargs,
1359 ) -> Generator[tuple[ParseResults, int, int], None, None]:
1360 """
1361 Scan the input string for expression matches. Each match will return the
1362 matching tokens, start location, and end location. May be called with optional
1363 ``max_matches`` argument, to clip scanning after 'n' matches are found. If
1364 ``overlap`` is specified, then overlapping matches will be reported.
1366 Note that the start and end locations are reported relative to the string
1367 being parsed. See :class:`parse_string` for more information on parsing
1368 strings with embedded tabs.
1370 Example:
1372 .. testcode::
1374 source = "sldjf123lsdjjkf345sldkjf879lkjsfd987"
1375 print(source)
1376 for tokens, start, end in Word(alphas).scan_string(source):
1377 print(' '*start + '^'*(end-start))
1378 print(' '*start + tokens[0])
1380 prints:
1382 .. testoutput::
1384 sldjf123lsdjjkf345sldkjf879lkjsfd987
1385 ^^^^^
1386 sldjf
1387 ^^^^^^^
1388 lsdjjkf
1389 ^^^^^^
1390 sldkjf
1391 ^^^^^^
1392 lkjsfd
1393 """
1394 maxMatches: int = deprecate_argument(kwargs, "maxMatches", _MAX_INT)
1396 max_matches = min(maxMatches, max_matches)
1397 if not self.streamlined:
1398 self.streamline()
1399 for e in self.ignoreExprs:
1400 e.streamline()
1402 if not self.keepTabs:
1403 instring = str(instring).expandtabs()
1404 instrlen = len(instring)
1405 loc = 0
1406 if always_skip_whitespace:
1407 preparser = Empty()
1408 preparser.ignoreExprs = self.ignoreExprs
1409 preparser.whiteChars = self.whiteChars
1410 preparseFn = preparser.preParse
1411 else:
1412 preparseFn = self.preParse
1413 parseFn = self._parse
1414 ParserElement.reset_cache()
1415 matches = 0
1416 try:
1417 while loc <= instrlen and matches < max_matches:
1418 try:
1419 preloc: int = preparseFn(instring, loc)
1420 nextLoc: int
1421 tokens: ParseResults
1422 nextLoc, tokens = parseFn(instring, preloc, callPreParse=False)
1423 except ParseException:
1424 loc = preloc + 1
1425 else:
1426 if nextLoc > loc:
1427 matches += 1
1428 if debug:
1429 print(
1430 {
1431 "tokens": tokens.as_list(),
1432 "start": preloc,
1433 "end": nextLoc,
1434 }
1435 )
1436 yield tokens, preloc, nextLoc
1437 if overlap:
1438 nextloc = preparseFn(instring, loc)
1439 if nextloc > loc:
1440 loc = nextLoc
1441 else:
1442 loc += 1
1443 else:
1444 loc = nextLoc
1445 else:
1446 loc = preloc + 1
1447 except ParseBaseException as exc:
1448 if ParserElement.verbose_stacktrace:
1449 raise
1451 # catch and re-raise exception from here, clears out pyparsing internal stack trace
1452 raise exc.with_traceback(None)
1454 def transform_string(self, instring: str, *, debug: bool = False) -> str:
1455 """
1456 Extension to :class:`scan_string`, to modify matching text with modified tokens that may
1457 be returned from a parse action. To use ``transform_string``, define a grammar and
1458 attach a parse action to it that modifies the returned token list.
1459 Invoking ``transform_string()`` on a target string will then scan for matches,
1460 and replace the matched text patterns according to the logic in the parse
1461 action. ``transform_string()`` returns the resulting transformed string.
1463 Example:
1465 .. testcode::
1467 quote = '''now is the winter of our discontent,
1468 made glorious summer by this sun of york.'''
1470 wd = Word(alphas)
1471 wd.set_parse_action(lambda toks: toks[0].title())
1473 print(wd.transform_string(quote))
1475 prints:
1477 .. testoutput::
1479 Now Is The Winter Of Our Discontent,
1480 Made Glorious Summer By This Sun Of York.
1481 """
1482 out: list[str] = []
1483 lastE = 0
1484 # force preservation of <TAB>s, to minimize unwanted transformation of string, and to
1485 # keep string locs straight between transform_string and scan_string
1486 self.keepTabs = True
1487 try:
1488 for t, s, e in self.scan_string(instring, debug=debug):
1489 if s > lastE:
1490 out.append(instring[lastE:s])
1491 lastE = e
1493 if not t:
1494 continue
1496 if isinstance(t, ParseResults):
1497 out += t.as_list()
1498 elif isinstance(t, Iterable) and not isinstance(t, str_type):
1499 out.extend(t)
1500 else:
1501 out.append(t)
1503 out.append(instring[lastE:])
1504 out = [o for o in out if o]
1505 return "".join([str(s) for s in _flatten(out)])
1506 except ParseBaseException as exc:
1507 if ParserElement.verbose_stacktrace:
1508 raise
1510 # catch and re-raise exception from here, clears out pyparsing internal stack trace
1511 raise exc.with_traceback(None)
1513 def search_string(
1514 self,
1515 instring: str,
1516 max_matches: int = _MAX_INT,
1517 *,
1518 debug: bool = False,
1519 **kwargs,
1520 ) -> ParseResults:
1521 """
1522 Another extension to :class:`scan_string`, simplifying the access to the tokens found
1523 to match the given parse expression. May be called with optional
1524 ``max_matches`` argument, to clip searching after 'n' matches are found.
1526 Example:
1528 .. testcode::
1530 quote = '''More than Iron, more than Lead,
1531 more than Gold I need Electricity'''
1533 # a capitalized word starts with an uppercase letter,
1534 # followed by zero or more lowercase letters
1535 cap_word = Word(alphas.upper(), alphas.lower())
1537 print(cap_word.search_string(quote))
1539 # the sum() builtin can be used to merge results
1540 # into a single ParseResults object
1541 print(sum(cap_word.search_string(quote)))
1543 prints:
1545 .. testoutput::
1547 [['More'], ['Iron'], ['Lead'], ['Gold'], ['I'], ['Electricity']]
1548 ['More', 'Iron', 'Lead', 'Gold', 'I', 'Electricity']
1549 """
1550 maxMatches: int = deprecate_argument(kwargs, "maxMatches", _MAX_INT)
1552 max_matches = min(maxMatches, max_matches)
1553 try:
1554 return ParseResults(
1555 [
1556 t
1557 for t, s, e in self.scan_string(
1558 instring,
1559 max_matches=max_matches,
1560 always_skip_whitespace=False,
1561 debug=debug,
1562 )
1563 ]
1564 )
1565 except ParseBaseException as exc:
1566 if ParserElement.verbose_stacktrace:
1567 raise
1569 # catch and re-raise exception from here, clears out pyparsing internal stack trace
1570 raise exc.with_traceback(None)
1572 def split(
1573 self,
1574 instring: str,
1575 maxsplit: int = _MAX_INT,
1576 include_separators: bool = False,
1577 **kwargs,
1578 ) -> Generator[str, None, None]:
1579 """
1580 Generator method to split a string using the given expression as a separator.
1581 May be called with optional ``maxsplit`` argument, to limit the number of splits;
1582 and the optional ``include_separators`` argument (default= ``False``), if the separating
1583 matching text should be included in the split results.
1585 Example:
1587 .. testcode::
1589 punc = one_of(list(".,;:/-!?"))
1590 print(list(punc.split(
1591 "This, this?, this sentence, is badly punctuated!")))
1593 prints:
1595 .. testoutput::
1597 ['This', ' this', '', ' this sentence', ' is badly punctuated', '']
1598 """
1599 includeSeparators: bool = deprecate_argument(kwargs, "includeSeparators", False)
1601 include_separators = includeSeparators or include_separators
1602 last = 0
1603 for t, s, e in self.scan_string(instring, max_matches=maxsplit):
1604 yield instring[last:s]
1605 if include_separators:
1606 yield t[0]
1607 last = e
1608 yield instring[last:]
1610 def __add__(self, other) -> ParserElement:
1611 """
1612 Implementation of ``+`` operator - returns :class:`And`. Adding strings to a :class:`ParserElement`
1613 converts them to :class:`Literal`\\ s by default.
1615 Example:
1617 .. testcode::
1619 greet = Word(alphas) + "," + Word(alphas) + "!"
1620 hello = "Hello, World!"
1621 print(hello, "->", greet.parse_string(hello))
1623 prints:
1625 .. testoutput::
1627 Hello, World! -> ['Hello', ',', 'World', '!']
1629 ``...`` may be used as a parse expression as a short form of :class:`SkipTo`:
1631 .. testcode::
1633 Literal('start') + ... + Literal('end')
1635 is equivalent to:
1637 .. testcode::
1639 Literal('start') + SkipTo('end')("_skipped*") + Literal('end')
1641 Note that the skipped text is returned with '_skipped' as a results name,
1642 and to support having multiple skips in the same parser, the value returned is
1643 a list of all skipped text.
1644 """
1645 if other is Ellipsis:
1646 return _PendingSkip(self)
1648 if isinstance(other, str_type):
1649 other = self._literalStringClass(other)
1650 if not isinstance(other, ParserElement):
1651 return NotImplemented
1652 return And([self, other])
1654 def __radd__(self, other) -> ParserElement:
1655 """
1656 Implementation of ``+`` operator when left operand is not a :class:`ParserElement`
1657 """
1658 if other is Ellipsis:
1659 return SkipTo(self)("_skipped*") + self
1661 if isinstance(other, str_type):
1662 other = self._literalStringClass(other)
1663 if not isinstance(other, ParserElement):
1664 return NotImplemented
1665 return other + self
1667 def __sub__(self, other) -> ParserElement:
1668 """
1669 Implementation of ``-`` operator, returns :class:`And` with error stop
1670 """
1671 if isinstance(other, str_type):
1672 other = self._literalStringClass(other)
1673 if not isinstance(other, ParserElement):
1674 return NotImplemented
1675 return self + And._ErrorStop() + other
1677 def __rsub__(self, other) -> ParserElement:
1678 """
1679 Implementation of ``-`` operator when left operand is not a :class:`ParserElement`
1680 """
1681 if isinstance(other, str_type):
1682 other = self._literalStringClass(other)
1683 if not isinstance(other, ParserElement):
1684 return NotImplemented
1685 return other - self
1687 def __mul__(self, other) -> ParserElement:
1688 """
1689 Implementation of ``*`` operator, allows use of ``expr * 3`` in place of
1690 ``expr + expr + expr``. Expressions may also be multiplied by a 2-integer
1691 tuple, similar to ``{min, max}`` multipliers in regular expressions. Tuples
1692 may also include ``None`` as in:
1694 - ``expr*(n, None)`` or ``expr*(n, )`` is equivalent
1695 to ``expr*n + ZeroOrMore(expr)``
1696 (read as "at least n instances of ``expr``")
1697 - ``expr*(None, n)`` is equivalent to ``expr*(0, n)``
1698 (read as "0 to n instances of ``expr``")
1699 - ``expr*(None, None)`` is equivalent to ``ZeroOrMore(expr)``
1700 - ``expr*(1, None)`` is equivalent to ``OneOrMore(expr)``
1702 Note that ``expr*(None, n)`` does not raise an exception if
1703 more than n exprs exist in the input stream; that is,
1704 ``expr*(None, n)`` does not enforce a maximum number of expr
1705 occurrences. If this behavior is desired, then write
1706 ``expr*(None, n) + ~expr``
1707 """
1708 if other is Ellipsis:
1709 other = (0, None)
1710 elif isinstance(other, tuple) and other[:1] == (Ellipsis,):
1711 other = ((0,) + other[1:] + (None,))[:2]
1713 if not isinstance(other, (int, tuple)):
1714 return NotImplemented
1716 if isinstance(other, int):
1717 minElements, optElements = other, 0
1718 else:
1719 other = tuple(o if o is not Ellipsis else None for o in other)
1720 other = (other + (None, None))[:2]
1721 if other[0] is None:
1722 other = (0, other[1])
1723 if isinstance(other[0], int) and other[1] is None:
1724 if other[0] == 0:
1725 return ZeroOrMore(self)
1726 if other[0] == 1:
1727 return OneOrMore(self)
1728 else:
1729 return self * other[0] + ZeroOrMore(self)
1730 elif isinstance(other[0], int) and isinstance(other[1], int):
1731 minElements, optElements = other
1732 optElements -= minElements
1733 else:
1734 return NotImplemented
1736 if minElements < 0:
1737 raise ValueError("cannot multiply ParserElement by negative value")
1738 if optElements < 0:
1739 raise ValueError(
1740 "second tuple value must be greater or equal to first tuple value"
1741 )
1742 if minElements == optElements == 0:
1743 return And([])
1745 if optElements:
1746 # Build the optional tail as a bounded ``ZeroOrMore`` instead of a
1747 # deeply nested ``Opt(self + Opt(self + ...))`` chain. The nested
1748 # form recursed ``optElements`` levels deep, which raised
1749 # RecursionError for large upper bounds (e.g. ``expr[..., 1000]``)
1750 # -- see issue #332. ``ZeroOrMore(..., max=optElements)`` is a flat
1751 # loop that still *exits early* (it stops at the first non-match,
1752 # just like the recursive form), so it preserves the original
1753 # early-exit behavior while no longer scaling the call stack with
1754 # the upper bound.
1755 optionalTail = ZeroOrMore(self, max=optElements)
1757 if minElements:
1758 if minElements == 1:
1759 ret = self + optionalTail
1760 else:
1761 ret = And([self] * minElements) + optionalTail
1762 else:
1763 ret = optionalTail
1764 else:
1765 if minElements == 1:
1766 ret = self
1767 else:
1768 ret = And([self] * minElements)
1769 return ret
1771 def __rmul__(self, other) -> ParserElement:
1772 return self.__mul__(other)
1774 def __or__(self, other) -> ParserElement:
1775 """
1776 Implementation of ``|`` operator - returns :class:`MatchFirst`
1778 .. versionchanged:: 3.1.0
1779 Support ``expr | ""`` as a synonym for ``Optional(expr)``.
1780 """
1781 if other is Ellipsis:
1782 return _PendingSkip(self, must_skip=True)
1784 if isinstance(other, str_type):
1785 # `expr | ""` is equivalent to `Opt(expr)`
1786 if other == "":
1787 return Opt(self)
1788 other = self._literalStringClass(other)
1789 if not isinstance(other, ParserElement):
1790 return NotImplemented
1791 return MatchFirst([self, other])
1793 def __ror__(self, other) -> ParserElement:
1794 """
1795 Implementation of ``|`` operator when left operand is not a :class:`ParserElement`
1796 """
1797 if isinstance(other, str_type):
1798 other = self._literalStringClass(other)
1799 if not isinstance(other, ParserElement):
1800 return NotImplemented
1801 return other | self
1803 def __xor__(self, other) -> ParserElement:
1804 """
1805 Implementation of ``^`` operator - returns :class:`Or`
1806 """
1807 if isinstance(other, str_type):
1808 other = self._literalStringClass(other)
1809 if not isinstance(other, ParserElement):
1810 return NotImplemented
1811 return Or([self, other])
1813 def __rxor__(self, other) -> ParserElement:
1814 """
1815 Implementation of ``^`` operator when left operand is not a :class:`ParserElement`
1816 """
1817 if isinstance(other, str_type):
1818 other = self._literalStringClass(other)
1819 if not isinstance(other, ParserElement):
1820 return NotImplemented
1821 return other ^ self
1823 def __and__(self, other) -> ParserElement:
1824 """
1825 Implementation of ``&`` operator - returns :class:`Each`
1826 """
1827 if isinstance(other, str_type):
1828 other = self._literalStringClass(other)
1829 if not isinstance(other, ParserElement):
1830 return NotImplemented
1831 return Each([self, other])
1833 def __rand__(self, other) -> ParserElement:
1834 """
1835 Implementation of ``&`` operator when left operand is not a :class:`ParserElement`
1836 """
1837 if isinstance(other, str_type):
1838 other = self._literalStringClass(other)
1839 if not isinstance(other, ParserElement):
1840 return NotImplemented
1841 return other & self
1843 def __invert__(self) -> ParserElement:
1844 """
1845 Implementation of ``~`` operator - returns :class:`NotAny`
1846 """
1847 return NotAny(self)
1849 # disable __iter__ to override legacy use of sequential access to __getitem__ to
1850 # iterate over a sequence
1851 __iter__ = None
1853 def __getitem__(self, key):
1854 """
1855 use ``[]`` indexing notation as a short form for expression repetition:
1857 - ``expr[n]`` is equivalent to ``expr*n``
1858 - ``expr[m, n]`` is equivalent to ``expr*(m, n)``
1859 - ``expr[n, ...]`` or ``expr[n,]`` is equivalent
1860 to ``expr*n + ZeroOrMore(expr)``
1861 (read as "at least n instances of ``expr``")
1862 - ``expr[..., n]`` is equivalent to ``expr*(0, n)``
1863 (read as "0 to n instances of ``expr``")
1864 - ``expr[...]`` and ``expr[0, ...]`` are equivalent to ``ZeroOrMore(expr)``
1865 - ``expr[1, ...]`` is equivalent to ``OneOrMore(expr)``
1867 ``None`` may be used in place of ``...``.
1869 Note that ``expr[..., n]`` and ``expr[m, n]`` do not raise an exception
1870 if more than ``n`` ``expr``\\ s exist in the input stream. If this behavior is
1871 desired, then write ``expr[..., n] + ~expr``.
1873 For repetition with a stop_on expression, use slice notation:
1875 - ``expr[...: end_expr]`` and ``expr[0, ...: end_expr]`` are equivalent to ``ZeroOrMore(expr, stop_on=end_expr)``
1876 - ``expr[1, ...: end_expr]`` is equivalent to ``OneOrMore(expr, stop_on=end_expr)``
1878 .. versionchanged:: 3.1.0
1879 Support for slice notation.
1880 """
1882 stop_on_defined = False
1883 stop_on = NoMatch()
1884 if isinstance(key, slice):
1885 key, stop_on = key.start, key.stop
1886 if key is None:
1887 key = ...
1888 stop_on_defined = True
1889 elif isinstance(key, tuple) and isinstance(key[-1], slice):
1890 key, stop_on = (key[0], key[1].start), key[1].stop
1891 stop_on_defined = True
1893 # convert single arg keys to tuples
1894 if isinstance(key, str_type):
1895 key = (key,)
1896 try:
1897 iter(key)
1898 except TypeError:
1899 key = (key, key)
1901 if len(key) > 2:
1902 raise TypeError(
1903 f"only 1 or 2 index arguments supported ({key[:5]}{f'... [{len(key)}]' if len(key) > 5 else ''})"
1904 )
1906 # clip to 2 elements
1907 ret = self * tuple(key[:2])
1908 ret = typing.cast(_MultipleMatch, ret)
1910 if stop_on_defined:
1911 ret.stopOn(stop_on)
1913 return ret
1915 def __call__(self, name: typing.Optional[str] = None) -> ParserElement:
1916 """
1917 Shortcut for :class:`set_results_name`, with ``list_all_matches=False``.
1919 If ``name`` is given with a trailing ``'*'`` character, then ``list_all_matches`` will be
1920 passed as ``True``.
1922 If ``name`` is omitted, same as calling :class:`copy`.
1924 Example:
1926 .. testcode::
1928 # these are equivalent
1929 userdata = (
1930 Word(alphas).set_results_name("name")
1931 + Word(nums + "-").set_results_name("socsecno")
1932 )
1934 userdata = Word(alphas)("name") + Word(nums + "-")("socsecno")
1935 """
1936 if name is not None:
1937 return self._setResultsName(name)
1939 return self.copy()
1941 def suppress(self) -> ParserElement:
1942 """
1943 Suppresses the output of this :class:`ParserElement`; useful to keep punctuation from
1944 cluttering up returned output.
1945 """
1946 return Suppress(self)
1948 def ignore_whitespace(self, recursive: bool = True) -> ParserElement:
1949 """
1950 Enables the skipping of whitespace before matching the characters in the
1951 :class:`ParserElement`'s defined pattern.
1953 :param recursive: If ``True`` (the default), also enable whitespace skipping in child elements (if any)
1954 """
1955 self.skipWhitespace = True
1956 return self
1958 def leave_whitespace(self, recursive: bool = True) -> ParserElement:
1959 """
1960 Disables the skipping of whitespace before matching the characters in the
1961 :class:`ParserElement`'s defined pattern. This is normally only used internally by
1962 the pyparsing module, but may be needed in some whitespace-sensitive grammars.
1964 :param recursive: If true (the default), also disable whitespace skipping in child elements (if any)
1965 """
1966 self.skipWhitespace = False
1967 return self
1969 def set_whitespace_chars(
1970 self, chars: Union[set[str], str], copy_defaults: bool = False
1971 ) -> ParserElement:
1972 """
1973 Overrides the default whitespace chars
1974 """
1975 self.skipWhitespace = True
1976 self.whiteChars = set(chars)
1977 self.copyDefaultWhiteChars = copy_defaults
1978 return self
1980 def parse_with_tabs(self) -> ParserElement:
1981 """
1982 Overrides default behavior to expand ``<TAB>`` s to spaces before parsing the input string.
1983 Must be called before ``parse_string`` when the input grammar contains elements that
1984 match ``<TAB>`` characters.
1985 """
1986 self.keepTabs = True
1987 return self
1989 def ignore(self, other: ParserElement) -> ParserElement:
1990 """
1991 Define expression to be ignored (e.g., comments) while doing pattern
1992 matching; may be called repeatedly, to define multiple comment or other
1993 ignorable patterns.
1995 Example:
1997 .. doctest::
1999 >>> patt = Word(alphas)[...]
2000 >>> print(patt.parse_string('ablaj /* comment */ lskjd'))
2001 ['ablaj']
2003 >>> patt = Word(alphas)[...].ignore(c_style_comment)
2004 >>> print(patt.parse_string('ablaj /* comment */ lskjd'))
2005 ['ablaj', 'lskjd']
2006 """
2007 if isinstance(other, str_type):
2008 other = Suppress(other)
2010 if isinstance(other, Suppress):
2011 if other not in self.ignoreExprs:
2012 self.ignoreExprs.append(other)
2013 else:
2014 self.ignoreExprs.append(Suppress(other.copy()))
2015 return self
2017 def set_debug_actions(
2018 self,
2019 start_action: DebugStartAction,
2020 success_action: DebugSuccessAction,
2021 exception_action: DebugExceptionAction,
2022 ) -> ParserElement:
2023 """
2024 Customize display of debugging messages while doing pattern matching:
2026 :param start_action: method to be called when an expression is about to be parsed;
2027 should have the signature::
2029 fn(input_string: str,
2030 location: int,
2031 expression: ParserElement,
2032 cache_hit: bool)
2034 :param success_action: method to be called when an expression has successfully parsed;
2035 should have the signature::
2037 fn(input_string: str,
2038 start_location: int,
2039 end_location: int,
2040 expression: ParserELement,
2041 parsed_tokens: ParseResults,
2042 cache_hit: bool)
2044 :param exception_action: method to be called when expression fails to parse;
2045 should have the signature::
2047 fn(input_string: str,
2048 location: int,
2049 expression: ParserElement,
2050 exception: Exception,
2051 cache_hit: bool)
2052 """
2053 self.debugActions = self.DebugActions(
2054 start_action or _default_start_debug_action, # type: ignore[truthy-function]
2055 success_action or _default_success_debug_action, # type: ignore[truthy-function]
2056 exception_action or _default_exception_debug_action, # type: ignore[truthy-function]
2057 )
2058 self.debug = any(self.debugActions)
2059 return self
2061 def set_debug(self, flag: bool = True, recurse: bool = False) -> ParserElement:
2062 """
2063 Enable display of debugging messages while doing pattern matching.
2064 Set ``flag`` to ``True`` to enable, ``False`` to disable.
2065 Set ``recurse`` to ``True`` to set the debug flag on this expression and all sub-expressions.
2067 Example:
2069 .. testcode::
2071 wd = Word(alphas).set_name("alphaword")
2072 integer = Word(nums).set_name("numword")
2073 term = wd | integer
2075 # turn on debugging for wd
2076 wd.set_debug()
2078 term[1, ...].parse_string("abc 123 xyz 890")
2080 prints:
2082 .. testoutput::
2083 :options: +NORMALIZE_WHITESPACE
2085 Match alphaword at loc 0(1,1)
2086 abc 123 xyz 890
2087 ^
2088 Matched alphaword -> ['abc']
2089 Match alphaword at loc 4(1,5)
2090 abc 123 xyz 890
2091 ^
2092 Match alphaword failed, ParseException raised: Expected alphaword, ...
2093 Match alphaword at loc 8(1,9)
2094 abc 123 xyz 890
2095 ^
2096 Matched alphaword -> ['xyz']
2097 Match alphaword at loc 12(1,13)
2098 abc 123 xyz 890
2099 ^
2100 Match alphaword failed, ParseException raised: Expected alphaword, ...
2101 abc 123 xyz 890
2102 ^
2103 Match alphaword failed, ParseException raised: Expected alphaword, found end of text ...
2105 The output shown is that produced by the default debug actions - custom debug actions can be
2106 specified using :meth:`set_debug_actions`. Prior to attempting
2107 to match the ``wd`` expression, the debugging message ``"Match <exprname> at loc <n>(<line>,<col>)"``
2108 is shown. Then if the parse succeeds, a ``"Matched"`` message is shown, or an ``"Exception raised"``
2109 message is shown. Also note the use of :meth:`set_name` to assign a human-readable name to the expression,
2110 which makes debugging and exception messages easier to understand - for instance, the default
2111 name created for the :class:`Word` expression without calling :meth:`set_name` is ``"W:(A-Za-z)"``.
2113 .. versionchanged:: 3.1.0
2114 ``recurse`` argument added.
2115 """
2116 if recurse:
2117 for expr in self.visit_all():
2118 expr.set_debug(flag, recurse=False)
2119 return self
2121 if flag:
2122 self.set_debug_actions(
2123 _default_start_debug_action,
2124 _default_success_debug_action,
2125 _default_exception_debug_action,
2126 )
2127 else:
2128 self.debug = False
2129 return self
2131 @property
2132 def default_name(self) -> str:
2133 if self._defaultName is None:
2134 self._defaultName = self._generateDefaultName()
2135 return self._defaultName
2137 @abstractmethod
2138 def _generateDefaultName(self) -> str:
2139 """
2140 Child classes must define this method, which defines how the ``default_name`` is set.
2141 """
2143 def set_name(self, name: typing.Optional[str]) -> ParserElement:
2144 """
2145 Define name for this expression, makes debugging and exception messages clearer. If
2146 `__diag__.enable_debug_on_named_expressions` is set to True, setting a name will also
2147 enable debug for this expression.
2149 If `name` is None, clears any custom name for this expression, and clears the
2150 debug flag is it was enabled via `__diag__.enable_debug_on_named_expressions`.
2152 Example:
2154 .. doctest::
2156 >>> integer = Word(nums)
2157 >>> integer.parse_string("ABC")
2158 Traceback (most recent call last):
2159 ParseException: Expected W:(0-9) (at char 0), (line:1, col:1)
2161 >>> integer.set_name("integer")
2162 integer
2163 >>> integer.parse_string("ABC")
2164 Traceback (most recent call last):
2165 ParseException: Expected integer (at char 0), (line:1, col:1)
2167 .. versionchanged:: 3.1.0
2168 Accept ``None`` as the ``name`` argument.
2169 """
2170 self.customName = name # type: ignore[assignment]
2171 self.errmsg = f"Expected {str(self)}"
2173 if __diag__.enable_debug_on_named_expressions:
2174 self.set_debug(name is not None)
2176 return self
2178 @property
2179 def name(self) -> str:
2180 """
2181 Returns a user-defined name if available, but otherwise defaults back to the auto-generated name
2182 """
2183 return self.customName if self.customName is not None else self.default_name
2185 @name.setter
2186 def name(self, new_name) -> None:
2187 self.set_name(new_name)
2189 def __str__(self) -> str:
2190 return self.name
2192 def __repr__(self) -> str:
2193 return str(self)
2195 def streamline(self) -> ParserElement:
2196 self.streamlined = True
2197 self._defaultName = None
2198 return self
2200 def recurse(self) -> list[ParserElement]:
2201 return []
2203 def _checkRecursion(self, parseElementList):
2204 subRecCheckList = parseElementList[:] + [self]
2205 for e in self.recurse():
2206 e._checkRecursion(subRecCheckList)
2208 def validate(self, validateTrace=None) -> None:
2209 """
2210 .. deprecated:: 3.0.0
2211 Do not use to check for left recursion.
2213 Check defined expressions for valid structure, check for infinite recursive definitions.
2215 """
2216 warnings.warn(
2217 "ParserElement.validate() is deprecated, and should not be used to check for left recursion",
2218 PyparsingDeprecationWarning,
2219 stacklevel=2,
2220 )
2221 self._checkRecursion([])
2223 def parse_file(
2224 self,
2225 file_or_filename: Union[str, Path, TextIO],
2226 encoding: str = "utf-8",
2227 parse_all: bool = False,
2228 **kwargs,
2229 ) -> ParseResults:
2230 """
2231 Execute the parse expression on the given file or filename.
2232 If a filename is specified (instead of a file object),
2233 the entire file is opened, read, and closed before parsing.
2234 """
2235 parseAll: bool = deprecate_argument(kwargs, "parseAll", False)
2237 parse_all = parse_all or parseAll
2238 try:
2239 file_or_filename = typing.cast(TextIO, file_or_filename)
2240 file_contents = file_or_filename.read()
2241 except AttributeError:
2242 file_or_filename = typing.cast(str, file_or_filename)
2243 with open(file_or_filename, "r", encoding=encoding) as f:
2244 file_contents = f.read()
2245 try:
2246 return self.parse_string(file_contents, parse_all)
2247 except ParseBaseException as exc:
2248 if ParserElement.verbose_stacktrace:
2249 raise
2251 # catch and re-raise exception from here, clears out pyparsing internal stack trace
2252 raise exc.with_traceback(None)
2254 def __eq__(self, other):
2255 if self is other:
2256 return True
2257 elif isinstance(other, str_type):
2258 return self.matches(other, parse_all=True)
2259 elif isinstance(other, ParserElement):
2260 return vars(self) == vars(other)
2261 return False
2263 def __hash__(self):
2264 return id(self)
2266 def matches(self, test_string: str, parse_all: bool = True, **kwargs) -> bool:
2267 """
2268 Method for quick testing of a parser against a test string. Good for simple
2269 inline microtests of sub expressions while building up larger parser.
2271 :param test_string: to test against this expression for a match
2272 :param parse_all: flag to pass to :meth:`parse_string` when running tests
2274 Example:
2276 .. doctest::
2278 >>> expr = Word(nums)
2279 >>> expr.matches("100")
2280 True
2281 """
2282 parseAll: bool = deprecate_argument(kwargs, "parseAll", True)
2284 parse_all = parse_all and parseAll
2285 try:
2286 self.parse_string(str(test_string), parse_all=parse_all)
2287 return True
2288 except ParseBaseException:
2289 return False
2291 def run_tests(
2292 self,
2293 tests: Union[str, list[str]],
2294 parse_all: bool = True,
2295 comment: typing.Optional[Union[ParserElement, str]] = "#",
2296 full_dump: bool = True,
2297 print_results: bool = True,
2298 failure_tests: bool = False,
2299 post_parse: typing.Optional[
2300 Callable[[str, ParseResults], typing.Optional[str]]
2301 ] = None,
2302 file: typing.Optional[TextIO] = None,
2303 with_line_numbers: bool = False,
2304 *,
2305 parseAll: bool = True,
2306 fullDump: bool = True,
2307 printResults: bool = True,
2308 failureTests: bool = False,
2309 postParse: typing.Optional[
2310 Callable[[str, ParseResults], typing.Optional[str]]
2311 ] = None,
2312 ) -> tuple[bool, list[tuple[str, Union[ParseResults, Exception]]]]:
2313 """
2314 Execute the parse expression on a series of test strings, showing each
2315 test, the parsed results or where the parse failed. Quick and easy way to
2316 run a parse expression against a list of sample strings.
2318 Parameters:
2320 - ``tests`` - a list of separate test strings, or a multiline string of test strings
2321 - ``parse_all`` - (default= ``True``) - flag to pass to :class:`parse_string` when running tests
2322 - ``comment`` - (default= ``'#'``) - expression for indicating embedded comments in the test
2323 string; pass None to disable comment filtering
2324 - ``full_dump`` - (default= ``True``) - dump results as list followed by results names in nested outline;
2325 if False, only dump nested list
2326 - ``print_results`` - (default= ``True``) prints test output to stdout
2327 - ``failure_tests`` - (default= ``False``) indicates if these tests are expected to fail parsing
2328 - ``post_parse`` - (default= ``None``) optional callback for successful parse results; called as
2329 `fn(test_string, parse_results)` and returns a string to be added to the test output
2330 - ``file`` - (default= ``None``) optional file-like object to which test output will be written;
2331 if None, will default to ``sys.stdout``
2332 - ``with_line_numbers`` - default= ``False``) show test strings with line and column numbers
2334 Returns: a (success, results) tuple, where success indicates that all tests succeeded
2335 (or failed if ``failure_tests`` is True), and the results contain a list of lines of each
2336 test's output
2338 Passing example:
2340 .. testcode::
2342 number_expr = pyparsing_common.number.copy()
2344 result = number_expr.run_tests('''
2345 # unsigned integer
2346 100
2347 # negative integer
2348 -100
2349 # float with scientific notation
2350 6.02e23
2351 # integer with scientific notation
2352 1e-12
2353 # negative decimal number without leading digit
2354 -.100
2355 ''')
2356 print("Success" if result[0] else "Failed!")
2358 prints:
2360 .. testoutput::
2361 :options: +NORMALIZE_WHITESPACE
2364 # unsigned integer
2365 100
2366 [100]
2368 # negative integer
2369 -100
2370 [-100]
2372 # float with scientific notation
2373 6.02e23
2374 [6.02e+23]
2376 # integer with scientific notation
2377 1e-12
2378 [1e-12]
2380 # negative decimal number without leading digit
2381 -.100
2382 [-0.1]
2383 Success
2385 Failure-test example:
2387 .. testcode::
2389 result = number_expr.run_tests('''
2390 # stray character
2391 100Z
2392 # too many '.'
2393 3.14.159
2394 ''', failure_tests=True)
2395 print("Success" if result[0] else "Failed!")
2397 prints:
2399 .. testoutput::
2400 :options: +NORMALIZE_WHITESPACE
2403 # stray character
2404 100Z
2405 100Z
2406 ^
2407 ParseException: Expected end of text, found 'Z' ...
2409 # too many '.'
2410 3.14.159
2411 3.14.159
2412 ^
2413 ParseException: Expected end of text, found '.' ...
2414 FAIL: Expected end of text, found '.' ...
2415 Success
2417 Each test string must be on a single line. If you want to test a string that spans multiple
2418 lines, create a test like this:
2420 .. testcode::
2422 expr = Word(alphanums)[1,...]
2423 expr.run_tests(r"this is a test\\n of strings that spans \\n 3 lines")
2425 .. testoutput::
2426 :options: +NORMALIZE_WHITESPACE
2427 :hide:
2430 this is a test\\n of strings that spans \\n 3 lines
2431 ['this', 'is', 'a', 'test', 'of', 'strings', 'that', 'spans', '3', 'lines']
2433 (Note that this is a raw string literal, you must include the leading ``'r'``.)
2434 """
2435 from .testing import pyparsing_test
2437 parseAll = parseAll and parse_all
2438 fullDump = fullDump and full_dump
2439 printResults = printResults and print_results
2440 failureTests = failureTests or failure_tests
2441 postParse = postParse or post_parse
2442 if isinstance(tests, str_type):
2443 tests = typing.cast(str, tests)
2444 line_strip = type(tests).strip
2445 tests = [line_strip(test_line) for test_line in tests.rstrip().splitlines()]
2446 comment_specified = comment is not None
2447 if comment_specified:
2448 if isinstance(comment, str_type):
2449 comment = typing.cast(str, comment)
2450 comment = Literal(comment)
2451 comment = typing.cast(ParserElement, comment)
2452 if file is None:
2453 file = sys.stdout
2454 print_ = file.write
2456 result: Union[ParseResults, Exception]
2457 allResults: list[tuple[str, Union[ParseResults, Exception]]] = []
2458 comments: list[str] = []
2459 success = True
2460 NL = Literal(r"\n").add_parse_action(replace_with("\n")).ignore(quoted_string)
2461 BOM = "\ufeff"
2462 nlstr = "\n"
2463 for t in tests:
2464 if comment_specified and comment.matches(t, False) or comments and not t:
2465 comments.append(
2466 pyparsing_test.with_line_numbers(t) if with_line_numbers else t
2467 )
2468 continue
2469 if not t:
2470 continue
2471 out = [
2472 f"{nlstr}{nlstr.join(comments) if comments else ''}",
2473 pyparsing_test.with_line_numbers(t) if with_line_numbers else t,
2474 ]
2475 comments.clear()
2476 try:
2477 # convert newline marks to actual newlines, and strip leading BOM if present
2478 t = NL.transform_string(t.lstrip(BOM))
2479 result = self.parse_string(t, parse_all=parse_all)
2480 except ParseBaseException as pe:
2481 fatal = "(FATAL) " if isinstance(pe, ParseFatalException) else ""
2482 out.append(pe.explain())
2483 out.append(f"FAIL: {fatal}{pe}")
2484 if ParserElement.verbose_stacktrace:
2485 out.extend(traceback.format_tb(pe.__traceback__))
2486 success = success and failureTests
2487 result = pe
2488 except Exception as exc:
2489 tag = "FAIL-EXCEPTION"
2491 # see if this exception was raised in a parse action
2492 tb = exc.__traceback__
2493 it = iter(traceback.walk_tb(tb))
2494 for f, line in it:
2495 if (f.f_code.co_filename, line) == pa_call_line_synth:
2496 next_f = next(it)[0]
2497 tag += f" (raised in parse action {next_f.f_code.co_name!r})"
2498 break
2500 out.append(f"{tag}: {type(exc).__name__}: {exc}")
2501 if ParserElement.verbose_stacktrace:
2502 out.extend(traceback.format_tb(exc.__traceback__))
2503 success = success and failureTests
2504 result = exc
2505 else:
2506 success = success and not failureTests
2507 if postParse is not None:
2508 try:
2509 pp_value = postParse(t, result)
2510 if pp_value is not None:
2511 if isinstance(pp_value, ParseResults):
2512 out.append(pp_value.dump())
2513 else:
2514 out.append(str(pp_value))
2515 else:
2516 out.append(result.dump())
2517 except Exception as e:
2518 out.append(result.dump(full=fullDump))
2519 out.append(
2520 f"{postParse.__name__} failed: {type(e).__name__}: {e}"
2521 )
2522 else:
2523 out.append(result.dump(full=fullDump))
2524 out.append("")
2526 if printResults:
2527 print_("\n".join(out))
2529 allResults.append((t, result))
2531 return success, allResults
2533 def create_diagram(
2534 self,
2535 output_html: Union[TextIO, Path, str],
2536 vertical: int = 3,
2537 show_results_names: bool = False,
2538 show_groups: bool = False,
2539 embed: bool = False,
2540 show_hidden: bool = False,
2541 **kwargs,
2542 ) -> None:
2543 """
2544 Create a railroad diagram for the parser.
2546 Parameters:
2548 - ``output_html`` (str or file-like object) - output target for generated
2549 diagram HTML
2550 - ``vertical`` (int) - threshold for formatting multiple alternatives vertically
2551 instead of horizontally (default=3)
2552 - ``show_results_names`` - bool flag whether diagram should show annotations for
2553 defined results names
2554 - ``show_groups`` - bool flag whether groups should be highlighted with an unlabeled surrounding box
2555 - ``show_hidden`` - bool flag to show diagram elements for internal elements that are usually hidden
2556 - ``embed`` - bool flag whether generated HTML should omit <HEAD>, <BODY>, and <DOCTYPE> tags to embed
2557 the resulting HTML in an enclosing HTML source
2558 - ``head`` - str containing additional HTML to insert into the <HEAD> section of the generated code;
2559 can be used to insert custom CSS styling
2560 - ``body`` - str containing additional HTML to insert at the beginning of the <BODY> section of the
2561 generated code
2563 Additional diagram-formatting keyword arguments can also be included;
2564 see railroad.Diagram class.
2566 .. versionchanged:: 3.1.0
2567 ``embed`` argument added.
2568 """
2570 try:
2571 from .diagram import to_railroad, railroad_to_html
2572 except ImportError as ie:
2573 raise Exception(
2574 "must ``pip install pyparsing[diagrams]`` to generate parser railroad diagrams"
2575 ) from ie
2577 self.streamline()
2579 railroad = to_railroad(
2580 self,
2581 vertical=vertical,
2582 show_results_names=show_results_names,
2583 show_groups=show_groups,
2584 show_hidden=show_hidden,
2585 diagram_kwargs=kwargs,
2586 )
2587 if not isinstance(output_html, (str, Path)):
2588 # we were passed a file-like object, just write to it
2589 output_html.write(railroad_to_html(railroad, embed=embed, **kwargs))
2590 return
2592 with open(output_html, "w", encoding="utf-8") as diag_file:
2593 diag_file.write(railroad_to_html(railroad, embed=embed, **kwargs))
2595 # Compatibility synonyms
2596 # fmt: off
2597 inlineLiteralsUsing = staticmethod(replaced_by_pep8("inlineLiteralsUsing", inline_literals_using))
2598 setDefaultWhitespaceChars = staticmethod(replaced_by_pep8(
2599 "setDefaultWhitespaceChars", set_default_whitespace_chars
2600 ))
2601 disableMemoization = staticmethod(replaced_by_pep8("disableMemoization", disable_memoization))
2602 enableLeftRecursion = staticmethod(replaced_by_pep8("enableLeftRecursion", enable_left_recursion))
2603 enablePackrat = staticmethod(replaced_by_pep8("enablePackrat", enable_packrat))
2604 resetCache = staticmethod(replaced_by_pep8("resetCache", reset_cache))
2606 setResultsName = replaced_by_pep8("setResultsName", set_results_name)
2607 setBreak = replaced_by_pep8("setBreak", set_break)
2608 setParseAction = replaced_by_pep8("setParseAction", set_parse_action)
2609 addParseAction = replaced_by_pep8("addParseAction", add_parse_action)
2610 addCondition = replaced_by_pep8("addCondition", add_condition)
2611 setFailAction = replaced_by_pep8("setFailAction", set_fail_action)
2612 tryParse = replaced_by_pep8("tryParse", try_parse)
2613 parseString = replaced_by_pep8("parseString", parse_string)
2614 scanString = replaced_by_pep8("scanString", scan_string)
2615 transformString = replaced_by_pep8("transformString", transform_string)
2616 searchString = replaced_by_pep8("searchString", search_string)
2617 ignoreWhitespace = replaced_by_pep8("ignoreWhitespace", ignore_whitespace)
2618 leaveWhitespace = replaced_by_pep8("leaveWhitespace", leave_whitespace)
2619 setWhitespaceChars = replaced_by_pep8("setWhitespaceChars", set_whitespace_chars)
2620 parseWithTabs = replaced_by_pep8("parseWithTabs", parse_with_tabs)
2621 setDebugActions = replaced_by_pep8("setDebugActions", set_debug_actions)
2622 setDebug = replaced_by_pep8("setDebug", set_debug)
2623 setName = replaced_by_pep8("setName", set_name)
2624 parseFile = replaced_by_pep8("parseFile", parse_file)
2625 runTests = replaced_by_pep8("runTests", run_tests)
2626 canParseNext = replaced_by_pep8("canParseNext", can_parse_next)
2627 defaultName = default_name
2628 # fmt: on
2631class _PendingSkip(ParserElement):
2632 # internal placeholder class to hold a place were '...' is added to a parser element,
2633 # once another ParserElement is added, this placeholder will be replaced with a SkipTo
2634 def __init__(self, expr: ParserElement, must_skip: bool = False) -> None:
2635 super().__init__()
2636 self.anchor = expr
2637 self.must_skip = must_skip
2639 def _generateDefaultName(self) -> str:
2640 return str(self.anchor + Empty()).replace("Empty", "...")
2642 def __add__(self, other) -> ParserElement:
2643 skipper = SkipTo(other).set_name("...")("_skipped*")
2644 if self.must_skip:
2646 def must_skip(t):
2647 if not t._skipped or t._skipped.as_list() == [""]:
2648 del t[0]
2649 t.pop("_skipped", None)
2651 def show_skip(t):
2652 if t._skipped.as_list()[-1:] == [""]:
2653 t.pop("_skipped")
2654 t["_skipped"] = f"missing <{self.anchor!r}>"
2656 return (
2657 self.anchor + skipper().add_parse_action(must_skip)
2658 | skipper().add_parse_action(show_skip)
2659 ) + other
2661 return self.anchor + skipper + other
2663 def __repr__(self):
2664 return self.defaultName
2666 def parseImpl(self, *args) -> ParseImplReturnType:
2667 raise Exception(
2668 "use of `...` expression without following SkipTo target expression"
2669 )
2672class Token(ParserElement):
2673 """Abstract :class:`ParserElement` subclass, for defining atomic
2674 matching patterns.
2675 """
2677 def __init__(self) -> None:
2678 super().__init__(savelist=False)
2680 def _generateDefaultName(self) -> str:
2681 return type(self).__name__
2684class NoMatch(Token):
2685 """
2686 A token that will never match.
2687 """
2689 def __init__(self) -> None:
2690 super().__init__()
2691 self._may_return_empty = True
2692 self.mayIndexError = False
2693 self.errmsg = "Unmatchable token"
2695 def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType:
2696 raise ParseException(instring, loc, self.errmsg, self)
2699class Literal(Token):
2700 """
2701 Token to exactly match a specified string.
2703 Example:
2705 .. doctest::
2707 >>> Literal('abc').parse_string('abc')
2708 ParseResults(['abc'], {})
2709 >>> Literal('abc').parse_string('abcdef')
2710 ParseResults(['abc'], {})
2711 >>> Literal('abc').parse_string('ab')
2712 Traceback (most recent call last):
2713 ParseException: Expected 'abc', found 'ab' (at char 0), (line: 1, col: 1)
2715 For case-insensitive matching, use :class:`CaselessLiteral`.
2717 For keyword matching (force word break before and after the matched string),
2718 use :class:`Keyword` or :class:`CaselessKeyword`.
2719 """
2721 def __new__(cls, match_string: str = "", **kwargs):
2722 # Performance tuning: select a subclass with optimized parseImpl
2723 if cls is Literal:
2724 matchString: str = deprecate_argument(kwargs, "matchString", "")
2726 match_string = matchString or match_string
2727 if not match_string:
2728 return super().__new__(Empty)
2729 if len(match_string) == 1:
2730 return super().__new__(_SingleCharLiteral)
2732 # Default behavior
2733 return super().__new__(cls)
2735 # Needed to make copy.copy() work correctly if we customize __new__
2736 def __getnewargs__(self):
2737 return (self.match,)
2739 def __init__(self, match_string: str = "", **kwargs) -> None:
2740 matchString: str = deprecate_argument(kwargs, "matchString", "")
2742 super().__init__()
2743 match_string = matchString or match_string
2744 self.match = match_string
2745 self.matchLen = len(match_string)
2746 self.firstMatchChar = match_string[:1]
2747 self.errmsg = f"Expected {self.name}"
2748 self._may_return_empty = False
2749 self.mayIndexError = False
2751 def _generateDefaultName(self) -> str:
2752 return repr(self.match)
2754 def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType:
2755 if instring[loc] == self.firstMatchChar and instring.startswith(
2756 self.match, loc
2757 ):
2758 return loc + self.matchLen, self.match
2759 raise ParseException(instring, loc, self.errmsg, self)
2762class Empty(Literal):
2763 """
2764 An empty token, will always match.
2765 """
2767 def __init__(self, match_string="", *, matchString="") -> None:
2768 super().__init__("")
2769 self._may_return_empty = True
2770 self.mayIndexError = False
2772 def _generateDefaultName(self) -> str:
2773 return "Empty"
2775 def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType:
2776 return loc, []
2779class _SingleCharLiteral(Literal):
2780 def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType:
2781 if instring[loc] == self.firstMatchChar:
2782 return loc + 1, self.match
2783 raise ParseException(instring, loc, self.errmsg, self)
2786ParserElement._literalStringClass = Literal
2789class Keyword(Token):
2790 """
2791 Token to exactly match a specified string as a keyword, that is,
2792 it must be immediately preceded and followed by whitespace or
2793 non-keyword characters. Compare with :class:`Literal`:
2795 - ``Literal("if")`` will match the leading ``'if'`` in
2796 ``'ifAndOnlyIf'``.
2797 - ``Keyword("if")`` will not; it will only match the leading
2798 ``'if'`` in ``'if x=1'``, or ``'if(y==2)'``
2800 Accepts two optional constructor arguments in addition to the
2801 keyword string:
2803 - ``ident_chars`` is a string of characters that would be valid
2804 identifier characters, defaulting to all alphanumerics + "_" and
2805 "$"
2806 - ``caseless`` allows case-insensitive matching, default is ``False``.
2808 Example:
2810 .. doctest::
2811 :options: +NORMALIZE_WHITESPACE
2813 >>> Keyword("start").parse_string("start")
2814 ParseResults(['start'], {})
2815 >>> Keyword("start").parse_string("starting")
2816 Traceback (most recent call last):
2817 ParseException: Expected Keyword 'start', keyword was immediately
2818 followed by keyword character, found 'ing' (at char 5), (line:1, col:6)
2820 .. doctest::
2821 :options: +NORMALIZE_WHITESPACE
2823 >>> Keyword("start").parse_string("starting").debug()
2824 Traceback (most recent call last):
2825 ParseException: Expected Keyword "start", keyword was immediately
2826 followed by keyword character, found 'ing' ...
2828 For case-insensitive matching, use :class:`CaselessKeyword`.
2829 """
2831 DEFAULT_KEYWORD_CHARS = alphanums + "_$"
2833 def __init__(
2834 self,
2835 match_string: str = "",
2836 ident_chars: typing.Optional[str] = None,
2837 caseless: bool = False,
2838 **kwargs,
2839 ) -> None:
2840 matchString = deprecate_argument(kwargs, "matchString", "")
2841 identChars = deprecate_argument(kwargs, "identChars", None)
2843 super().__init__()
2844 identChars = identChars or ident_chars
2845 if identChars is None:
2846 identChars = Keyword.DEFAULT_KEYWORD_CHARS
2847 match_string = matchString or match_string
2848 self.match = match_string
2849 self.matchLen = len(match_string)
2850 self.firstMatchChar = match_string[:1]
2851 if not self.firstMatchChar:
2852 raise ValueError("null string passed to Keyword; use Empty() instead")
2853 self.errmsg = f"Expected {type(self).__name__} {self.name}"
2854 self._may_return_empty = False
2855 self.mayIndexError = False
2856 self.caseless = caseless
2857 if caseless:
2858 self.caselessmatch = match_string.upper()
2859 identChars = identChars.upper()
2860 self.ident_chars = set(identChars)
2862 @property
2863 def identChars(self) -> set[str]:
2864 """
2865 .. deprecated:: 3.3.0
2866 use ident_chars instead.
2868 Property returning the characters being used as keyword characters for this expression.
2869 """
2870 return self.ident_chars
2872 def _generateDefaultName(self) -> str:
2873 return repr(self.match)
2875 def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType:
2876 errmsg = self.errmsg or ""
2877 errloc = loc
2878 if self.caseless:
2879 if instring[loc : loc + self.matchLen].upper() == self.caselessmatch:
2880 if loc == 0 or instring[loc - 1].upper() not in self.identChars:
2881 if (
2882 loc >= len(instring) - self.matchLen
2883 or instring[loc + self.matchLen].upper() not in self.identChars
2884 ):
2885 return loc + self.matchLen, self.match
2887 # followed by keyword char
2888 errmsg += ", was immediately followed by keyword character"
2889 errloc = loc + self.matchLen
2890 else:
2891 # preceded by keyword char
2892 errmsg += ", keyword was immediately preceded by keyword character"
2893 errloc = loc - 1
2894 # else no match just raise plain exception
2896 elif (
2897 instring[loc] == self.firstMatchChar
2898 and self.matchLen == 1
2899 or instring.startswith(self.match, loc)
2900 ):
2901 if loc == 0 or instring[loc - 1] not in self.identChars:
2902 if (
2903 loc >= len(instring) - self.matchLen
2904 or instring[loc + self.matchLen] not in self.identChars
2905 ):
2906 return loc + self.matchLen, self.match
2908 # followed by keyword char
2909 errmsg += ", keyword was immediately followed by keyword character"
2910 errloc = loc + self.matchLen
2911 else:
2912 # preceded by keyword char
2913 errmsg += ", keyword was immediately preceded by keyword character"
2914 errloc = loc - 1
2915 # else no match just raise plain exception
2917 raise ParseException(instring, errloc, errmsg, self)
2919 @staticmethod
2920 def set_default_keyword_chars(chars) -> None:
2921 """
2922 Overrides the default characters used by :class:`Keyword` expressions.
2923 """
2924 Keyword.DEFAULT_KEYWORD_CHARS = chars
2926 # Compatibility synonyms
2927 setDefaultKeywordChars = staticmethod(
2928 replaced_by_pep8("setDefaultKeywordChars", set_default_keyword_chars)
2929 )
2932class CaselessLiteral(Literal):
2933 """
2934 Token to match a specified string, ignoring case of letters.
2935 Note: the matched results will always be in the case of the given
2936 match string, NOT the case of the input text.
2938 Example:
2940 .. doctest::
2942 >>> CaselessLiteral("CMD")[1, ...].parse_string("cmd CMD Cmd10")
2943 ParseResults(['CMD', 'CMD', 'CMD'], {})
2945 (Contrast with example for :class:`CaselessKeyword`.)
2946 """
2948 def __init__(self, match_string: str = "", **kwargs) -> None:
2949 matchString: str = deprecate_argument(kwargs, "matchString", "")
2951 match_string = matchString or match_string
2952 super().__init__(match_string.upper())
2953 # Preserve the defining literal.
2954 self.returnString = match_string
2955 self.errmsg = f"Expected {self.name}"
2957 def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType:
2958 if instring[loc : loc + self.matchLen].upper() == self.match:
2959 return loc + self.matchLen, self.returnString
2960 raise ParseException(instring, loc, self.errmsg, self)
2963class CaselessKeyword(Keyword):
2964 """
2965 Caseless version of :class:`Keyword`.
2967 Example:
2969 .. doctest::
2971 >>> CaselessKeyword("CMD")[1, ...].parse_string("cmd CMD Cmd10")
2972 ParseResults(['CMD', 'CMD'], {})
2974 (Contrast with example for :class:`CaselessLiteral`.)
2975 """
2977 def __init__(
2978 self, match_string: str = "", ident_chars: typing.Optional[str] = None, **kwargs
2979 ) -> None:
2980 matchString: str = deprecate_argument(kwargs, "matchString", "")
2981 identChars: typing.Optional[str] = deprecate_argument(
2982 kwargs, "identChars", None
2983 )
2985 identChars = identChars or ident_chars
2986 match_string = matchString or match_string
2987 super().__init__(match_string, identChars, caseless=True)
2990class CloseMatch(Token):
2991 """A variation on :class:`Literal` which matches "close" matches,
2992 that is, strings with at most 'n' mismatching characters.
2993 :class:`CloseMatch` takes parameters:
2995 - ``match_string`` - string to be matched
2996 - ``caseless`` - a boolean indicating whether to ignore casing when comparing characters
2997 - ``max_mismatches`` - (``default=1``) maximum number of
2998 mismatches allowed to count as a match
3000 The results from a successful parse will contain the matched text
3001 from the input string and the following named results:
3003 - ``mismatches`` - a list of the positions within the
3004 match_string where mismatches were found
3005 - ``original`` - the original match_string used to compare
3006 against the input string
3008 If ``mismatches`` is an empty list, then the match was an exact
3009 match.
3011 Example:
3013 .. doctest::
3014 :options: +NORMALIZE_WHITESPACE
3016 >>> patt = CloseMatch("ATCATCGAATGGA")
3017 >>> patt.parse_string("ATCATCGAAXGGA")
3018 ParseResults(['ATCATCGAAXGGA'],
3019 {'original': 'ATCATCGAATGGA', 'mismatches': [9]})
3021 >>> patt.parse_string("ATCAXCGAAXGGA")
3022 Traceback (most recent call last):
3023 ParseException: Expected 'ATCATCGAATGGA' (with up to 1 mismatches),
3024 found 'ATCAXCGAAXGGA' (at char 0), (line:1, col:1)
3026 # exact match
3027 >>> patt.parse_string("ATCATCGAATGGA")
3028 ParseResults(['ATCATCGAATGGA'],
3029 {'original': 'ATCATCGAATGGA', 'mismatches': []})
3031 # close match allowing up to 2 mismatches
3032 >>> patt = CloseMatch("ATCATCGAATGGA", max_mismatches=2)
3033 >>> patt.parse_string("ATCAXCGAAXGGA")
3034 ParseResults(['ATCAXCGAAXGGA'],
3035 {'original': 'ATCATCGAATGGA', 'mismatches': [4, 9]})
3036 """
3038 def __init__(
3039 self,
3040 match_string: str,
3041 max_mismatches: typing.Optional[int] = None,
3042 *,
3043 caseless=False,
3044 **kwargs,
3045 ) -> None:
3046 maxMismatches: int = deprecate_argument(kwargs, "maxMismatches", 1)
3048 maxMismatches = max_mismatches if max_mismatches is not None else maxMismatches
3049 super().__init__()
3050 self.match_string = match_string
3051 self.maxMismatches = maxMismatches
3052 self.errmsg = f"Expected {self.match_string!r} (with up to {self.maxMismatches} mismatches)"
3053 self.caseless = caseless
3054 self.mayIndexError = False
3055 self._may_return_empty = False
3057 def _generateDefaultName(self) -> str:
3058 return f"{type(self).__name__}:{self.match_string!r}"
3060 def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType:
3061 start = loc
3062 instrlen = len(instring)
3063 maxloc = start + len(self.match_string)
3065 if maxloc <= instrlen:
3066 match_string = self.match_string
3067 match_stringloc = 0
3068 mismatches = []
3069 maxMismatches = self.maxMismatches
3071 for match_stringloc, s_m in enumerate(
3072 zip(instring[loc:maxloc], match_string)
3073 ):
3074 src, mat = s_m
3075 if self.caseless:
3076 src, mat = src.lower(), mat.lower()
3078 if src != mat:
3079 mismatches.append(match_stringloc)
3080 if len(mismatches) > maxMismatches:
3081 break
3082 else:
3083 loc = start + match_stringloc + 1
3084 results = ParseResults([instring[start:loc]])
3085 results["original"] = match_string
3086 results["mismatches"] = mismatches
3087 return loc, results
3089 raise ParseException(instring, loc, self.errmsg, self)
3092class Word(Token):
3093 """Token for matching words composed of allowed character sets.
3095 Parameters:
3097 - ``init_chars`` - string of all characters that should be used to
3098 match as a word; "ABC" will match "AAA", "ABAB", "CBAC", etc.;
3099 if ``body_chars`` is also specified, then this is the string of
3100 initial characters
3101 - ``body_chars`` - string of characters that
3102 can be used for matching after a matched initial character as
3103 given in ``init_chars``; if omitted, same as the initial characters
3104 (default=``None``)
3105 - ``min`` - minimum number of characters to match (default=1)
3106 - ``max`` - maximum number of characters to match (default=0)
3107 - ``exact`` - exact number of characters to match (default=0)
3108 - ``as_keyword`` - match as a keyword (default=``False``)
3109 - ``exclude_chars`` - characters that might be
3110 found in the input ``body_chars`` string but which should not be
3111 accepted for matching ;useful to define a word of all
3112 printables except for one or two characters, for instance
3113 (default=``None``)
3115 :class:`srange` is useful for defining custom character set strings
3116 for defining :class:`Word` expressions, using range notation from
3117 regular expression character sets.
3119 A common mistake is to use :class:`Word` to match a specific literal
3120 string, as in ``Word("Address")``. Remember that :class:`Word`
3121 uses the string argument to define *sets* of matchable characters.
3122 This expression would match "Add", "AAA", "dAred", or any other word
3123 made up of the characters 'A', 'd', 'r', 'e', and 's'. To match an
3124 exact literal string, use :class:`Literal` or :class:`Keyword`.
3126 pyparsing includes helper strings for building Words:
3128 - :attr:`alphas`
3129 - :attr:`nums`
3130 - :attr:`alphanums`
3131 - :attr:`hexnums`
3132 - :attr:`alphas8bit` (alphabetic characters in ASCII range 128-255
3133 - accented, tilded, umlauted, etc.)
3134 - :attr:`punc8bit` (non-alphabetic characters in ASCII range
3135 128-255 - currency, symbols, superscripts, diacriticals, etc.)
3136 - :attr:`printables` (any non-whitespace character)
3138 ``alphas``, ``nums``, and ``printables`` are also defined in several
3139 Unicode sets - see :class:`pyparsing_unicode`.
3141 Example:
3143 .. testcode::
3145 # a word composed of digits
3146 integer = Word(nums)
3147 # Two equivalent alternate forms:
3148 Word("0123456789")
3149 Word(srange("[0-9]"))
3151 # a word with a leading capital, and zero or more lowercase
3152 capitalized_word = Word(alphas.upper(), alphas.lower())
3154 # hostnames are alphanumeric, with leading alpha, and '-'
3155 hostname = Word(alphas, alphanums + '-')
3157 # roman numeral
3158 # (not a strict parser, accepts invalid mix of characters)
3159 roman = Word("IVXLCDM")
3161 # any string of non-whitespace characters, except for ','
3162 csv_value = Word(printables, exclude_chars=",")
3164 :raises ValueError: If ``min`` and ``max`` are both specified
3165 and the test ``min <= max`` fails.
3167 .. versionchanged:: 3.1.0
3168 Raises :exc:`ValueError` if ``min`` > ``max``.
3169 """
3171 def __init__(
3172 self,
3173 init_chars: str = "",
3174 body_chars: typing.Optional[str] = None,
3175 min: int = 1,
3176 max: int = 0,
3177 exact: int = 0,
3178 as_keyword: bool = False,
3179 exclude_chars: typing.Optional[str] = None,
3180 **kwargs,
3181 ) -> None:
3182 initChars: typing.Optional[str] = deprecate_argument(kwargs, "initChars", None)
3183 bodyChars: typing.Optional[str] = deprecate_argument(kwargs, "bodyChars", None)
3184 asKeyword: bool = deprecate_argument(kwargs, "asKeyword", False)
3185 excludeChars: typing.Optional[str] = deprecate_argument(
3186 kwargs, "excludeChars", None
3187 )
3189 initChars = initChars or init_chars
3190 bodyChars = bodyChars or body_chars
3191 asKeyword = asKeyword or as_keyword
3192 excludeChars = excludeChars or exclude_chars
3193 super().__init__()
3194 if not initChars:
3195 raise ValueError(
3196 f"invalid {type(self).__name__}, initChars cannot be empty string"
3197 )
3199 initChars_set = set(initChars)
3200 if excludeChars:
3201 excludeChars_set = set(excludeChars)
3202 initChars_set -= excludeChars_set
3203 if bodyChars:
3204 bodyChars = "".join(set(bodyChars) - excludeChars_set)
3205 self.init_chars = initChars_set
3206 self.initCharsOrig = "".join(sorted(initChars_set))
3208 if bodyChars:
3209 self.bodyChars = set(bodyChars)
3210 self.bodyCharsOrig = "".join(sorted(bodyChars))
3211 else:
3212 self.bodyChars = initChars_set
3213 self.bodyCharsOrig = self.initCharsOrig
3215 self.maxSpecified = max > 0
3217 if min < 1:
3218 raise ValueError(
3219 "cannot specify a minimum length < 1; use Opt(Word()) if zero-length word is permitted"
3220 )
3222 if self.maxSpecified and min > max:
3223 raise ValueError(
3224 f"invalid args, if min and max both specified min must be <= max (min={min}, max={max})"
3225 )
3227 self.minLen = min
3229 if max > 0:
3230 self.maxLen = max
3231 else:
3232 self.maxLen = _MAX_INT
3234 if exact > 0:
3235 min = max = exact
3236 self.maxLen = exact
3237 self.minLen = exact
3239 self.errmsg = f"Expected {self.name}"
3240 self.mayIndexError = False
3241 self.asKeyword = asKeyword
3242 if self.asKeyword:
3243 self.errmsg += " as a keyword"
3245 # see if we can make a regex for this Word
3246 if " " not in (self.initChars | self.bodyChars):
3247 if len(self.initChars) == 1:
3248 re_leading_fragment = re.escape(self.initCharsOrig)
3249 else:
3250 re_leading_fragment = f"[{_collapse_string_to_ranges(self.initChars)}]"
3252 if self.bodyChars == self.initChars:
3253 if max == 0 and self.minLen == 1:
3254 repeat = "+"
3255 elif max == 1:
3256 repeat = ""
3257 else:
3258 if self.minLen != self.maxLen:
3259 repeat = f"{{{self.minLen},{'' if self.maxLen == _MAX_INT else self.maxLen}}}"
3260 else:
3261 repeat = f"{{{self.minLen}}}"
3262 self.reString = f"{re_leading_fragment}{repeat}"
3263 else:
3264 if max == 1:
3265 re_body_fragment = ""
3266 repeat = ""
3267 else:
3268 re_body_fragment = f"[{_collapse_string_to_ranges(self.bodyChars)}]"
3269 if max == 0 and self.minLen == 1:
3270 repeat = "*"
3271 elif max == 2:
3272 repeat = "?" if min <= 1 else ""
3273 else:
3274 if min != max:
3275 repeat = f"{{{min - 1 if min > 0 else ''},{max - 1 if max > 0 else ''}}}"
3276 else:
3277 repeat = f"{{{min - 1 if min > 0 else ''}}}"
3279 self.reString = f"{re_leading_fragment}{re_body_fragment}{repeat}"
3281 if self.asKeyword:
3282 self.reString = rf"\b{self.reString}\b"
3284 try:
3285 self.re = re.compile(self.reString)
3286 except re.error:
3287 self.re = None # type: ignore[assignment]
3288 else:
3289 self.re_match = self.re.match
3290 self.parseImpl = self.parseImpl_regex # type: ignore[method-assign]
3292 @property
3293 def initChars(self) -> set[str]:
3294 """
3295 .. deprecated:: 3.3.0
3296 use `init_chars` instead.
3298 Property returning the initial chars to be used when matching this
3299 Word expression. If no body chars were specified, the initial characters
3300 will also be the body characters.
3301 """
3302 return set(self.init_chars)
3304 def copy(self) -> Word:
3305 """
3306 Returns a copy of this expression.
3308 Generally only used internally by pyparsing.
3309 """
3310 ret: Word = cast(Word, super().copy())
3311 if hasattr(self, "re_match"):
3312 ret.re_match = self.re_match
3313 ret.parseImpl = ret.parseImpl_regex # type: ignore[method-assign]
3314 return ret
3316 def _generateDefaultName(self) -> str:
3317 def charsAsStr(s):
3318 max_repr_len = 16
3319 s = _collapse_string_to_ranges(s, re_escape=False)
3321 if len(s) > max_repr_len:
3322 return s[: max_repr_len - 3] + "..."
3324 return s
3326 if self.initChars != self.bodyChars:
3327 base = f"W:({charsAsStr(self.initChars)}, {charsAsStr(self.bodyChars)})"
3328 else:
3329 base = f"W:({charsAsStr(self.initChars)})"
3331 # add length specification
3332 if self.minLen > 1 or self.maxLen != _MAX_INT:
3333 if self.minLen == self.maxLen:
3334 if self.minLen == 1:
3335 return base[2:]
3336 else:
3337 return base + f"{{{self.minLen}}}"
3338 elif self.maxLen == _MAX_INT:
3339 return base + f"{{{self.minLen},...}}"
3340 else:
3341 return base + f"{{{self.minLen},{self.maxLen}}}"
3342 return base
3344 def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType:
3345 if instring[loc] not in self.initChars:
3346 raise ParseException(instring, loc, self.errmsg, self)
3348 start = loc
3349 loc += 1
3350 instrlen = len(instring)
3351 body_chars: set[str] = self.bodyChars
3352 maxloc = start + self.maxLen
3353 maxloc = min(maxloc, instrlen)
3354 while loc < maxloc and instring[loc] in body_chars:
3355 loc += 1
3357 throw_exception = False
3358 if loc - start < self.minLen:
3359 throw_exception = True
3360 elif self.maxSpecified and loc < instrlen and instring[loc] in body_chars:
3361 throw_exception = True
3362 elif self.asKeyword and (
3363 (start > 0 and instring[start - 1] in body_chars)
3364 or (loc < instrlen and instring[loc] in body_chars)
3365 ):
3366 throw_exception = True
3368 if throw_exception:
3369 raise ParseException(instring, loc, self.errmsg, self)
3371 return loc, instring[start:loc]
3373 def parseImpl_regex(self, instring, loc, do_actions=True) -> ParseImplReturnType:
3374 result = self.re_match(instring, loc)
3375 if not result:
3376 raise ParseException(instring, loc, self.errmsg, self)
3378 loc = result.end()
3379 return loc, result[0]
3382class Char(Word):
3383 """A short-cut class for defining :class:`Word` ``(characters, exact=1)``,
3384 when defining a match of any single character in a string of
3385 characters.
3386 """
3388 def __init__(
3389 self,
3390 charset: str,
3391 as_keyword: bool = False,
3392 exclude_chars: typing.Optional[str] = None,
3393 **kwargs,
3394 ) -> None:
3395 asKeyword: bool = deprecate_argument(kwargs, "asKeyword", False)
3396 excludeChars: typing.Optional[str] = deprecate_argument(
3397 kwargs, "excludeChars", None
3398 )
3400 asKeyword = asKeyword or as_keyword
3401 excludeChars = excludeChars or exclude_chars
3402 super().__init__(
3403 charset, exact=1, as_keyword=asKeyword, exclude_chars=excludeChars
3404 )
3407class Regex(Token):
3408 r"""Token for matching strings that match a given regular
3409 expression. Defined with string specifying the regular expression in
3410 a form recognized by the stdlib Python `re module <https://docs.python.org/3/library/re.html>`_.
3411 If the given regex contains named groups (defined using ``(?P<name>...)``),
3412 these will be preserved as named :class:`ParseResults`.
3414 If instead of the Python stdlib ``re`` module you wish to use a different RE module
3415 (such as the ``regex`` module), you can do so by building your ``Regex`` object with
3416 a compiled RE that was compiled using ``regex``.
3418 The parameters ``pattern`` and ``flags`` are passed
3419 to the ``re.compile()`` function as-is. See the Python
3420 `re module <https://docs.python.org/3/library/re.html>`_ module for an
3421 explanation of the acceptable patterns and flags.
3423 Example:
3425 .. testcode::
3427 realnum = Regex(r"[+-]?\d+\.\d*")
3428 # ref: https://stackoverflow.com/questions/267399/how-do-you-match-only-valid-roman-numerals-with-a-regular-expression
3429 roman = Regex(r"M{0,4}(CM|CD|D?{0,3})(XC|XL|L?X{0,3})(IX|IV|V?I{0,3})")
3431 # named fields in a regex will be returned as named results
3432 date = Regex(r'(?P<year>\d{4})-(?P<month>\d\d?)-(?P<day>\d\d?)')
3434 # the Regex class will accept regular expressions compiled using the
3435 # re module
3436 import re
3437 parser = pp.Regex(re.compile(r'[0-9]'))
3438 """
3440 def __init__(
3441 self,
3442 pattern: Any,
3443 flags: Union[re.RegexFlag, int] = 0,
3444 as_group_list: bool = False,
3445 as_match: bool = False,
3446 **kwargs,
3447 ) -> None:
3448 super().__init__()
3449 asGroupList: bool = deprecate_argument(kwargs, "asGroupList", False)
3450 asMatch: bool = deprecate_argument(kwargs, "asMatch", False)
3452 asGroupList = asGroupList or as_group_list
3453 asMatch = asMatch or as_match
3455 if isinstance(pattern, str_type):
3456 if not pattern:
3457 raise ValueError("null string passed to Regex; use Empty() instead")
3459 self._re = None
3460 self._may_return_empty = None # type: ignore [assignment]
3461 self.reString = self.pattern = pattern
3463 elif hasattr(pattern, "pattern") and hasattr(pattern, "match"):
3464 self._re = pattern
3465 self._may_return_empty = None # type: ignore [assignment]
3466 self.pattern = self.reString = pattern.pattern
3468 elif callable(pattern):
3469 # defer creating this pattern until we really need it
3470 self.pattern = pattern
3471 self._may_return_empty = None # type: ignore [assignment]
3472 self._re = None
3474 else:
3475 raise TypeError(
3476 "Regex may only be constructed with a string or a compiled RE object,"
3477 " or a callable that takes no arguments and returns a string or a"
3478 " compiled RE object"
3479 )
3481 self.flags = flags
3482 self.errmsg = f"Expected {self.name}"
3483 self.mayIndexError = False
3484 self.asGroupList = asGroupList
3485 self.asMatch = asMatch
3486 if self.asGroupList:
3487 self.parseImpl = self.parseImplAsGroupList # type: ignore [method-assign]
3488 if self.asMatch:
3489 self.parseImpl = self.parseImplAsMatch # type: ignore [method-assign]
3491 def copy(self) -> Regex:
3492 """
3493 Returns a copy of this expression.
3495 Generally only used internally by pyparsing.
3496 """
3497 ret: Regex = cast(Regex, super().copy())
3498 if self.asGroupList:
3499 ret.parseImpl = ret.parseImplAsGroupList # type: ignore [method-assign]
3500 if self.asMatch:
3501 ret.parseImpl = ret.parseImplAsMatch # type: ignore [method-assign]
3502 return ret
3504 @cached_property
3505 def re(self) -> re.Pattern:
3506 """
3507 Property returning the compiled regular expression for this Regex.
3509 Generally only used internally by pyparsing.
3510 """
3511 if self._re:
3512 return self._re
3514 if callable(self.pattern):
3515 # replace self.pattern with the string returned by calling self.pattern()
3516 self.pattern = cast(Callable[[], str], self.pattern)()
3518 # see if we got a compiled RE back instead of a str - if so, we're done
3519 if hasattr(self.pattern, "pattern") and hasattr(self.pattern, "match"):
3520 self._re = cast(re.Pattern[str], self.pattern)
3521 self.pattern = self.reString = self._re.pattern
3522 return self._re
3524 try:
3525 self._re = re.compile(self.pattern, self.flags)
3526 except re.error:
3527 raise ValueError(f"invalid pattern ({self.pattern!r}) passed to Regex")
3528 else:
3529 self._may_return_empty = self.re.match("", pos=0) is not None
3530 return self._re
3532 @cached_property
3533 def re_match(self) -> Callable[[str, int], Any]:
3534 return self.re.match
3536 @property
3537 def mayReturnEmpty(self):
3538 if self._may_return_empty is None:
3539 # force compile of regex pattern, to set may_return_empty flag
3540 self.re # noqa
3541 return self._may_return_empty
3543 @mayReturnEmpty.setter
3544 def mayReturnEmpty(self, value):
3545 self._may_return_empty = value
3547 def _generateDefaultName(self) -> str:
3548 unescaped = repr(self.pattern).replace("\\\\", "\\")
3549 return f"Re:({unescaped})"
3551 def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType:
3552 # explicit check for matching past the length of the string;
3553 # this is done because the re module will not complain about
3554 # a match with `pos > len(instring)`, it will just return ""
3555 if loc > len(instring) and self.mayReturnEmpty:
3556 raise ParseException(instring, loc, self.errmsg, self)
3558 result = self.re_match(instring, loc)
3559 if not result:
3560 raise ParseException(instring, loc, self.errmsg, self)
3562 loc = result.end()
3563 ret = ParseResults(result[0])
3564 d = result.groupdict()
3566 for k, v in d.items():
3567 ret[k] = v
3569 return loc, ret
3571 def parseImplAsGroupList(self, instring, loc, do_actions=True):
3572 if loc > len(instring) and self.mayReturnEmpty:
3573 raise ParseException(instring, loc, self.errmsg, self)
3575 result = self.re_match(instring, loc)
3576 if not result:
3577 raise ParseException(instring, loc, self.errmsg, self)
3579 loc = result.end()
3580 ret = result.groups()
3581 return loc, ret
3583 def parseImplAsMatch(self, instring, loc, do_actions=True):
3584 if loc > len(instring) and self.mayReturnEmpty:
3585 raise ParseException(instring, loc, self.errmsg, self)
3587 result = self.re_match(instring, loc)
3588 if not result:
3589 raise ParseException(instring, loc, self.errmsg, self)
3591 loc = result.end()
3592 ret = result
3593 return loc, ret
3595 def sub(self, repl: str) -> ParserElement:
3596 r"""
3597 Return :class:`Regex` with an attached parse action to transform the parsed
3598 result as if called using `re.sub(expr, repl, string) <https://docs.python.org/3/library/re.html#re.sub>`_.
3600 Example:
3602 .. testcode::
3604 make_html = Regex(r"(\w+):(.*?):").sub(r"<\1>\2</\1>")
3605 print(make_html.transform_string("h1:main title:"))
3607 .. testoutput::
3609 <h1>main title</h1>
3610 """
3611 if self.asGroupList:
3612 raise TypeError("cannot use sub() with Regex(as_group_list=True)")
3614 if self.asMatch and callable(repl):
3615 raise TypeError(
3616 "cannot use sub() with a callable with Regex(as_match=True)"
3617 )
3619 if self.asMatch:
3621 def pa(tokens):
3622 return tokens[0].expand(repl)
3624 else:
3626 def pa(tokens):
3627 return self.re.sub(repl, tokens[0])
3629 return self.add_parse_action(pa)
3632class QuotedString(Token):
3633 r"""
3634 Token for matching strings that are delimited by quoting characters.
3636 Defined with the following parameters:
3638 - ``quote_char`` - string of one or more characters defining the
3639 quote delimiting string
3640 - ``esc_char`` - character to re_escape quotes, typically backslash
3641 (default= ``None``)
3642 - ``esc_quote`` - special quote sequence to re_escape an embedded quote
3643 string (such as SQL's ``""`` to re_escape an embedded ``"``)
3644 (default= ``None``)
3645 - ``multiline`` - boolean indicating whether quotes can span
3646 multiple lines (default= ``False``)
3647 - ``unquote_results`` - boolean indicating whether the matched text
3648 should be unquoted (default= ``True``)
3649 - ``end_quote_char`` - string of one or more characters defining the
3650 end of the quote delimited string (default= ``None`` => same as
3651 quote_char)
3652 - ``convert_whitespace_escapes`` - convert escaped whitespace
3653 (``'\t'``, ``'\n'``, etc.) to actual whitespace
3654 (default= ``True``)
3656 .. caution:: ``convert_whitespace_escapes`` has no effect if
3657 ``unquote_results`` is ``False``.
3659 Example:
3661 .. doctest::
3663 >>> qs = QuotedString('"')
3664 >>> print(qs.search_string('lsjdf "This is the quote" sldjf'))
3665 [['This is the quote']]
3666 >>> complex_qs = QuotedString('{{', end_quote_char='}}')
3667 >>> print(complex_qs.search_string(
3668 ... 'lsjdf {{This is the "quote"}} sldjf'))
3669 [['This is the "quote"']]
3670 >>> sql_qs = QuotedString('"', esc_quote='""')
3671 >>> print(sql_qs.search_string(
3672 ... 'lsjdf "This is the quote with ""embedded"" quotes" sldjf'))
3673 [['This is the quote with "embedded" quotes']]
3674 """
3676 ws_map = dict(((r"\t", "\t"), (r"\n", "\n"), (r"\f", "\f"), (r"\r", "\r")))
3678 def __init__(
3679 self,
3680 quote_char: str = "",
3681 esc_char: typing.Optional[str] = None,
3682 esc_quote: typing.Optional[str] = None,
3683 multiline: bool = False,
3684 unquote_results: bool = True,
3685 end_quote_char: typing.Optional[str] = None,
3686 convert_whitespace_escapes: bool = True,
3687 **kwargs,
3688 ) -> None:
3689 super().__init__()
3690 quoteChar: str = deprecate_argument(kwargs, "quoteChar", "")
3691 escChar: str = deprecate_argument(kwargs, "escChar", None)
3692 escQuote: str = deprecate_argument(kwargs, "escQuote", None)
3693 unquoteResults: bool = deprecate_argument(kwargs, "unquoteResults", True)
3694 endQuoteChar: typing.Optional[str] = deprecate_argument(
3695 kwargs, "endQuoteChar", None
3696 )
3697 convertWhitespaceEscapes: bool = deprecate_argument(
3698 kwargs, "convertWhitespaceEscapes", True
3699 )
3701 esc_char = escChar or esc_char
3702 esc_quote = escQuote or esc_quote
3703 unquote_results = unquoteResults and unquote_results
3704 end_quote_char = endQuoteChar or end_quote_char
3705 convert_whitespace_escapes = (
3706 convertWhitespaceEscapes and convert_whitespace_escapes
3707 )
3708 quote_char = quoteChar or quote_char
3710 # remove white space from quote chars
3711 quote_char = quote_char.strip()
3712 if not quote_char:
3713 raise ValueError("quote_char cannot be the empty string")
3715 if end_quote_char is None:
3716 end_quote_char = quote_char
3717 else:
3718 end_quote_char = end_quote_char.strip()
3719 if not end_quote_char:
3720 raise ValueError("end_quote_char cannot be the empty string")
3722 self.quote_char: str = quote_char
3723 self.quote_char_len: int = len(quote_char)
3724 self.first_quote_char: str = quote_char[0]
3725 self.end_quote_char: str = end_quote_char
3726 self.end_quote_char_len: int = len(end_quote_char)
3727 self.esc_char: str = esc_char or ""
3728 self.has_esc_char: bool = esc_char is not None
3729 self.esc_quote: str = esc_quote or ""
3730 self.unquote_results: bool = unquote_results
3731 self.convert_whitespace_escapes: bool = convert_whitespace_escapes
3732 self.multiline = multiline
3733 self.re_flags = re.RegexFlag(0)
3735 # fmt: off
3736 # build up re pattern for the content between the quote delimiters
3737 inner_pattern: list[str] = []
3739 if esc_quote:
3740 inner_pattern.append(rf"(?:{re.escape(esc_quote)})")
3742 if esc_char:
3743 inner_pattern.append(rf"(?:{re.escape(esc_char)}.)")
3745 if len(self.end_quote_char) > 1:
3746 inner_pattern.append(
3747 "(?:"
3748 + "|".join(
3749 f"(?:{re.escape(self.end_quote_char[:i])}(?!{re.escape(self.end_quote_char[i:])}))"
3750 for i in range(len(self.end_quote_char) - 1, 0, -1)
3751 )
3752 + ")"
3753 )
3755 if self.multiline:
3756 self.re_flags |= re.MULTILINE | re.DOTALL
3757 inner_pattern.append(
3758 rf"(?:[^{_escape_regex_range_chars(self.end_quote_char[0])}"
3759 rf"{(_escape_regex_range_chars(self.esc_char) if self.has_esc_char else '')}])"
3760 )
3761 else:
3762 inner_pattern.append(
3763 rf"(?:[^{_escape_regex_range_chars(self.end_quote_char[0])}\n\r"
3764 rf"{(_escape_regex_range_chars(self.esc_char) if self.has_esc_char else '')}])"
3765 )
3767 self.pattern = "".join(
3768 [
3769 re.escape(self.quote_char),
3770 "(?:",
3771 '|'.join(inner_pattern),
3772 ")*",
3773 re.escape(self.end_quote_char),
3774 ]
3775 )
3777 if self.unquote_results:
3778 if self.convert_whitespace_escapes:
3779 self.unquote_scan_re = re.compile(
3780 rf"({'|'.join(re.escape(k) for k in self.ws_map)})"
3781 rf"|(\\[0-7]{3}|\\0|\\x[0-9a-fA-F]{2}|\\u[0-9a-fA-F]{4})"
3782 rf"|({re.escape(self.esc_char)}.)"
3783 rf"|(\n|.)",
3784 flags=self.re_flags,
3785 )
3786 else:
3787 self.unquote_scan_re = re.compile(
3788 rf"({re.escape(self.esc_char)}.)"
3789 rf"|(\n|.)",
3790 flags=self.re_flags
3791 )
3792 # fmt: on
3794 try:
3795 self.re = re.compile(self.pattern, self.re_flags)
3796 self.reString = self.pattern
3797 self.re_match = self.re.match
3798 except re.error:
3799 raise ValueError(f"invalid pattern {self.pattern!r} passed to Regex")
3801 self.errmsg = f"Expected {self.name}"
3802 self.mayIndexError = False
3803 self._may_return_empty = True
3805 def _generateDefaultName(self) -> str:
3806 if self.quote_char == self.end_quote_char and isinstance(
3807 self.quote_char, str_type
3808 ):
3809 return f"string enclosed in {self.quote_char!r}"
3811 return f"quoted string, starting with {self.quote_char} ending with {self.end_quote_char}"
3813 def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType:
3814 # check first character of opening quote to see if that is a match
3815 # before doing the more complicated regex match
3816 result = (
3817 instring[loc] == self.first_quote_char
3818 and self.re_match(instring, loc)
3819 or None
3820 )
3821 if not result:
3822 raise ParseException(instring, loc, self.errmsg, self)
3824 # get ending loc and matched string from regex matching result
3825 loc = result.end()
3826 ret = result[0]
3828 if self.unquote_results:
3829 # strip off quotes
3830 ret = ret[self.quote_char_len : -self.end_quote_char_len]
3832 if isinstance(ret, str_type):
3833 # fmt: off
3834 if self.convert_whitespace_escapes:
3835 # as we iterate over matches in the input string,
3836 # collect from whichever match group of the unquote_scan_re
3837 # regex matches (only 1 group will match at any given time)
3838 ret = "".join(
3839 # match group 1 matches \t, \n, etc.
3840 self.ws_map[g] if (g := match[1])
3841 # match group 2 matches escaped octal, null, hex, and Unicode
3842 # sequences
3843 else _convert_escaped_numerics_to_char(g[1:]) if (g := match[2])
3844 # match group 3 matches escaped characters
3845 else g[-1] if (g := match[3])
3846 # match group 4 matches any character
3847 else match[4]
3848 for match in self.unquote_scan_re.finditer(ret)
3849 )
3850 else:
3851 ret = "".join(
3852 # match group 1 matches escaped characters
3853 g[-1] if (g := match[1])
3854 # match group 2 matches any character
3855 else match[2]
3856 for match in self.unquote_scan_re.finditer(ret)
3857 )
3858 # fmt: on
3860 # replace escaped quotes
3861 if self.esc_quote:
3862 ret = ret.replace(self.esc_quote, self.end_quote_char)
3864 return loc, ret
3867class CharsNotIn(Token):
3868 """Token for matching words composed of characters *not* in a given
3869 set (will include whitespace in matched characters if not listed in
3870 the provided exclusion set - see example). Defined with string
3871 containing all disallowed characters, and an optional minimum,
3872 maximum, and/or exact length. The default value for ``min`` is
3873 1 (a minimum value < 1 is not valid); the default values for
3874 ``max`` and ``exact`` are 0, meaning no maximum or exact
3875 length restriction.
3877 Example:
3879 .. testcode::
3881 # define a comma-separated-value as anything that is not a ','
3882 csv_value = CharsNotIn(',')
3883 print(
3884 DelimitedList(csv_value).parse_string(
3885 "dkls,lsdkjf,s12 34,@!#,213"
3886 )
3887 )
3889 prints:
3891 .. testoutput::
3893 ['dkls', 'lsdkjf', 's12 34', '@!#', '213']
3894 """
3896 def __init__(
3897 self, not_chars: str = "", min: int = 1, max: int = 0, exact: int = 0, **kwargs
3898 ) -> None:
3899 super().__init__()
3900 notChars: str = deprecate_argument(kwargs, "notChars", "")
3902 self.skipWhitespace = False
3903 self.notChars = not_chars or notChars
3904 self.notCharsSet = set(self.notChars)
3906 if min < 1:
3907 raise ValueError(
3908 "cannot specify a minimum length < 1; use"
3909 " Opt(CharsNotIn()) if zero-length char group is permitted"
3910 )
3912 self.minLen = min
3914 if max > 0:
3915 self.maxLen = max
3916 else:
3917 self.maxLen = _MAX_INT
3919 if exact > 0:
3920 self.maxLen = exact
3921 self.minLen = exact
3923 self.errmsg = f"Expected {self.name}"
3924 self._may_return_empty = self.minLen == 0
3925 self.mayIndexError = False
3927 def _generateDefaultName(self) -> str:
3928 not_chars_str = _collapse_string_to_ranges(self.notChars)
3929 if len(not_chars_str) > 16:
3930 return f"!W:({self.notChars[: 16 - 3]}...)"
3931 else:
3932 return f"!W:({self.notChars})"
3934 def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType:
3935 notchars = self.notCharsSet
3936 if instring[loc] in notchars:
3937 raise ParseException(instring, loc, self.errmsg, self)
3939 start = loc
3940 loc += 1
3941 maxlen = min(start + self.maxLen, len(instring))
3942 while loc < maxlen and instring[loc] not in notchars:
3943 loc += 1
3945 if loc - start < self.minLen:
3946 raise ParseException(instring, loc, self.errmsg, self)
3948 return loc, instring[start:loc]
3951class White(Token):
3952 """Special matching class for matching whitespace. Normally,
3953 whitespace is ignored by pyparsing grammars. This class is included
3954 when some whitespace structures are significant. Define with
3955 a string containing the whitespace characters to be matched; default
3956 is ``" \\t\\r\\n"``. Also takes optional ``min``,
3957 ``max``, and ``exact`` arguments, as defined for the
3958 :class:`Word` class.
3959 """
3961 whiteStrs = {
3962 " ": "<SP>",
3963 "\t": "<TAB>",
3964 "\n": "<LF>",
3965 "\r": "<CR>",
3966 "\f": "<FF>",
3967 "\u00a0": "<NBSP>",
3968 "\u1680": "<OGHAM_SPACE_MARK>",
3969 "\u180e": "<MONGOLIAN_VOWEL_SEPARATOR>",
3970 "\u2000": "<EN_QUAD>",
3971 "\u2001": "<EM_QUAD>",
3972 "\u2002": "<EN_SPACE>",
3973 "\u2003": "<EM_SPACE>",
3974 "\u2004": "<THREE-PER-EM_SPACE>",
3975 "\u2005": "<FOUR-PER-EM_SPACE>",
3976 "\u2006": "<SIX-PER-EM_SPACE>",
3977 "\u2007": "<FIGURE_SPACE>",
3978 "\u2008": "<PUNCTUATION_SPACE>",
3979 "\u2009": "<THIN_SPACE>",
3980 "\u200a": "<HAIR_SPACE>",
3981 "\u200b": "<ZERO_WIDTH_SPACE>",
3982 "\u202f": "<NNBSP>",
3983 "\u205f": "<MMSP>",
3984 "\u3000": "<IDEOGRAPHIC_SPACE>",
3985 }
3987 def __init__(
3988 self, ws: str = " \t\r\n", min: int = 1, max: int = 0, exact: int = 0
3989 ) -> None:
3990 super().__init__()
3991 self.matchWhite = ws
3992 self.set_whitespace_chars(
3993 "".join(c for c in self.whiteStrs if c not in self.matchWhite),
3994 copy_defaults=True,
3995 )
3996 # self.leave_whitespace()
3997 self._may_return_empty = True
3998 self.errmsg = f"Expected {self.name}"
4000 self.minLen = min
4002 if max > 0:
4003 self.maxLen = max
4004 else:
4005 self.maxLen = _MAX_INT
4007 if exact > 0:
4008 self.maxLen = exact
4009 self.minLen = exact
4011 def _generateDefaultName(self) -> str:
4012 return "".join(White.whiteStrs[c] for c in self.matchWhite)
4014 def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType:
4015 if instring[loc] not in self.matchWhite:
4016 raise ParseException(instring, loc, self.errmsg, self)
4017 start = loc
4018 loc += 1
4019 maxloc = start + self.maxLen
4020 maxloc = min(maxloc, len(instring))
4021 while loc < maxloc and instring[loc] in self.matchWhite:
4022 loc += 1
4024 if loc - start < self.minLen:
4025 raise ParseException(instring, loc, self.errmsg, self)
4027 return loc, instring[start:loc]
4030class PositionToken(Token):
4031 def __init__(self) -> None:
4032 super().__init__()
4033 self._may_return_empty = True
4034 self.mayIndexError = False
4037class GoToColumn(PositionToken):
4038 """Token to advance to a specific column of input text; useful for
4039 tabular report scraping.
4040 """
4042 def __init__(self, colno: int) -> None:
4043 super().__init__()
4044 self.col = colno
4046 def preParse(self, instring: str, loc: int) -> int:
4047 if col(loc, instring) == self.col:
4048 return loc
4050 instrlen = len(instring)
4051 if self.ignoreExprs:
4052 loc = self._skipIgnorables(instring, loc)
4053 while (
4054 loc < instrlen
4055 and instring[loc].isspace()
4056 and col(loc, instring) != self.col
4057 ):
4058 loc += 1
4060 return loc
4062 def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType:
4063 thiscol = col(loc, instring)
4064 if thiscol > self.col:
4065 raise ParseException(instring, loc, "Text not in expected column", self)
4066 newloc = loc + self.col - thiscol
4067 ret = instring[loc:newloc]
4068 return newloc, ret
4071class LineStart(PositionToken):
4072 r"""Matches if current position is at the logical beginning of a line (after skipping whitespace)
4073 within the parse string
4075 Example:
4077 .. testcode::
4079 test = '''\
4080 AAA this line
4081 AAA and this line
4082 AAA and even this line
4083 B AAA but definitely not this line
4084 '''
4086 for t in (LineStart() + 'AAA' + rest_of_line).search_string(test):
4087 print(t)
4089 prints:
4091 .. testoutput::
4093 ['AAA', ' this line']
4094 ['AAA', ' and this line']
4095 ['AAA', ' and even this line']
4097 """
4099 def __init__(self) -> None:
4100 super().__init__()
4101 self.leave_whitespace()
4102 self.orig_whiteChars = set() | self.whiteChars
4103 self.whiteChars.discard("\n")
4104 self.skipper = Empty().set_whitespace_chars(self.whiteChars)
4105 self.set_name("start of line")
4107 def preParse(self, instring: str, loc: int) -> int:
4108 if loc == 0:
4109 return loc
4111 ret = self.skipper.preParse(instring, loc)
4113 if "\n" in self.orig_whiteChars:
4114 while instring[ret : ret + 1] == "\n":
4115 ret = self.skipper.preParse(instring, ret + 1)
4117 return ret
4119 def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType:
4120 if col(loc, instring) == 1:
4121 return loc, []
4122 raise ParseException(instring, loc, self.errmsg, self)
4125class LineEnd(PositionToken):
4126 """Matches if current position is at the end of a line within the
4127 parse string
4128 """
4130 def __init__(self) -> None:
4131 super().__init__()
4132 self.whiteChars.discard("\n")
4133 self.set_whitespace_chars(self.whiteChars, copy_defaults=False)
4134 self.set_name("end of line")
4136 def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType:
4137 if loc < len(instring):
4138 if instring[loc] == "\n":
4139 return loc + 1, "\n"
4140 else:
4141 raise ParseException(instring, loc, self.errmsg, self)
4142 elif loc == len(instring):
4143 return loc + 1, []
4144 else:
4145 raise ParseException(instring, loc, self.errmsg, self)
4148class StringStart(PositionToken):
4149 """Matches if current position is at the beginning of the parse
4150 string
4151 """
4153 def __init__(self) -> None:
4154 super().__init__()
4155 self.set_name("start of text")
4157 def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType:
4158 # see if entire string up to here is just whitespace and ignoreables
4159 if loc != 0 and loc != self.preParse(instring, 0):
4160 raise ParseException(instring, loc, self.errmsg, self)
4162 return loc, []
4165class StringEnd(PositionToken):
4166 """
4167 Matches if current position is at the end of the parse string
4168 """
4170 def __init__(self) -> None:
4171 super().__init__()
4172 self.set_name("end of text")
4174 def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType:
4175 if loc < len(instring):
4176 raise ParseException(instring, loc, self.errmsg, self)
4177 if loc == len(instring):
4178 return loc + 1, []
4179 if loc > len(instring):
4180 return loc, []
4182 raise ParseException(instring, loc, self.errmsg, self)
4185class WordStart(PositionToken):
4186 """Matches if the current position is at the beginning of a
4187 :class:`Word`, and is not preceded by any character in a given
4188 set of ``word_chars`` (default= ``printables``). To emulate the
4189 ``\b`` behavior of regular expressions, use
4190 ``WordStart(alphanums)``. ``WordStart`` will also match at
4191 the beginning of the string being parsed, or at the beginning of
4192 a line.
4193 """
4195 def __init__(self, word_chars: str = printables, **kwargs) -> None:
4196 wordChars: str = deprecate_argument(kwargs, "wordChars", printables)
4198 wordChars = word_chars if wordChars == printables else wordChars
4199 super().__init__()
4200 self.wordChars = set(wordChars)
4201 self.set_name("start of a word")
4203 def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType:
4204 if loc != 0:
4205 if (
4206 instring[loc - 1] in self.wordChars
4207 or instring[loc] not in self.wordChars
4208 ):
4209 raise ParseException(instring, loc, self.errmsg, self)
4210 return loc, []
4213class WordEnd(PositionToken):
4214 """Matches if the current position is at the end of a :class:`Word`,
4215 and is not followed by any character in a given set of ``word_chars``
4216 (default= ``printables``). To emulate the ``\b`` behavior of
4217 regular expressions, use ``WordEnd(alphanums)``. ``WordEnd``
4218 will also match at the end of the string being parsed, or at the end
4219 of a line.
4220 """
4222 def __init__(self, word_chars: str = printables, **kwargs) -> None:
4223 wordChars: str = deprecate_argument(kwargs, "wordChars", printables)
4225 wordChars = word_chars if wordChars == printables else wordChars
4226 super().__init__()
4227 self.wordChars = set(wordChars)
4228 self.skipWhitespace = False
4229 self.set_name("end of a word")
4231 def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType:
4232 instrlen = len(instring)
4233 if instrlen > 0 and loc < instrlen:
4234 if (
4235 instring[loc] in self.wordChars
4236 or instring[loc - 1] not in self.wordChars
4237 ):
4238 raise ParseException(instring, loc, self.errmsg, self)
4239 return loc, []
4242class Tag(Token):
4243 """
4244 A meta-element for inserting a named result into the parsed
4245 tokens that may be checked later in a parse action or while
4246 processing the parsed results. Accepts an optional tag value,
4247 defaulting to `True`.
4249 Example:
4251 .. doctest::
4253 >>> end_punc = "." | ("!" + Tag("enthusiastic"))
4254 >>> greeting = "Hello," + Word(alphas) + end_punc
4256 >>> result = greeting.parse_string("Hello, World.")
4257 >>> print(result.dump())
4258 ['Hello,', 'World', '.']
4260 >>> result = greeting.parse_string("Hello, World!")
4261 >>> print(result.dump())
4262 ['Hello,', 'World', '!']
4263 - enthusiastic: True
4265 .. versionadded:: 3.1.0
4266 """
4268 def __init__(self, tag_name: str, value: Any = True) -> None:
4269 super().__init__()
4270 self._may_return_empty = True
4271 self.mayIndexError = False
4272 self.leave_whitespace()
4273 self.tag_name = tag_name
4274 self.tag_value = value
4275 self.add_parse_action(self._add_tag)
4276 self.show_in_diagram = False
4278 def _add_tag(self, tokens: ParseResults):
4279 tokens[self.tag_name] = self.tag_value
4281 def _generateDefaultName(self) -> str:
4282 return f"{type(self).__name__}:{self.tag_name}={self.tag_value!r}"
4285class ParseExpression(ParserElement):
4286 """Abstract subclass of ParserElement, for combining and
4287 post-processing parsed tokens.
4288 """
4290 def __init__(
4291 self, exprs: typing.Iterable[ParserElement], savelist: bool = False
4292 ) -> None:
4293 super().__init__(savelist)
4294 self.exprs: list[ParserElement]
4295 if isinstance(exprs, _generatorType):
4296 exprs = list(exprs)
4298 if isinstance(exprs, str_type):
4299 self.exprs = [self._literalStringClass(exprs)]
4300 elif isinstance(exprs, ParserElement):
4301 self.exprs = [exprs]
4302 elif isinstance(exprs, Iterable):
4303 exprs = list(exprs)
4304 # if sequence of strings provided, wrap with Literal
4305 if any(isinstance(expr, str_type) for expr in exprs):
4306 exprs = (
4307 self._literalStringClass(e) if isinstance(e, str_type) else e
4308 for e in exprs
4309 )
4310 self.exprs = list(exprs)
4311 else:
4312 try:
4313 self.exprs = list(exprs)
4314 except TypeError:
4315 self.exprs = [exprs]
4316 self.callPreparse = False
4318 def recurse(self) -> list[ParserElement]:
4319 return self.exprs[:]
4321 def append(self, other) -> ParserElement:
4322 """
4323 Add an expression to the list of expressions related to this ParseExpression instance.
4324 """
4325 self.exprs.append(other)
4326 self._defaultName = None
4327 return self
4329 def leave_whitespace(self, recursive: bool = True) -> ParserElement:
4330 """
4331 Extends ``leave_whitespace`` defined in base class, and also invokes ``leave_whitespace`` on
4332 all contained expressions.
4333 """
4334 super().leave_whitespace(recursive)
4336 if recursive:
4337 self.exprs = [e.copy() for e in self.exprs]
4338 for e in self.exprs:
4339 e.leave_whitespace(recursive)
4340 return self
4342 def ignore_whitespace(self, recursive: bool = True) -> ParserElement:
4343 """
4344 Extends ``ignore_whitespace`` defined in base class, and also invokes ``ignore_whitespace`` on
4345 all contained expressions.
4346 """
4347 super().ignore_whitespace(recursive)
4348 if recursive:
4349 self.exprs = [e.copy() for e in self.exprs]
4350 for e in self.exprs:
4351 e.ignore_whitespace(recursive)
4352 return self
4354 def ignore(self, other) -> ParserElement:
4355 """
4356 Define expression to be ignored (e.g., comments) while doing pattern
4357 matching; may be called repeatedly, to define multiple comment or other
4358 ignorable patterns.
4359 """
4360 if isinstance(other, Suppress):
4361 if other not in self.ignoreExprs:
4362 super().ignore(other)
4363 for e in self.exprs:
4364 e.ignore(self.ignoreExprs[-1])
4365 else:
4366 super().ignore(other)
4367 for e in self.exprs:
4368 e.ignore(self.ignoreExprs[-1])
4369 return self
4371 def _generateDefaultName(self) -> str:
4372 return f"{type(self).__name__}:({self.exprs})"
4374 def streamline(self) -> ParserElement:
4375 if self.streamlined:
4376 return self
4378 super().streamline()
4380 for e in self.exprs:
4381 e.streamline()
4383 # collapse nested :class:`And`'s of the form ``And(And(And(a, b), c), d)`` to ``And(a, b, c, d)``
4384 # but only if there are no parse actions or resultsNames on the nested And's
4385 # (likewise for :class:`Or`'s and :class:`MatchFirst`'s)
4386 if len(self.exprs) == 2:
4387 other = self.exprs[0]
4388 if (
4389 isinstance(other, self.__class__)
4390 and not other.parseAction
4391 and other.resultsName is None
4392 and not other.debug
4393 ):
4394 self.exprs = other.exprs[:] + [self.exprs[1]]
4395 self._defaultName = None
4396 self._may_return_empty |= other.mayReturnEmpty
4397 self.mayIndexError |= other.mayIndexError
4399 other = self.exprs[-1]
4400 if (
4401 isinstance(other, self.__class__)
4402 and not other.parseAction
4403 and other.resultsName is None
4404 and not other.debug
4405 ):
4406 self.exprs = self.exprs[:-1] + other.exprs[:]
4407 self._defaultName = None
4408 self._may_return_empty |= other.mayReturnEmpty
4409 self.mayIndexError |= other.mayIndexError
4411 self.errmsg = f"Expected {self}"
4413 return self
4415 def validate(self, validateTrace=None) -> None:
4416 warnings.warn(
4417 "ParserElement.validate() is deprecated, and should not be used to check for left recursion",
4418 PyparsingDeprecationWarning,
4419 stacklevel=2,
4420 )
4421 tmp = (validateTrace if validateTrace is not None else [])[:] + [self]
4422 for e in self.exprs:
4423 e.validate(tmp)
4424 self._checkRecursion([])
4426 def copy(self) -> ParserElement:
4427 """
4428 Returns a copy of this expression.
4430 Generally only used internally by pyparsing.
4431 """
4432 ret = super().copy()
4433 ret = typing.cast(ParseExpression, ret)
4434 ret.exprs = [e.copy() for e in self.exprs]
4435 return ret
4437 def _setResultsName(self, name, list_all_matches=False) -> ParserElement:
4438 if not (
4439 __diag__.warn_ungrouped_named_tokens_in_collection
4440 and Diagnostics.warn_ungrouped_named_tokens_in_collection
4441 not in self.suppress_warnings_
4442 ):
4443 return super()._setResultsName(name, list_all_matches)
4445 for e in self.exprs:
4446 if (
4447 isinstance(e, ParserElement)
4448 and e.resultsName
4449 and (
4450 Diagnostics.warn_ungrouped_named_tokens_in_collection
4451 not in e.suppress_warnings_
4452 )
4453 ):
4454 warning = (
4455 "warn_ungrouped_named_tokens_in_collection:"
4456 f" setting results name {name!r} on {type(self).__name__} expression"
4457 f" collides with {e.resultsName!r} on contained expression"
4458 )
4459 warnings.warn(warning, PyparsingDiagnosticWarning, stacklevel=3)
4460 break
4462 return super()._setResultsName(name, list_all_matches)
4464 # Compatibility synonyms
4465 # fmt: off
4466 leaveWhitespace = replaced_by_pep8("leaveWhitespace", leave_whitespace)
4467 ignoreWhitespace = replaced_by_pep8("ignoreWhitespace", ignore_whitespace)
4468 # fmt: on
4471class And(ParseExpression):
4472 """
4473 Requires all given :class:`ParserElement` s to be found in the given order.
4474 Expressions may be separated by whitespace.
4475 May be constructed using the ``'+'`` operator.
4476 May also be constructed using the ``'-'`` operator, which will
4477 suppress backtracking.
4479 Example:
4481 .. testcode::
4483 integer = Word(nums)
4484 name_expr = Word(alphas)[1, ...]
4486 expr = And([integer("id"), name_expr("name"), integer("age")])
4487 # more easily written as:
4488 expr = integer("id") + name_expr("name") + integer("age")
4489 """
4491 class _ErrorStop(Empty):
4492 def __init__(self, *args, **kwargs) -> None:
4493 super().__init__(*args, **kwargs)
4494 self.leave_whitespace()
4496 def _generateDefaultName(self) -> str:
4497 return "-"
4499 def __init__(
4500 self,
4501 exprs_arg: typing.Iterable[Union[ParserElement, str]],
4502 savelist: bool = True,
4503 ) -> None:
4504 # instantiate exprs as a list, converting strs to ParserElements
4505 exprs: list[ParserElement] = [
4506 self._literalStringClass(e) if isinstance(e, str) else e for e in exprs_arg
4507 ]
4509 # convert any Ellipsis elements to SkipTo
4510 if Ellipsis in exprs:
4512 # Ellipsis cannot be the last element
4513 if exprs[-1] is Ellipsis:
4514 raise Exception("cannot construct And with sequence ending in ...")
4516 tmp: list[ParserElement] = []
4517 for cur_expr, next_expr in zip(exprs, exprs[1:]):
4518 if cur_expr is Ellipsis:
4519 tmp.append(SkipTo(next_expr)("_skipped*"))
4520 else:
4521 tmp.append(cur_expr)
4523 exprs[:-1] = tmp
4525 super().__init__(exprs, savelist)
4526 if self.exprs:
4527 self._may_return_empty = all(e.mayReturnEmpty for e in self.exprs)
4528 if not isinstance(self.exprs[0], White):
4529 self.set_whitespace_chars(
4530 self.exprs[0].whiteChars,
4531 copy_defaults=self.exprs[0].copyDefaultWhiteChars,
4532 )
4533 self.skipWhitespace = self.exprs[0].skipWhitespace
4534 else:
4535 self.skipWhitespace = False
4536 else:
4537 self._may_return_empty = True
4538 self.callPreparse = True
4540 def streamline(self) -> ParserElement:
4541 """
4542 Collapse `And` expressions like `And(And(And(A, B), C), D)`
4543 to `And(A, B, C, D)`.
4545 .. doctest::
4547 >>> expr = Word("A") + Word("B") + Word("C") + Word("D")
4548 >>> # Using '+' operator creates nested And expression
4549 >>> expr
4550 {{{W:(A) W:(B)} W:(C)} W:(D)}
4551 >>> # streamline simplifies to a single And with multiple expressions
4552 >>> expr.streamline()
4553 {W:(A) W:(B) W:(C) W:(D)}
4555 Guards against collapsing out expressions that have special features,
4556 such as results names or parse actions.
4558 Resolves pending Skip commands defined using `...` terms.
4559 """
4560 # collapse any _PendingSkip's
4561 if self.exprs and any(
4562 isinstance(e, ParseExpression)
4563 and e.exprs
4564 and isinstance(e.exprs[-1], _PendingSkip)
4565 for e in self.exprs[:-1]
4566 ):
4567 deleted_expr_marker = NoMatch()
4568 for i, e in enumerate(self.exprs[:-1]):
4569 if e is deleted_expr_marker:
4570 continue
4571 if (
4572 isinstance(e, ParseExpression)
4573 and e.exprs
4574 and isinstance(e.exprs[-1], _PendingSkip)
4575 ):
4576 e.exprs[-1] = e.exprs[-1] + self.exprs[i + 1]
4577 self.exprs[i + 1] = deleted_expr_marker
4578 self.exprs = [e for e in self.exprs if e is not deleted_expr_marker]
4580 super().streamline()
4582 # link any IndentedBlocks to the prior expression
4583 prev: ParserElement
4584 cur: ParserElement
4585 for prev, cur in zip(self.exprs, self.exprs[1:]):
4586 # traverse cur or any first embedded expr of cur looking for an IndentedBlock
4587 # (but watch out for recursive grammar)
4588 seen = set()
4589 while True:
4590 if id(cur) in seen:
4591 break
4592 seen.add(id(cur))
4593 if isinstance(cur, IndentedBlock):
4594 prev.add_parse_action(
4595 lambda s, l, t, cur_=cur: setattr(
4596 cur_, "parent_anchor", col(l, s)
4597 )
4598 )
4599 break
4600 subs = cur.recurse()
4601 next_first = next(iter(subs), None)
4602 if next_first is None:
4603 break
4604 cur = typing.cast(ParserElement, next_first)
4606 self._may_return_empty = all(e.mayReturnEmpty for e in self.exprs)
4607 return self
4609 def parseImpl(self, instring, loc, do_actions=True):
4610 # pass False as callPreParse arg to _parse for first element, since we already
4611 # pre-parsed the string as part of our And pre-parsing
4612 loc, resultlist = self.exprs[0]._parse(
4613 instring, loc, do_actions, callPreParse=False
4614 )
4615 errorStop = False
4616 for e in self.exprs[1:]:
4617 # if isinstance(e, And._ErrorStop):
4618 if type(e) is And._ErrorStop:
4619 errorStop = True
4620 continue
4621 if errorStop:
4622 try:
4623 loc, exprtokens = e._parse(instring, loc, do_actions)
4624 except ParseSyntaxException:
4625 raise
4626 except ParseBaseException as pe:
4627 pe.__traceback__ = None
4628 raise ParseSyntaxException._from_exception(pe)
4629 except IndexError:
4630 raise ParseSyntaxException(
4631 instring, len(instring), self.errmsg, self
4632 )
4633 else:
4634 loc, exprtokens = e._parse(instring, loc, do_actions)
4635 resultlist += exprtokens
4636 return loc, resultlist
4638 def __iadd__(self, other):
4639 if isinstance(other, str_type):
4640 other = self._literalStringClass(other)
4641 if not isinstance(other, ParserElement):
4642 return NotImplemented
4643 return self.append(other) # And([self, other])
4645 def _checkRecursion(self, parseElementList):
4646 subRecCheckList = parseElementList[:] + [self]
4647 for e in self.exprs:
4648 e._checkRecursion(subRecCheckList)
4649 if not e.mayReturnEmpty:
4650 break
4652 def _generateDefaultName(self) -> str:
4653 inner = " ".join(str(e) for e in self.exprs)
4654 # strip off redundant inner {}'s
4655 while len(inner) > 1 and inner[0 :: len(inner) - 1] == "{}":
4656 inner = inner[1:-1]
4657 return f"{{{inner}}}"
4660class Or(ParseExpression):
4661 """Requires that at least one :class:`ParserElement` is found. If
4662 two expressions match, the expression that matches the longest
4663 string will be used. May be constructed using the ``'^'``
4664 operator.
4666 Example:
4668 .. testcode::
4670 # construct Or using '^' operator
4672 number = Word(nums) ^ Combine(Word(nums) + '.' + Word(nums))
4673 print(number.search_string("123 3.1416 789"))
4675 prints:
4677 .. testoutput::
4679 [['123'], ['3.1416'], ['789']]
4680 """
4682 def __init__(
4683 self, exprs: typing.Iterable[ParserElement], savelist: bool = False
4684 ) -> None:
4685 super().__init__(exprs, savelist)
4686 if self.exprs:
4687 self._may_return_empty = any(e.mayReturnEmpty for e in self.exprs)
4688 self.skipWhitespace = all(e.skipWhitespace for e in self.exprs)
4689 else:
4690 self._may_return_empty = True
4692 def streamline(self) -> ParserElement:
4693 super().streamline()
4694 if self.exprs:
4695 self._may_return_empty = any(e.mayReturnEmpty for e in self.exprs)
4696 self.saveAsList = any(e.saveAsList for e in self.exprs)
4697 self.skipWhitespace = all(
4698 e.skipWhitespace and not isinstance(e, White) for e in self.exprs
4699 )
4700 else:
4701 self.saveAsList = False
4702 return self
4704 def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType:
4705 maxExcLoc = -1
4706 maxException = None
4707 matches: list[tuple[int, ParserElement]] = []
4708 fatals: list[ParseFatalException] = []
4709 if all(e.callPreparse for e in self.exprs):
4710 loc = self.preParse(instring, loc)
4711 for e in self.exprs:
4712 try:
4713 loc2 = e.try_parse(instring, loc, raise_fatal=True)
4714 except ParseFatalException as pfe:
4715 pfe.__traceback__ = None
4716 pfe.parser_element = e
4717 fatals.append(pfe)
4718 maxException = None
4719 maxExcLoc = -1
4720 except ParseException as err:
4721 if not fatals:
4722 err.__traceback__ = None
4723 if err.loc > maxExcLoc:
4724 maxException = err
4725 maxExcLoc = err.loc
4726 except IndexError:
4727 if len(instring) > maxExcLoc:
4728 maxException = ParseException(
4729 instring, len(instring), e.errmsg, self
4730 )
4731 maxExcLoc = len(instring)
4732 else:
4733 # save match among all matches, to retry longest to shortest
4734 matches.append((loc2, e))
4736 if matches:
4737 # re-evaluate all matches in descending order of length of match, in case attached actions
4738 # might change whether or how much they match of the input.
4739 matches.sort(key=itemgetter(0), reverse=True)
4741 if not do_actions:
4742 # no further conditions or parse actions to change the selection of
4743 # alternative, so the first match will be the best match
4744 best_expr = matches[0][1]
4745 return best_expr._parse(instring, loc, do_actions)
4747 longest: tuple[int, typing.Optional[ParseResults]] = -1, None
4748 for loc1, expr1 in matches:
4749 if loc1 <= longest[0]:
4750 # already have a longer match than this one will deliver, we are done
4751 return longest
4753 try:
4754 loc2, toks = expr1._parse(instring, loc, do_actions)
4755 except ParseException as err:
4756 err.__traceback__ = None
4757 if err.loc > maxExcLoc:
4758 maxException = err
4759 maxExcLoc = err.loc
4760 else:
4761 if loc2 >= loc1:
4762 return loc2, toks
4763 # didn't match as much as before
4764 elif loc2 > longest[0]:
4765 longest = loc2, toks
4767 if longest != (-1, None):
4768 return longest
4770 if fatals:
4771 if len(fatals) > 1:
4772 fatals.sort(key=lambda e: -e.loc)
4773 if fatals[0].loc == fatals[1].loc:
4774 fatals.sort(key=lambda e: (-e.loc, -len(str(e.parser_element))))
4775 max_fatal = fatals[0]
4776 raise max_fatal
4778 if maxException is not None:
4779 # infer from this check that all alternatives failed at the current position
4780 # so emit this collective error message instead of any single error message
4781 parse_start_loc = self.preParse(instring, loc)
4782 if maxExcLoc == parse_start_loc:
4783 maxException.msg = self.errmsg or ""
4784 raise maxException
4786 raise ParseException(instring, loc, "no defined alternatives to match", self)
4788 def __ixor__(self, other):
4789 if isinstance(other, str_type):
4790 other = self._literalStringClass(other)
4791 if not isinstance(other, ParserElement):
4792 return NotImplemented
4793 return self.append(other) # Or([self, other])
4795 def _generateDefaultName(self) -> str:
4796 return f"{{{' ^ '.join(str(e) for e in self.exprs)}}}"
4798 def _setResultsName(self, name, list_all_matches=False) -> ParserElement:
4799 if (
4800 __diag__.warn_multiple_tokens_in_named_alternation
4801 and Diagnostics.warn_multiple_tokens_in_named_alternation
4802 not in self.suppress_warnings_
4803 ):
4804 if any(
4805 isinstance(e, And)
4806 and Diagnostics.warn_multiple_tokens_in_named_alternation
4807 not in e.suppress_warnings_
4808 for e in self.exprs
4809 ):
4810 warning = (
4811 "warn_multiple_tokens_in_named_alternation:"
4812 f" setting results name {name!r} on {type(self).__name__} expression"
4813 " will return a list of all parsed tokens in an And alternative,"
4814 " in prior versions only the first token was returned; enclose"
4815 " contained argument in Group"
4816 )
4817 warnings.warn(warning, PyparsingDiagnosticWarning, stacklevel=3)
4819 return super()._setResultsName(name, list_all_matches)
4822class MatchFirst(ParseExpression):
4823 """Requires that at least one :class:`ParserElement` is found. If
4824 more than one expression matches, the first one listed is the one that will
4825 match. May be constructed using the ``'|'`` operator.
4827 Example: Construct MatchFirst using '|' operator
4829 .. doctest::
4831 # watch the order of expressions to match
4832 >>> number = Word(nums) | Combine(Word(nums) + '.' + Word(nums))
4833 >>> print(number.search_string("123 3.1416 789")) # Fail!
4834 [['123'], ['3'], ['1416'], ['789']]
4836 # put more selective expression first
4837 >>> number = Combine(Word(nums) + '.' + Word(nums)) | Word(nums)
4838 >>> print(number.search_string("123 3.1416 789")) # Better
4839 [['123'], ['3.1416'], ['789']]
4840 """
4842 def __init__(
4843 self, exprs: typing.Iterable[ParserElement], savelist: bool = False
4844 ) -> None:
4845 super().__init__(exprs, savelist)
4846 if self.exprs:
4847 self._may_return_empty = any(e.mayReturnEmpty for e in self.exprs)
4848 self.skipWhitespace = all(e.skipWhitespace for e in self.exprs)
4849 else:
4850 self._may_return_empty = True
4852 def streamline(self) -> ParserElement:
4853 if self.streamlined:
4854 return self
4856 super().streamline()
4857 if self.exprs:
4858 self.saveAsList = any(e.saveAsList for e in self.exprs)
4859 self._may_return_empty = any(e.mayReturnEmpty for e in self.exprs)
4860 self.skipWhitespace = all(
4861 e.skipWhitespace and not isinstance(e, White) for e in self.exprs
4862 )
4863 else:
4864 self.saveAsList = False
4865 self._may_return_empty = True
4866 return self
4868 def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType:
4869 maxExcLoc = -1
4870 maxException = None
4872 for e in self.exprs:
4873 try:
4874 return e._parse(instring, loc, do_actions)
4875 except ParseFatalException as pfe:
4876 pfe.__traceback__ = None
4877 pfe.parser_element = e
4878 raise
4879 except ParseException as err:
4880 if err.loc > maxExcLoc:
4881 maxException = err
4882 maxExcLoc = err.loc
4883 except IndexError:
4884 if len(instring) > maxExcLoc:
4885 maxException = ParseException(
4886 instring, len(instring), e.errmsg, self
4887 )
4888 maxExcLoc = len(instring)
4890 if maxException is not None:
4891 # infer from this check that all alternatives failed at the current position
4892 # so emit this collective error message instead of any individual error message
4893 parse_start_loc = self.preParse(instring, loc)
4894 if maxExcLoc == parse_start_loc:
4895 maxException.msg = self.errmsg or ""
4896 raise maxException
4898 raise ParseException(instring, loc, "no defined alternatives to match", self)
4900 def __ior__(self, other):
4901 if isinstance(other, str_type):
4902 other = self._literalStringClass(other)
4903 if not isinstance(other, ParserElement):
4904 return NotImplemented
4905 return self.append(other) # MatchFirst([self, other])
4907 def _generateDefaultName(self) -> str:
4908 return f"{{{' | '.join(str(e) for e in self.exprs)}}}"
4910 def _setResultsName(self, name, list_all_matches=False) -> ParserElement:
4911 if (
4912 __diag__.warn_multiple_tokens_in_named_alternation
4913 and Diagnostics.warn_multiple_tokens_in_named_alternation
4914 not in self.suppress_warnings_
4915 ):
4916 if any(
4917 isinstance(e, And)
4918 and Diagnostics.warn_multiple_tokens_in_named_alternation
4919 not in e.suppress_warnings_
4920 for e in self.exprs
4921 ):
4922 warning = (
4923 "warn_multiple_tokens_in_named_alternation:"
4924 f" setting results name {name!r} on {type(self).__name__} expression"
4925 " will return a list of all parsed tokens in an And alternative,"
4926 " in prior versions only the first token was returned; enclose"
4927 " contained argument in Group"
4928 )
4929 warnings.warn(warning, PyparsingDiagnosticWarning, stacklevel=3)
4931 return super()._setResultsName(name, list_all_matches)
4934class Each(ParseExpression):
4935 """Requires all given :class:`ParserElement` s to be found, but in
4936 any order. Expressions may be separated by whitespace.
4938 May be constructed using the ``'&'`` operator.
4940 Example:
4942 .. testcode::
4944 color = one_of("RED ORANGE YELLOW GREEN BLUE PURPLE BLACK WHITE BROWN")
4945 shape_type = one_of("SQUARE CIRCLE TRIANGLE STAR HEXAGON OCTAGON")
4946 integer = Word(nums)
4947 shape_attr = "shape:" + shape_type("shape")
4948 posn_attr = "posn:" + Group(integer("x") + ',' + integer("y"))("posn")
4949 color_attr = "color:" + color("color")
4950 size_attr = "size:" + integer("size")
4952 # use Each (using operator '&') to accept attributes in any order
4953 # (shape and posn are required, color and size are optional)
4954 shape_spec = shape_attr & posn_attr & Opt(color_attr) & Opt(size_attr)
4956 shape_spec.run_tests('''
4957 shape: SQUARE color: BLACK posn: 100, 120
4958 shape: CIRCLE size: 50 color: BLUE posn: 50,80
4959 color:GREEN size:20 shape:TRIANGLE posn:20,40
4960 '''
4961 )
4963 prints:
4965 .. testoutput::
4966 :options: +NORMALIZE_WHITESPACE
4969 shape: SQUARE color: BLACK posn: 100, 120
4970 ['shape:', 'SQUARE', 'color:', 'BLACK', 'posn:', ['100', ',', '120']]
4971 - color: 'BLACK'
4972 - posn: ['100', ',', '120']
4973 - x: '100'
4974 - y: '120'
4975 - shape: 'SQUARE'
4976 ...
4978 shape: CIRCLE size: 50 color: BLUE posn: 50,80
4979 ['shape:', 'CIRCLE', 'size:', '50', 'color:', 'BLUE',
4980 'posn:', ['50', ',', '80']]
4981 - color: 'BLUE'
4982 - posn: ['50', ',', '80']
4983 - x: '50'
4984 - y: '80'
4985 - shape: 'CIRCLE'
4986 - size: '50'
4987 ...
4989 color:GREEN size:20 shape:TRIANGLE posn:20,40
4990 ['color:', 'GREEN', 'size:', '20', 'shape:', 'TRIANGLE',
4991 'posn:', ['20', ',', '40']]
4992 - color: 'GREEN'
4993 - posn: ['20', ',', '40']
4994 - x: '20'
4995 - y: '40'
4996 - shape: 'TRIANGLE'
4997 - size: '20'
4998 ...
4999 """
5001 def __init__(
5002 self, exprs: typing.Iterable[ParserElement], savelist: bool = True
5003 ) -> None:
5004 super().__init__(exprs, savelist)
5005 if self.exprs:
5006 self._may_return_empty = all(e.mayReturnEmpty for e in self.exprs)
5007 else:
5008 self._may_return_empty = True
5009 self.skipWhitespace = True
5010 self.initExprGroups = True
5011 self.saveAsList = True
5013 def __iand__(self, other):
5014 if isinstance(other, str_type):
5015 other = self._literalStringClass(other)
5016 if not isinstance(other, ParserElement):
5017 return NotImplemented
5018 return self.append(other) # Each([self, other])
5020 def streamline(self) -> ParserElement:
5021 super().streamline()
5022 if self.exprs:
5023 self._may_return_empty = all(e.mayReturnEmpty for e in self.exprs)
5024 else:
5025 self._may_return_empty = True
5026 return self
5028 def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType:
5029 if self.initExprGroups:
5030 self.opt1map = dict(
5031 (id(e.expr), e) for e in self.exprs if isinstance(e, Opt)
5032 )
5033 opt1 = [e.expr for e in self.exprs if isinstance(e, Opt)]
5034 opt2 = [
5035 e
5036 for e in self.exprs
5037 if e.mayReturnEmpty and not isinstance(e, (Opt, Regex, ZeroOrMore))
5038 ]
5039 self.optionals = opt1 + opt2
5040 self.multioptionals = [
5041 e.expr.set_results_name(e.resultsName, list_all_matches=True)
5042 for e in self.exprs
5043 if isinstance(e, _MultipleMatch)
5044 ]
5045 self.multirequired = [
5046 e.expr.set_results_name(e.resultsName, list_all_matches=True)
5047 for e in self.exprs
5048 if isinstance(e, OneOrMore)
5049 ]
5050 self.required = [
5051 e for e in self.exprs if not isinstance(e, (Opt, ZeroOrMore, OneOrMore))
5052 ]
5053 self.required += self.multirequired
5054 self.initExprGroups = False
5056 tmpLoc = loc
5057 tmpReqd = self.required[:]
5058 tmpOpt = self.optionals[:]
5059 multis = self.multioptionals[:]
5060 matchOrder: list[ParserElement] = []
5062 keepMatching = True
5063 failed: list[ParserElement] = []
5064 fatals: list[ParseFatalException] = []
5065 while keepMatching:
5066 tmpExprs = tmpReqd + tmpOpt + multis
5067 failed.clear()
5068 fatals.clear()
5069 for e in tmpExprs:
5070 try:
5071 tmpLoc = e.try_parse(instring, tmpLoc, raise_fatal=True)
5072 except ParseFatalException as pfe:
5073 pfe.__traceback__ = None
5074 pfe.parser_element = e
5075 fatals.append(pfe)
5076 failed.append(e)
5077 except ParseException:
5078 failed.append(e)
5079 else:
5080 matchOrder.append(self.opt1map.get(id(e), e))
5081 if e in tmpReqd:
5082 tmpReqd.remove(e)
5083 elif e in tmpOpt:
5084 tmpOpt.remove(e)
5085 if len(failed) == len(tmpExprs):
5086 keepMatching = False
5088 # look for any ParseFatalExceptions
5089 if fatals:
5090 if len(fatals) > 1:
5091 fatals.sort(key=lambda e: -e.loc)
5092 if fatals[0].loc == fatals[1].loc:
5093 fatals.sort(key=lambda e: (-e.loc, -len(str(e.parser_element))))
5094 max_fatal = fatals[0]
5095 raise max_fatal
5097 if tmpReqd:
5098 missing = ", ".join([str(e) for e in tmpReqd])
5099 raise ParseException(
5100 instring,
5101 loc,
5102 f"Missing one or more required elements ({missing})",
5103 )
5105 # add any unmatched Opts, in case they have default values defined
5106 matchOrder += [e for e in self.exprs if isinstance(e, Opt) and e.expr in tmpOpt]
5108 total_results = ParseResults([])
5109 for e in matchOrder:
5110 loc, results = e._parse(instring, loc, do_actions)
5111 total_results += results
5113 return loc, total_results
5115 def _generateDefaultName(self) -> str:
5116 return f"{{{' & '.join(str(e) for e in self.exprs)}}}"
5119class ParseElementEnhance(ParserElement):
5120 """Abstract subclass of :class:`ParserElement`, for combining and
5121 post-processing parsed tokens.
5122 """
5124 def __init__(self, expr: Union[ParserElement, str], savelist: bool = False) -> None:
5125 super().__init__(savelist)
5126 if isinstance(expr, str_type):
5127 expr_str = typing.cast(str, expr)
5128 if issubclass(self._literalStringClass, Token):
5129 expr = self._literalStringClass(expr_str) # type: ignore[call-arg]
5130 elif issubclass(type(self), self._literalStringClass):
5131 expr = Literal(expr_str)
5132 else:
5133 expr = self._literalStringClass(Literal(expr_str)) # type: ignore[assignment, call-arg]
5134 expr = typing.cast(ParserElement, expr)
5135 self.expr = expr
5136 if expr is not None:
5137 self.mayIndexError = expr.mayIndexError
5138 self._may_return_empty = expr.mayReturnEmpty
5139 self.set_whitespace_chars(
5140 expr.whiteChars, copy_defaults=expr.copyDefaultWhiteChars
5141 )
5142 self.skipWhitespace = expr.skipWhitespace
5143 self.saveAsList = expr.saveAsList
5144 self.callPreparse = expr.callPreparse
5145 self.ignoreExprs.extend(expr.ignoreExprs)
5147 def recurse(self) -> list[ParserElement]:
5148 return [self.expr] if self.expr is not None else []
5150 def parseImpl(self, instring, loc, do_actions=True):
5151 if self.expr is None:
5152 raise ParseException(instring, loc, "No expression defined", self)
5154 try:
5155 return self.expr._parse(instring, loc, do_actions, callPreParse=False)
5156 except ParseSyntaxException:
5157 raise
5158 except ParseBaseException as pbe:
5159 pbe.pstr = pbe.pstr or instring
5160 pbe.loc = pbe.loc or loc
5161 pbe.parser_element = pbe.parser_element or self
5162 if not isinstance(self, Forward) and self.customName is not None:
5163 if self.errmsg:
5164 pbe.msg = self.errmsg
5165 raise
5167 def leave_whitespace(self, recursive: bool = True) -> ParserElement:
5168 """
5169 Extends ``leave_whitespace`` defined in base class, and also invokes ``leave_whitespace`` on
5170 the contained expression.
5171 """
5172 super().leave_whitespace(recursive)
5174 if recursive:
5175 if self.expr is not None:
5176 self.expr = self.expr.copy()
5177 self.expr.leave_whitespace(recursive)
5178 return self
5180 def ignore_whitespace(self, recursive: bool = True) -> ParserElement:
5181 """
5182 Extends ``ignore_whitespace`` defined in base class, and also invokes ``ignore_whitespace`` on
5183 the contained expression.
5184 """
5185 super().ignore_whitespace(recursive)
5187 if recursive:
5188 if self.expr is not None:
5189 self.expr = self.expr.copy()
5190 self.expr.ignore_whitespace(recursive)
5191 return self
5193 def ignore(self, other) -> ParserElement:
5194 """
5195 Define expression to be ignored (e.g., comments) while doing pattern
5196 matching; may be called repeatedly, to define multiple comment or other
5197 ignorable patterns.
5198 """
5199 if not isinstance(other, Suppress) or other not in self.ignoreExprs:
5200 super().ignore(other)
5201 if self.expr is not None:
5202 self.expr.ignore(self.ignoreExprs[-1])
5204 return self
5206 def streamline(self) -> ParserElement:
5207 super().streamline()
5208 if self.expr is not None:
5209 self.expr.streamline()
5210 return self
5212 def _checkRecursion(self, parseElementList):
5213 if self in parseElementList:
5214 raise RecursiveGrammarException(parseElementList + [self])
5215 subRecCheckList = parseElementList[:] + [self]
5216 if self.expr is not None:
5217 self.expr._checkRecursion(subRecCheckList)
5219 def validate(self, validateTrace=None) -> None:
5220 warnings.warn(
5221 "ParserElement.validate() is deprecated, and should not be used to check for left recursion",
5222 PyparsingDeprecationWarning,
5223 stacklevel=2,
5224 )
5225 if validateTrace is None:
5226 validateTrace = []
5227 tmp = validateTrace[:] + [self]
5228 if self.expr is not None:
5229 self.expr.validate(tmp)
5230 self._checkRecursion([])
5232 def _generateDefaultName(self) -> str:
5233 return f"{type(self).__name__}:({self.expr})"
5235 # Compatibility synonyms
5236 # fmt: off
5237 leaveWhitespace = replaced_by_pep8("leaveWhitespace", leave_whitespace)
5238 ignoreWhitespace = replaced_by_pep8("ignoreWhitespace", ignore_whitespace)
5239 # fmt: on
5242class IndentedBlock(ParseElementEnhance):
5243 """
5244 Expression to match one or more expressions at a given indentation level.
5245 Useful for parsing text where structure is implied by indentation (like Python source code).
5247 Example:
5249 .. testcode::
5251 '''
5252 BNF:
5253 statement ::= assignment_stmt | if_stmt
5254 assignment_stmt ::= identifier '=' rvalue
5255 rvalue ::= identifier | integer
5256 if_stmt ::= 'if' bool_condition block
5257 block ::= ([indent] statement)...
5258 identifier ::= [A..Za..z]
5259 integer ::= [0..9]...
5260 bool_condition ::= 'TRUE' | 'FALSE'
5261 '''
5263 IF, TRUE, FALSE = Keyword.using_each("IF TRUE FALSE".split())
5265 statement = Forward()
5266 identifier = Char(alphas)
5267 integer = Word(nums).add_parse_action(lambda t: int(t[0]))
5268 rvalue = identifier | integer
5269 assignment_stmt = identifier + "=" + rvalue
5271 if_stmt = IF + (TRUE | FALSE) + IndentedBlock(statement)
5273 statement <<= Group(assignment_stmt | if_stmt)
5275 result = if_stmt.parse_string('''
5276 IF TRUE
5277 a = 1000
5278 b = 2000
5279 IF FALSE
5280 z = 100
5281 ''')
5282 print(result.dump())
5284 .. testoutput::
5286 ['IF', 'TRUE', [['a', '=', 1000], ['b', '=', 2000], ['IF', 'FALSE', [['z', '=', 100]]]]]
5287 [0]:
5288 IF
5289 [1]:
5290 TRUE
5291 [2]:
5292 [['a', '=', 1000], ['b', '=', 2000], ['IF', 'FALSE', [['z', '=', 100]]]]
5293 [0]:
5294 ['a', '=', 1000]
5295 [1]:
5296 ['b', '=', 2000]
5297 [2]:
5298 ['IF', 'FALSE', [['z', '=', 100]]]
5299 [0]:
5300 IF
5301 [1]:
5302 FALSE
5303 [2]:
5304 [['z', '=', 100]]
5305 [0]:
5306 ['z', '=', 100]
5307 """
5309 class _Indent(Empty):
5310 def __init__(self, ref_col: int) -> None:
5311 super().__init__()
5312 self.errmsg = f"expected indent at column {ref_col}"
5313 self.add_condition(lambda s, l, t: col(l, s) == ref_col)
5315 class _IndentGreater(Empty):
5316 def __init__(self, ref_col: int) -> None:
5317 super().__init__()
5318 self.errmsg = f"expected indent at column greater than {ref_col}"
5319 self.add_condition(lambda s, l, t: col(l, s) > ref_col)
5321 def __init__(
5322 self, expr: ParserElement, *, recursive: bool = False, grouped: bool = True
5323 ) -> None:
5324 super().__init__(expr, savelist=True)
5325 # if recursive:
5326 # raise NotImplementedError("IndentedBlock with recursive is not implemented")
5327 self._recursive = recursive
5328 self._grouped = grouped
5329 self.parent_anchor = 1
5331 def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType:
5332 # advance parse position to non-whitespace by using an Empty()
5333 # this should be the column to be used for all subsequent indented lines
5334 anchor_loc = Empty().preParse(instring, loc)
5336 # see if self.expr matches at the current location - if not it will raise an exception
5337 # and no further work is necessary
5338 self.expr.try_parse(instring, anchor_loc, do_actions=do_actions)
5340 indent_col = col(anchor_loc, instring)
5341 peer_detect_expr = self._Indent(indent_col)
5343 inner_expr = Empty() + peer_detect_expr + self.expr
5344 if self._recursive:
5345 sub_indent = self._IndentGreater(indent_col)
5346 nested_block = IndentedBlock(
5347 self.expr, recursive=self._recursive, grouped=self._grouped
5348 )
5349 nested_block.set_debug(self.debug)
5350 nested_block.parent_anchor = indent_col
5351 inner_expr += Opt(sub_indent + nested_block)
5353 inner_expr.set_name(f"inner {hex(id(inner_expr))[-4:].upper()}@{indent_col}")
5354 block = OneOrMore(inner_expr)
5356 trailing_undent = self._Indent(self.parent_anchor) | StringEnd()
5358 if self._grouped:
5359 wrapper = Group
5360 else:
5361 wrapper = lambda expr: expr # type: ignore[misc, assignment]
5362 return (wrapper(block) + Optional(trailing_undent)).parseImpl(
5363 instring, anchor_loc, do_actions
5364 )
5367class AtStringStart(ParseElementEnhance):
5368 """Matches if expression matches at the beginning of the parse
5369 string::
5371 AtStringStart(Word(nums)).parse_string("123")
5372 # prints ["123"]
5374 AtStringStart(Word(nums)).parse_string(" 123")
5375 # raises ParseException
5376 """
5378 def __init__(self, expr: Union[ParserElement, str]) -> None:
5379 super().__init__(expr)
5380 self.callPreparse = False
5382 def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType:
5383 if loc != 0:
5384 raise ParseException(instring, loc, "not found at string start")
5385 return super().parseImpl(instring, loc, do_actions)
5388class AtLineStart(ParseElementEnhance):
5389 r"""Matches if an expression matches at the beginning of a line within
5390 the parse string
5392 Example:
5394 .. testcode::
5396 test = '''\
5397 BBB this line
5398 BBB and this line
5399 BBB but not this one
5400 A BBB and definitely not this one
5401 '''
5403 for t in (AtLineStart('BBB') + rest_of_line).search_string(test):
5404 print(t)
5406 prints:
5408 .. testoutput::
5410 ['BBB', ' this line']
5411 ['BBB', ' and this line']
5412 """
5414 def __init__(self, expr: Union[ParserElement, str]) -> None:
5415 super().__init__(expr)
5416 self.callPreparse = False
5418 def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType:
5419 if col(loc, instring) != 1:
5420 raise ParseException(instring, loc, "not found at line start")
5421 return super().parseImpl(instring, loc, do_actions)
5424class FollowedBy(ParseElementEnhance):
5425 """Lookahead matching of the given parse expression.
5426 ``FollowedBy`` does *not* advance the parsing position within
5427 the input string, it only verifies that the specified parse
5428 expression matches at the current position. ``FollowedBy``
5429 always returns a null token list. If any results names are defined
5430 in the lookahead expression, those *will* be returned for access by
5431 name.
5433 Example:
5435 .. testcode::
5437 # use FollowedBy to match a label only if it is followed by a ':'
5438 data_word = Word(alphas)
5439 label = data_word + FollowedBy(':')
5440 attr_expr = Group(
5441 label + Suppress(':')
5442 + OneOrMore(data_word, stop_on=label).set_parse_action(' '.join)
5443 )
5445 attr_expr[1, ...].parse_string(
5446 "shape: SQUARE color: BLACK posn: upper left").pprint()
5448 prints:
5450 .. testoutput::
5452 [['shape', 'SQUARE'], ['color', 'BLACK'], ['posn', 'upper left']]
5453 """
5455 def __init__(self, expr: Union[ParserElement, str]) -> None:
5456 super().__init__(expr)
5457 self._may_return_empty = True
5459 def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType:
5460 # by using self._expr.parse and deleting the contents of the returned ParseResults list
5461 # we keep any named results that were defined in the FollowedBy expression
5462 _, ret = self.expr._parse(instring, loc, do_actions=do_actions)
5463 del ret[:]
5465 return loc, ret
5468class PrecededBy(ParseElementEnhance):
5469 """Lookbehind matching of the given parse expression.
5470 ``PrecededBy`` does not advance the parsing position within the
5471 input string, it only verifies that the specified parse expression
5472 matches prior to the current position. ``PrecededBy`` always
5473 returns a null token list, but if a results name is defined on the
5474 given expression, it is returned.
5476 Parameters:
5478 - ``expr`` - expression that must match prior to the current parse
5479 location
5480 - ``retreat`` - (default= ``None``) - (int) maximum number of characters
5481 to lookbehind prior to the current parse location
5483 If the lookbehind expression is a string, :class:`Literal`,
5484 :class:`Keyword`, or a :class:`Word` or :class:`CharsNotIn`
5485 with a specified exact or maximum length, then the retreat
5486 parameter is not required. Otherwise, retreat must be specified to
5487 give a maximum number of characters to look back from
5488 the current parse position for a lookbehind match.
5490 Example:
5492 .. testcode::
5494 # VB-style variable names with type prefixes
5495 int_var = PrecededBy("#") + pyparsing_common.identifier
5496 str_var = PrecededBy("$") + pyparsing_common.identifier
5497 """
5499 def __init__(self, expr: Union[ParserElement, str], retreat: int = 0) -> None:
5500 super().__init__(expr)
5501 self.expr = self.expr().leave_whitespace()
5502 self._may_return_empty = True
5503 self.mayIndexError = False
5504 self.exact = False
5505 if isinstance(expr, str_type):
5506 expr = typing.cast(str, expr)
5507 retreat = len(expr)
5508 self.exact = True
5509 elif isinstance(expr, (Literal, Keyword)):
5510 retreat = expr.matchLen
5511 self.exact = True
5512 elif isinstance(expr, (Word, CharsNotIn)) and expr.maxLen != _MAX_INT:
5513 retreat = expr.maxLen
5514 self.exact = True
5515 elif isinstance(expr, PositionToken):
5516 retreat = 0
5517 self.exact = True
5518 self.retreat = retreat
5519 self.errmsg = f"not preceded by {expr}"
5520 self.skipWhitespace = False
5521 self.parseAction.append(lambda s, l, t: t.__delitem__(slice(None, None)))
5523 def parseImpl(self, instring, loc=0, do_actions=True) -> ParseImplReturnType:
5524 if self.exact:
5525 if loc < self.retreat:
5526 raise ParseException(instring, loc, self.errmsg, self)
5527 start = loc - self.retreat
5528 _, ret = self.expr._parse(instring, start)
5529 return loc, ret
5531 # retreat specified a maximum lookbehind window, iterate
5532 test_expr = self.expr + StringEnd()
5533 instring_slice = instring[max(0, loc - self.retreat) : loc]
5534 last_expr: ParseBaseException = ParseException(instring, loc, self.errmsg, self)
5536 for offset in range(1, min(loc, self.retreat + 1) + 1):
5537 try:
5538 # print('trying', offset, instring_slice, repr(instring_slice[loc - offset:]))
5539 _, ret = test_expr._parse(instring_slice, len(instring_slice) - offset)
5540 except ParseBaseException as pbe:
5541 last_expr = pbe
5542 else:
5543 break
5544 else:
5545 raise last_expr
5547 return loc, ret
5550class Located(ParseElementEnhance):
5551 """
5552 Decorates a returned token with its starting and ending
5553 locations in the input string.
5555 This helper adds the following results names:
5557 - ``locn_start`` - location where matched expression begins
5558 - ``locn_end`` - location where matched expression ends
5559 - ``value`` - the actual parsed results
5561 Be careful if the input text contains ``<TAB>`` characters, you
5562 may want to call :class:`ParserElement.parse_with_tabs`
5564 Example:
5566 .. testcode::
5568 wd = Word(alphas)
5569 for match in Located(wd).search_string("ljsdf123lksdjjf123lkkjj1222"):
5570 print(match)
5572 prints:
5574 .. testoutput::
5576 [0, ['ljsdf'], 5]
5577 [8, ['lksdjjf'], 15]
5578 [18, ['lkkjj'], 23]
5579 """
5581 def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType:
5582 start = loc
5583 loc, tokens = self.expr._parse(instring, start, do_actions, callPreParse=False)
5584 ret_tokens = ParseResults([start, tokens, loc])
5585 ret_tokens["locn_start"] = start
5586 ret_tokens["value"] = tokens
5587 ret_tokens["locn_end"] = loc
5588 if self.resultsName:
5589 # must return as a list, so that the name will be attached to the complete group
5590 return loc, [ret_tokens]
5591 else:
5592 return loc, ret_tokens
5595class NotAny(ParseElementEnhance):
5596 """
5597 Lookahead to disallow matching with the given parse expression.
5598 ``NotAny`` does *not* advance the parsing position within the
5599 input string, it only verifies that the specified parse expression
5600 does *not* match at the current position. Also, ``NotAny`` does
5601 *not* skip over leading whitespace. ``NotAny`` always returns
5602 a null token list. May be constructed using the ``'~'`` operator.
5604 Example:
5606 .. testcode::
5608 AND, OR, NOT = map(CaselessKeyword, "AND OR NOT".split())
5610 # take care not to mistake keywords for identifiers
5611 ident = ~(AND | OR | NOT) + Word(alphas)
5612 boolean_term = Opt(NOT) + ident
5614 # very crude boolean expression - to support parenthesis groups and
5615 # operation hierarchy, use infix_notation
5616 boolean_expr = boolean_term + ((AND | OR) + boolean_term)[...]
5618 # integers that are followed by "." are actually floats
5619 integer = Word(nums) + ~Char(".")
5620 """
5622 def __init__(self, expr: Union[ParserElement, str]) -> None:
5623 super().__init__(expr)
5624 # do NOT use self.leave_whitespace(), don't want to propagate to exprs
5625 # self.leave_whitespace()
5626 self.skipWhitespace = False
5628 self._may_return_empty = True
5629 self.errmsg = f"Found unwanted token, {self.expr}"
5631 def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType:
5632 if self.expr.can_parse_next(instring, loc, do_actions=do_actions):
5633 raise ParseException(instring, loc, self.errmsg, self)
5634 return loc, []
5636 def _generateDefaultName(self) -> str:
5637 return f"~{{{self.expr}}}"
5640class _MultipleMatch(ParseElementEnhance):
5641 def __init__(
5642 self,
5643 expr: Union[str, ParserElement],
5644 stop_on: typing.Optional[Union[ParserElement, str]] = None,
5645 max: typing.Optional[int] = None,
5646 **kwargs,
5647 ) -> None:
5648 stopOn: typing.Optional[Union[ParserElement, str]] = deprecate_argument(
5649 kwargs, "stopOn", None
5650 )
5652 super().__init__(expr)
5653 if max is not None and max <= 0:
5654 raise ValueError("max must be greater than 0")
5655 stopOn = stopOn or stop_on
5656 self.saveAsList = True
5657 self.max_count = max
5658 ender = stopOn
5659 if isinstance(ender, str_type):
5660 ender = self._literalStringClass(ender)
5661 self.stopOn(ender)
5663 def stop_on(self, ender) -> ParserElement:
5664 if isinstance(ender, str_type):
5665 ender = self._literalStringClass(ender)
5666 self.not_ender = ~ender if ender is not None else None
5667 return self
5669 stopOn = stop_on
5671 def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType:
5672 self_expr_parse = self.expr._parse
5673 self_skip_ignorables = self._skipIgnorables
5674 check_ender = False
5675 if self.not_ender is not None:
5676 try_not_ender = self.not_ender.try_parse
5677 check_ender = True
5679 # must be at least one (but first see if we are the stopOn sentinel;
5680 # if so, fail)
5681 if check_ender:
5682 try_not_ender(instring, loc)
5683 loc, tokens = self_expr_parse(instring, loc, do_actions)
5684 match_count = 1
5685 try:
5686 hasIgnoreExprs = not not self.ignoreExprs
5687 while self.max_count is None or match_count < self.max_count:
5688 if check_ender:
5689 try_not_ender(instring, loc)
5690 if hasIgnoreExprs:
5691 preloc = self_skip_ignorables(instring, loc)
5692 else:
5693 preloc = loc
5694 loc, tmptokens = self_expr_parse(instring, preloc, do_actions)
5695 tokens += tmptokens
5696 match_count += 1
5697 except (ParseException, IndexError):
5698 pass
5700 return loc, tokens
5702 def _setResultsName(self, name, list_all_matches=False) -> ParserElement:
5703 if (
5704 __diag__.warn_ungrouped_named_tokens_in_collection
5705 and Diagnostics.warn_ungrouped_named_tokens_in_collection
5706 not in self.suppress_warnings_
5707 ):
5708 for e in [self.expr] + self.expr.recurse():
5709 if (
5710 isinstance(e, ParserElement)
5711 and e.resultsName
5712 and (
5713 Diagnostics.warn_ungrouped_named_tokens_in_collection
5714 not in e.suppress_warnings_
5715 )
5716 ):
5717 warning = (
5718 "warn_ungrouped_named_tokens_in_collection:"
5719 f" setting results name {name!r} on {type(self).__name__} expression"
5720 f" collides with {e.resultsName!r} on contained expression"
5721 )
5722 warnings.warn(warning, PyparsingDiagnosticWarning, stacklevel=3)
5723 break
5725 return super()._setResultsName(name, list_all_matches)
5728class OneOrMore(_MultipleMatch):
5729 """
5730 Repetition of one or more of the given expression.
5732 Parameters:
5734 - ``expr`` - expression that must match one or more times
5735 - ``stop_on`` - (default= ``None``) - expression for a terminating sentinel
5736 (only required if the sentinel would ordinarily match the repetition
5737 expression)
5738 - ``max`` - (default= ``None``) - maximum number times to match ``expr``
5739 (does not raise an exception if the expression occurs more than ``max``
5740 times, simply limits the number of matches to make)
5742 Example:
5744 .. doctest::
5746 >>> data_word = Word(alphas)
5747 >>> label = data_word + FollowedBy(':')
5748 >>> attr_expr = Group(
5749 ... label + Suppress(':')
5750 ... + OneOrMore(data_word).set_parse_action(' '.join))
5752 >>> text = "shape: SQUARE posn: upper left color: BLACK"
5754 # Fail! read 'posn' as data instead of next label
5755 >>> attr_expr[1, ...].parse_string(text).pprint()
5756 [['shape', 'SQUARE posn']]
5758 # use stop_on attribute for OneOrMore
5759 # to avoid reading label string as part of the data
5760 >>> attr_expr = Group(
5761 ... label + Suppress(':')
5762 ... + OneOrMore(
5763 ... data_word, stop_on=label).set_parse_action(' '.join))
5764 >>> OneOrMore(attr_expr).parse_string(text).pprint() # Better
5765 [['shape', 'SQUARE'], ['posn', 'upper left'], ['color', 'BLACK']]
5767 # could also be written as
5768 >>> (attr_expr * (1,)).parse_string(text).pprint()
5769 [['shape', 'SQUARE'], ['posn', 'upper left'], ['color', 'BLACK']]
5770 """
5772 def _generateDefaultName(self) -> str:
5773 return f"{{{self.expr}}}..."
5776class ZeroOrMore(_MultipleMatch):
5777 """
5778 Optional repetition of zero or more of the given expression.
5780 Parameters:
5782 - ``expr`` - expression that must match zero or more times
5783 - ``stop_on`` - expression for a terminating sentinel
5784 (only required if the sentinel would ordinarily match the repetition
5785 expression) - (default= ``None``)
5787 Example: similar to :class:`OneOrMore`
5788 """
5790 def __init__(
5791 self,
5792 expr: Union[str, ParserElement],
5793 stop_on: typing.Optional[Union[ParserElement, str]] = None,
5794 max: typing.Optional[int] = None,
5795 **kwargs,
5796 ) -> None:
5797 stopOn: Union[ParserElement, str] = deprecate_argument(kwargs, "stopOn", None)
5799 super().__init__(expr, stop_on=stopOn or stop_on, max=max)
5800 self._may_return_empty = True
5802 def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType:
5803 try:
5804 return super().parseImpl(instring, loc, do_actions)
5805 except (ParseException, IndexError):
5806 return loc, ParseResults([], name=self.resultsName)
5808 def _generateDefaultName(self) -> str:
5809 return f"[{self.expr}]..."
5812class DelimitedList(ParseElementEnhance):
5813 """Helper to define a delimited list of expressions - the delimiter
5814 defaults to ','. By default, the list elements and delimiters can
5815 have intervening whitespace, and comments, but this can be
5816 overridden by passing ``combine=True`` in the constructor. If
5817 ``combine`` is set to ``True``, the matching tokens are
5818 returned as a single token string, with the delimiters included;
5819 otherwise, the matching tokens are returned as a list of tokens,
5820 with the delimiters suppressed.
5822 If ``allow_trailing_delim`` is set to True, then the list may end with
5823 a delimiter.
5825 Example:
5827 .. doctest::
5829 >>> DelimitedList(Word(alphas)).parse_string("aa,bb,cc")
5830 ParseResults(['aa', 'bb', 'cc'], {})
5831 >>> DelimitedList(Word(hexnums), delim=':', combine=True
5832 ... ).parse_string("AA:BB:CC:DD:EE")
5833 ParseResults(['AA:BB:CC:DD:EE'], {})
5835 .. versionadded:: 3.1.0
5836 """
5838 def __init__(
5839 self,
5840 expr: Union[str, ParserElement],
5841 delim: Union[str, ParserElement] = ",",
5842 combine: bool = False,
5843 min: typing.Optional[int] = None,
5844 max: typing.Optional[int] = None,
5845 *,
5846 allow_trailing_delim: bool = False,
5847 ) -> None:
5848 if isinstance(expr, str_type):
5849 expr = ParserElement._literalStringClass(expr)
5850 expr = typing.cast(ParserElement, expr)
5852 if min is not None and min < 1:
5853 raise ValueError("min must be greater than 0")
5855 if max is not None and min is not None and max < min:
5856 raise ValueError("max must be greater than, or equal to min")
5858 self.content = expr
5859 self.raw_delim = str(delim)
5860 self.delim = delim
5861 self.combine = combine
5862 if not combine:
5863 self.delim = Suppress(delim) if not isinstance(delim, Suppress) else delim
5864 self.min = min or 1
5865 self.max = max
5866 self.allow_trailing_delim = allow_trailing_delim
5868 delim_list_expr = self.content + (self.delim + self.content) * (
5869 self.min - 1,
5870 None if self.max is None else self.max - 1,
5871 )
5872 if self.allow_trailing_delim:
5873 delim_list_expr += Opt(self.delim)
5875 if self.combine:
5876 delim_list_expr = Combine(delim_list_expr)
5878 super().__init__(delim_list_expr, savelist=True)
5880 def _generateDefaultName(self) -> str:
5881 content_expr = self.content.streamline()
5882 return f"{content_expr} [{self.raw_delim} {content_expr}]..."
5885class _NullToken:
5886 def __bool__(self):
5887 return False
5889 def __str__(self):
5890 return ""
5893class Opt(ParseElementEnhance):
5894 """
5895 Optional matching of the given expression.
5897 :param expr: expression that must match zero or more times
5898 :param default: (optional) - value to be returned
5899 if the optional expression is not found.
5901 Example:
5903 .. testcode::
5905 # US postal code can be a 5-digit zip, plus optional 4-digit qualifier
5906 zip = Combine(Word(nums, exact=5) + Opt('-' + Word(nums, exact=4)))
5907 zip.run_tests('''
5908 # traditional ZIP code
5909 12345
5911 # ZIP+4 form
5912 12101-0001
5914 # invalid ZIP
5915 98765-
5916 ''')
5918 prints:
5920 .. testoutput::
5921 :options: +NORMALIZE_WHITESPACE
5924 # traditional ZIP code
5925 12345
5926 ['12345']
5928 # ZIP+4 form
5929 12101-0001
5930 ['12101-0001']
5932 # invalid ZIP
5933 98765-
5934 98765-
5935 ^
5936 ParseException: Expected end of text, found '-' (at char 5), (line:1, col:6)
5937 FAIL: Expected end of text, found '-' (at char 5), (line:1, col:6)
5938 """
5940 __optionalNotMatched = _NullToken()
5942 def __init__(
5943 self, expr: Union[ParserElement, str], default: Any = __optionalNotMatched
5944 ) -> None:
5945 super().__init__(expr, savelist=False)
5946 self.saveAsList = self.expr.saveAsList
5947 self.defaultValue = default
5948 self._may_return_empty = True
5950 def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType:
5951 self_expr = self.expr
5952 try:
5953 loc, tokens = self_expr._parse(
5954 instring, loc, do_actions, callPreParse=False
5955 )
5956 except (ParseException, IndexError):
5957 default_value = self.defaultValue
5958 if default_value is not self.__optionalNotMatched:
5959 if self_expr.resultsName:
5960 tokens = ParseResults([default_value])
5961 tokens[self_expr.resultsName] = default_value
5962 else:
5963 tokens = [default_value] # type: ignore[assignment]
5964 else:
5965 tokens = [] # type: ignore[assignment]
5966 return loc, tokens
5968 def _generateDefaultName(self) -> str:
5969 inner = str(self.expr)
5970 # strip off redundant inner {}'s
5971 while len(inner) > 1 and inner[0 :: len(inner) - 1] == "{}":
5972 inner = inner[1:-1]
5973 return f"[{inner}]"
5976Optional = Opt
5979class SkipTo(ParseElementEnhance):
5980 """
5981 Token for skipping over all undefined text until the matched
5982 expression is found.
5984 :param expr: target expression marking the end of the data to be skipped
5985 :param include: if ``True``, the target expression is also parsed
5986 (the skipped text and target expression are returned
5987 as a 2-element list) (default= ``False``).
5989 :param ignore: (default= ``None``) used to define grammars
5990 (typically quoted strings and comments)
5991 that might contain false matches to the target expression
5993 :param fail_on: (default= ``None``) define expressions that
5994 are not allowed to be included in the skipped test;
5995 if found before the target expression is found,
5996 the :class:`SkipTo` is not a match
5998 Example:
6000 .. testcode::
6002 report = '''
6003 Outstanding Issues Report - 1 Jan 2000
6005 # | Severity | Description | Days Open
6006 -----+----------+-------------------------------------------+-----------
6007 101 | Critical | Intermittent system crash | 6
6008 94 | Cosmetic | Spelling error on Login ('log|n') | 14
6009 79 | Minor | System slow when running too many reports | 47
6010 '''
6011 integer = Word(nums)
6012 SEP = Suppress('|')
6013 # use SkipTo to simply match everything up until the next SEP
6014 # - ignore quoted strings, so that a '|' character inside a quoted string does not match
6015 # - parse action will call token.strip() for each matched token, i.e., the description body
6016 string_data = SkipTo(SEP, ignore=quoted_string)
6017 string_data.set_parse_action(token_map(str.strip))
6018 ticket_expr = (integer("issue_num") + SEP
6019 + string_data("sev") + SEP
6020 + string_data("desc") + SEP
6021 + integer("days_open"))
6023 for tkt in ticket_expr.search_string(report):
6024 print(tkt.dump())
6026 prints:
6028 .. testoutput::
6030 ['101', 'Critical', 'Intermittent system crash', '6']
6031 - days_open: '6'
6032 - desc: 'Intermittent system crash'
6033 - issue_num: '101'
6034 - sev: 'Critical'
6035 ['94', 'Cosmetic', "Spelling error on Login ('log|n')", '14']
6036 - days_open: '14'
6037 - desc: "Spelling error on Login ('log|n')"
6038 - issue_num: '94'
6039 - sev: 'Cosmetic'
6040 ['79', 'Minor', 'System slow when running too many reports', '47']
6041 - days_open: '47'
6042 - desc: 'System slow when running too many reports'
6043 - issue_num: '79'
6044 - sev: 'Minor'
6045 """
6047 def __init__(
6048 self,
6049 other: Union[ParserElement, str],
6050 include: bool = False,
6051 ignore: typing.Optional[Union[ParserElement, str]] = None,
6052 fail_on: typing.Optional[Union[ParserElement, str]] = None,
6053 **kwargs,
6054 ) -> None:
6055 failOn: typing.Optional[Union[ParserElement, str]] = deprecate_argument(
6056 kwargs, "failOn", None
6057 )
6059 super().__init__(other)
6060 failOn = failOn or fail_on
6061 self.ignoreExpr = ignore
6062 self._may_return_empty = True
6063 self.mayIndexError = False
6064 self.includeMatch = include
6065 self.saveAsList = False
6066 if isinstance(failOn, str_type):
6067 self.failOn = self._literalStringClass(failOn)
6068 else:
6069 self.failOn = failOn
6070 self.errmsg = f"No match found for {self.expr}"
6071 self.ignorer = Empty().leave_whitespace()
6072 self._update_ignorer()
6074 def _update_ignorer(self):
6075 # rebuild internal ignore expr from current ignore exprs and assigned ignoreExpr
6076 self.ignorer.ignoreExprs.clear()
6077 for e in self.expr.ignoreExprs:
6078 self.ignorer.ignore(e)
6079 if self.ignoreExpr:
6080 self.ignorer.ignore(self.ignoreExpr)
6082 def ignore(self, expr):
6083 """
6084 Define expression to be ignored (e.g., comments) while doing pattern
6085 matching; may be called repeatedly, to define multiple comment or other
6086 ignorable patterns.
6087 """
6088 super().ignore(expr)
6089 self._update_ignorer()
6091 def parseImpl(self, instring, loc, do_actions=True):
6092 startloc = loc
6093 instrlen = len(instring)
6094 self_expr_parse = self.expr._parse
6095 self_failOn_canParseNext = (
6096 self.failOn.can_parse_next if self.failOn is not None else None
6097 )
6098 ignorer_try_parse = self.ignorer.try_parse if self.ignorer.ignoreExprs else None
6100 tmploc = loc
6101 while tmploc <= instrlen:
6102 if self_failOn_canParseNext is not None:
6103 # break if failOn expression matches
6104 if self_failOn_canParseNext(instring, tmploc):
6105 break
6107 if ignorer_try_parse is not None:
6108 # advance past ignore expressions
6109 prev_tmploc = tmploc
6110 while 1:
6111 try:
6112 tmploc = ignorer_try_parse(instring, tmploc)
6113 except ParseBaseException:
6114 break
6115 # see if all ignorers matched, but didn't actually ignore anything
6116 if tmploc == prev_tmploc:
6117 break
6118 prev_tmploc = tmploc
6120 try:
6121 self_expr_parse(instring, tmploc, do_actions=False, callPreParse=False)
6122 except (ParseException, IndexError):
6123 # no match, advance loc in string
6124 tmploc += 1
6125 else:
6126 # matched skipto expr, done
6127 break
6129 else:
6130 # ran off the end of the input string without matching skipto expr, fail
6131 raise ParseException(instring, loc, self.errmsg, self)
6133 # build up return values
6134 loc = tmploc
6135 skiptext = instring[startloc:loc]
6136 skipresult = ParseResults(skiptext)
6138 if self.includeMatch:
6139 loc, mat = self_expr_parse(instring, loc, do_actions, callPreParse=False)
6140 skipresult += mat
6142 return loc, skipresult
6145class Forward(ParseElementEnhance):
6146 """
6147 Forward declaration of an expression to be defined later -
6148 used for recursive grammars, such as algebraic infix notation.
6149 When the expression is known, it is assigned to the ``Forward``
6150 instance using the ``'<<'`` operator.
6152 .. Note::
6154 Take care when assigning to ``Forward`` not to overlook
6155 precedence of operators.
6157 Specifically, ``'|'`` has a lower precedence than ``'<<'``, so that::
6159 fwd_expr << a | b | c
6161 will actually be evaluated as::
6163 (fwd_expr << a) | b | c
6165 thereby leaving b and c out as parseable alternatives.
6166 It is recommended that you explicitly group the values
6167 inserted into the :class:`Forward`::
6169 fwd_expr << (a | b | c)
6171 Converting to use the ``'<<='`` operator instead will avoid this problem.
6173 See :meth:`ParseResults.pprint` for an example of a recursive
6174 parser created using :class:`Forward`.
6175 """
6177 def __init__(
6178 self, other: typing.Optional[Union[ParserElement, str]] = None
6179 ) -> None:
6180 self.caller_frame = traceback.extract_stack(limit=2)[0]
6181 super().__init__(other, savelist=False) # type: ignore[arg-type]
6182 self.lshift_line = None
6184 def __lshift__(self, other) -> Forward:
6185 """
6186 ``'<<'`` operator overload to "inject" an expression into a :class:`Forward`.
6187 """
6188 if hasattr(self, "caller_frame"):
6189 del self.caller_frame
6190 if isinstance(other, str_type):
6191 other = self._literalStringClass(other)
6193 if not isinstance(other, ParserElement):
6194 return NotImplemented
6196 self.expr = other
6197 self.streamlined = other.streamlined
6198 self.mayIndexError = self.expr.mayIndexError
6199 self._may_return_empty = self.expr.mayReturnEmpty
6200 self.set_whitespace_chars(
6201 self.expr.whiteChars, copy_defaults=self.expr.copyDefaultWhiteChars
6202 )
6203 self.skipWhitespace = self.expr.skipWhitespace
6204 self.saveAsList = self.expr.saveAsList
6205 self.ignoreExprs.extend(self.expr.ignoreExprs)
6206 self.lshift_line = traceback.extract_stack(limit=2)[-2] # type: ignore[assignment]
6207 return self
6209 def __ilshift__(self, other) -> Forward:
6210 """
6211 ``'<<='`` operator overload to "inject" an expression into a :class:`Forward`.
6212 """
6213 if not isinstance(other, ParserElement):
6214 return NotImplemented
6216 return self << other
6218 def __or__(self, other) -> ParserElement:
6219 caller_line = traceback.extract_stack(limit=2)[-2]
6220 if (
6221 __diag__.warn_on_match_first_with_lshift_operator
6222 and caller_line == self.lshift_line
6223 and Diagnostics.warn_on_match_first_with_lshift_operator
6224 not in self.suppress_warnings_
6225 ):
6226 warnings.warn(
6227 "warn_on_match_first_with_lshift_operator:"
6228 " using '<<' operator with '|' is probably an error, use '<<='",
6229 PyparsingDiagnosticWarning,
6230 stacklevel=2,
6231 )
6232 ret = super().__or__(other)
6233 return ret
6235 def __del__(self):
6236 # see if we are getting dropped because of '=' reassignment of var instead of '<<=' or '<<'
6237 if (
6238 self.expr is None
6239 and __diag__.warn_on_assignment_to_Forward
6240 and Diagnostics.warn_on_assignment_to_Forward not in self.suppress_warnings_
6241 ):
6242 warnings.warn_explicit(
6243 "warn_on_assignment_to_Forward:"
6244 " Forward defined here but no expression attached later using '<<=' or '<<'",
6245 UserWarning,
6246 filename=self.caller_frame.filename,
6247 lineno=self.caller_frame.lineno,
6248 )
6250 def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType:
6251 if (
6252 self.expr is None
6253 and __diag__.warn_on_parse_using_empty_Forward
6254 and Diagnostics.warn_on_parse_using_empty_Forward
6255 not in self.suppress_warnings_
6256 ):
6257 # walk stack until parse_string, scan_string, search_string, or transform_string is found
6258 parse_fns = (
6259 "parse_string",
6260 "scan_string",
6261 "search_string",
6262 "transform_string",
6263 )
6264 tb = traceback.extract_stack(limit=200)
6265 for i, frm in enumerate(reversed(tb), start=1):
6266 if frm.name in parse_fns:
6267 stacklevel = i + 1
6268 break
6269 else:
6270 stacklevel = 2
6271 warnings.warn(
6272 "warn_on_parse_using_empty_Forward:"
6273 " Forward expression was never assigned a value, will not parse any input",
6274 PyparsingDiagnosticWarning,
6275 stacklevel=stacklevel,
6276 )
6277 if not ParserElement._left_recursion_enabled:
6278 return super().parseImpl(instring, loc, do_actions)
6280 # ## Bounded Recursion algorithm ##
6281 # Recursion only needs to be processed at ``Forward`` elements, since they are
6282 # the only ones that can actually refer to themselves. The general idea is
6283 # to handle recursion stepwise: We start at no recursion, then recurse once,
6284 # recurse twice, ..., until more recursion offers no benefit (we hit the bound).
6285 #
6286 # The "trick" here is that each ``Forward`` gets evaluated in two contexts
6287 # - to *match* a specific recursion level, and
6288 # - to *search* the bounded recursion level
6289 # and the two run concurrently. The *search* must *match* each recursion level
6290 # to find the best possible match. This is handled by a memo table, which
6291 # provides the previous match to the next level match attempt.
6292 #
6293 # See also "Left Recursion in Parsing Expression Grammars", Medeiros et al.
6294 #
6295 # There is a complication since we not only *parse* but also *transform* via
6296 # actions: We do not want to run the actions too often while expanding. Thus,
6297 # we expand using `do_actions=False` and only run `do_actions=True` if the next
6298 # recursion level is acceptable.
6299 with ParserElement.recursion_lock:
6300 memo = ParserElement.recursion_memos
6301 try:
6302 # we are parsing at a specific recursion expansion - use it as-is
6303 prev_loc, prev_result = memo[loc, self, do_actions]
6304 if isinstance(prev_result, Exception):
6305 raise prev_result
6306 return prev_loc, prev_result.copy()
6307 except KeyError:
6308 act_key = (loc, self, True)
6309 peek_key = (loc, self, False)
6310 # we are searching for the best recursion expansion - keep on improving
6311 # both `do_actions` cases must be tracked separately here!
6312 prev_loc, prev_peek = memo[peek_key] = (
6313 loc - 1,
6314 ParseException(
6315 instring, loc, "Forward recursion without base case", self
6316 ),
6317 )
6318 if do_actions:
6319 memo[act_key] = memo[peek_key]
6320 while True:
6321 try:
6322 new_loc, new_peek = super().parseImpl(instring, loc, False)
6323 except ParseException:
6324 # we failed before getting any match - do not hide the error
6325 if isinstance(prev_peek, Exception):
6326 raise
6327 new_loc, new_peek = prev_loc, prev_peek
6328 # the match did not get better: we are done
6329 if new_loc <= prev_loc:
6330 if do_actions:
6331 # replace the match for do_actions=False as well,
6332 # in case the action did backtrack
6333 prev_loc, prev_result = memo[peek_key] = memo[act_key]
6334 del memo[peek_key], memo[act_key]
6335 return prev_loc, copy.copy(prev_result)
6336 del memo[peek_key]
6337 return prev_loc, copy.copy(prev_peek)
6338 # the match did get better: see if we can improve further
6339 if do_actions:
6340 try:
6341 memo[act_key] = super().parseImpl(instring, loc, True)
6342 except ParseException as e:
6343 memo[peek_key] = memo[act_key] = (new_loc, e)
6344 raise
6345 prev_loc, prev_peek = memo[peek_key] = new_loc, new_peek
6347 def leave_whitespace(self, recursive: bool = True) -> ParserElement:
6348 """
6349 Extends ``leave_whitespace`` defined in base class.
6350 """
6351 self.skipWhitespace = False
6352 return self
6354 def ignore_whitespace(self, recursive: bool = True) -> ParserElement:
6355 """
6356 Extends ``ignore_whitespace`` defined in base class.
6357 """
6358 self.skipWhitespace = True
6359 return self
6361 def streamline(self) -> ParserElement:
6362 if not self.streamlined:
6363 self.streamlined = True
6364 if self.expr is not None:
6365 self.expr.streamline()
6366 return self
6368 def validate(self, validateTrace=None) -> None:
6369 warnings.warn(
6370 "ParserElement.validate() is deprecated, and should not be used to check for left recursion",
6371 PyparsingDeprecationWarning,
6372 stacklevel=2,
6373 )
6374 if validateTrace is None:
6375 validateTrace = []
6377 if self not in validateTrace:
6378 tmp = validateTrace[:] + [self]
6379 if self.expr is not None:
6380 self.expr.validate(tmp)
6381 self._checkRecursion([])
6383 def _generateDefaultName(self) -> str:
6384 # Avoid infinite recursion by setting a temporary _defaultName
6385 save_default_name = self._defaultName
6386 self._defaultName = ": ..."
6388 # Use the string representation of main expression.
6389 try:
6390 if self.expr is not None:
6391 ret_string = str(self.expr)[:1000]
6392 else:
6393 ret_string = "None"
6394 except Exception:
6395 ret_string = "..."
6397 self._defaultName = save_default_name
6398 return f"{type(self).__name__}: {ret_string}"
6400 def copy(self) -> ParserElement:
6401 """
6402 Returns a copy of this expression.
6404 Generally only used internally by pyparsing.
6405 """
6406 if self.expr is not None:
6407 return super().copy()
6408 else:
6409 ret = Forward()
6410 ret <<= self
6411 return ret
6413 def _setResultsName(self, name, list_all_matches=False) -> ParserElement:
6414 # fmt: off
6415 if (
6416 __diag__.warn_name_set_on_empty_Forward
6417 and Diagnostics.warn_name_set_on_empty_Forward not in self.suppress_warnings_
6418 and self.expr is None
6419 ):
6420 warning = (
6421 "warn_name_set_on_empty_Forward:"
6422 f" setting results name {name!r} on {type(self).__name__} expression"
6423 " that has no contained expression"
6424 )
6425 warnings.warn(warning, PyparsingDiagnosticWarning, stacklevel=3)
6426 # fmt: on
6428 return super()._setResultsName(name, list_all_matches)
6430 # Compatibility synonyms
6431 # fmt: off
6432 leaveWhitespace = replaced_by_pep8("leaveWhitespace", leave_whitespace)
6433 ignoreWhitespace = replaced_by_pep8("ignoreWhitespace", ignore_whitespace)
6434 # fmt: on
6437class TokenConverter(ParseElementEnhance):
6438 """
6439 Abstract subclass of :class:`ParseElementEnhance`, for converting parsed results.
6440 """
6442 def __init__(self, expr: Union[ParserElement, str], savelist=False) -> None:
6443 super().__init__(expr) # , savelist)
6444 self.saveAsList = False
6447class Combine(TokenConverter):
6448 """Converter to concatenate all matching tokens to a single string.
6449 By default, the matching patterns must also be contiguous in the
6450 input string; this can be disabled by specifying
6451 ``'adjacent=False'`` in the constructor.
6453 Example:
6455 .. doctest::
6457 >>> real = Word(nums) + '.' + Word(nums)
6458 >>> print(real.parse_string('3.1416'))
6459 ['3', '.', '1416']
6461 >>> # will also erroneously match the following
6462 >>> print(real.parse_string('3. 1416'))
6463 ['3', '.', '1416']
6465 >>> real = Combine(Word(nums) + '.' + Word(nums))
6466 >>> print(real.parse_string('3.1416'))
6467 ['3.1416']
6469 >>> # no match when there are internal spaces
6470 >>> print(real.parse_string('3. 1416'))
6471 Traceback (most recent call last):
6472 ParseException: Expected W:(0123...)
6473 """
6475 def __init__(
6476 self,
6477 expr: ParserElement,
6478 join_string: str = "",
6479 adjacent: bool = True,
6480 *,
6481 joinString: typing.Optional[str] = None,
6482 ) -> None:
6483 super().__init__(expr)
6484 joinString = joinString if joinString is not None else join_string
6485 # suppress whitespace-stripping in contained parse expressions, but re-enable it on the Combine itself
6486 if adjacent:
6487 self.leave_whitespace()
6488 self.adjacent = adjacent
6489 self.skipWhitespace = True
6490 self.joinString = joinString
6491 self.callPreparse = True
6493 def ignore(self, other) -> ParserElement:
6494 """
6495 Define expression to be ignored (e.g., comments) while doing pattern
6496 matching; may be called repeatedly, to define multiple comment or other
6497 ignorable patterns.
6498 """
6499 if self.adjacent:
6500 ParserElement.ignore(self, other)
6501 else:
6502 super().ignore(other)
6503 return self
6505 def postParse(self, instring, loc, tokenlist):
6506 retToks = tokenlist.copy()
6507 del retToks[:]
6508 retToks += ParseResults(
6509 ["".join(tokenlist._asStringList(self.joinString))], modal=self.modalResults
6510 )
6512 if self.resultsName and retToks.haskeys():
6513 return [retToks]
6514 else:
6515 return retToks
6518class Group(TokenConverter):
6519 """Converter to return the matched tokens as a list - useful for
6520 returning tokens of :class:`ZeroOrMore` and :class:`OneOrMore` expressions.
6522 The optional ``aslist`` argument when set to True will return the
6523 parsed tokens as a Python list instead of a pyparsing ParseResults.
6525 Example:
6527 .. doctest::
6529 >>> ident = Word(alphas)
6530 >>> num = Word(nums)
6531 >>> term = ident | num
6532 >>> func = ident + Opt(DelimitedList(term))
6533 >>> print(func.parse_string("fn a, b, 100"))
6534 ['fn', 'a', 'b', '100']
6536 >>> func = ident + Group(Opt(DelimitedList(term)))
6537 >>> print(func.parse_string("fn a, b, 100"))
6538 ['fn', ['a', 'b', '100']]
6539 """
6541 def __init__(self, expr: ParserElement, aslist: bool = False) -> None:
6542 super().__init__(expr)
6543 self.saveAsList = True
6544 self._asPythonList = aslist
6546 def postParse(self, instring, loc, tokenlist):
6547 if self._asPythonList:
6548 return ParseResults.List(
6549 tokenlist.as_list()
6550 if isinstance(tokenlist, ParseResults)
6551 else list(tokenlist)
6552 )
6554 return [tokenlist]
6557class Dict(TokenConverter):
6558 """Converter to return a repetitive expression as a list, but also
6559 as a dictionary. Each element can also be referenced using the first
6560 token in the expression as its key. Useful for tabular report
6561 scraping when the first column can be used as a item key.
6563 The optional ``asdict`` argument when set to True will return the
6564 parsed tokens as a Python dict instead of a pyparsing ParseResults.
6566 Example:
6568 .. doctest::
6570 >>> data_word = Word(alphas)
6571 >>> label = data_word + FollowedBy(':')
6573 >>> attr_expr = (
6574 ... label + Suppress(':')
6575 ... + OneOrMore(data_word, stop_on=label)
6576 ... .set_parse_action(' '.join)
6577 ... )
6579 >>> text = "shape: SQUARE posn: upper left color: light blue texture: burlap"
6581 >>> # print attributes as plain groups
6582 >>> print(attr_expr[1, ...].parse_string(text).dump())
6583 ['shape', 'SQUARE', 'posn', 'upper left', 'color', 'light blue', 'texture', 'burlap']
6585 # instead of OneOrMore(expr), parse using Dict(Group(expr)[1, ...])
6586 # Dict will auto-assign names.
6587 >>> result = Dict(Group(attr_expr)[1, ...]).parse_string(text)
6588 >>> print(result.dump())
6589 [['shape', 'SQUARE'], ['posn', 'upper left'], ['color', 'light blue'], ['texture', 'burlap']]
6590 - color: 'light blue'
6591 - posn: 'upper left'
6592 - shape: 'SQUARE'
6593 - texture: 'burlap'
6594 [0]:
6595 ['shape', 'SQUARE']
6596 [1]:
6597 ['posn', 'upper left']
6598 [2]:
6599 ['color', 'light blue']
6600 [3]:
6601 ['texture', 'burlap']
6603 # access named fields as dict entries, or output as dict
6604 >>> print(result['shape'])
6605 SQUARE
6606 >>> print(result.as_dict())
6607 {'shape': 'SQUARE', 'posn': 'upper left', 'color': 'light blue', 'texture': 'burlap'}
6609 See more examples at :class:`ParseResults` of accessing fields by results name.
6610 """
6612 def __init__(self, expr: ParserElement, asdict: bool = False) -> None:
6613 super().__init__(expr)
6614 self.saveAsList = True
6615 self._asPythonDict = asdict
6617 def postParse(self, instring, loc, tokenlist):
6618 for i, tok in enumerate(tokenlist):
6619 if len(tok) == 0:
6620 continue
6622 ikey = tok[0]
6623 if isinstance(ikey, int):
6624 ikey = str(ikey).strip()
6626 if len(tok) == 1:
6627 tokenlist[ikey] = _ParseResultsWithOffset("", i)
6629 elif len(tok) == 2 and not isinstance(tok[1], ParseResults):
6630 tokenlist[ikey] = _ParseResultsWithOffset(tok[1], i)
6632 else:
6633 try:
6634 dictvalue = tok.copy() # ParseResults(i)
6635 except Exception:
6636 exc = TypeError(
6637 "could not extract dict values from parsed results"
6638 " - Dict expression must contain Grouped expressions"
6639 )
6640 raise exc from None
6642 del dictvalue[0]
6644 if len(dictvalue) != 1 or (
6645 isinstance(dictvalue, ParseResults) and dictvalue.haskeys()
6646 ):
6647 tokenlist[ikey] = _ParseResultsWithOffset(dictvalue, i)
6648 else:
6649 tokenlist[ikey] = _ParseResultsWithOffset(dictvalue[0], i)
6651 if self._asPythonDict:
6652 return [tokenlist.as_dict()] if self.resultsName else tokenlist.as_dict()
6654 return [tokenlist] if self.resultsName else tokenlist
6657class Suppress(TokenConverter):
6658 """Converter for ignoring the results of a parsed expression.
6660 Example:
6662 .. doctest::
6664 >>> source = "a, b, c,d"
6665 >>> wd = Word(alphas)
6666 >>> wd_list1 = wd + (',' + wd)[...]
6667 >>> print(wd_list1.parse_string(source))
6668 ['a', ',', 'b', ',', 'c', ',', 'd']
6670 # often, delimiters that are useful during parsing are just in the
6671 # way afterward - use Suppress to keep them out of the parsed output
6672 >>> wd_list2 = wd + (Suppress(',') + wd)[...]
6673 >>> print(wd_list2.parse_string(source))
6674 ['a', 'b', 'c', 'd']
6676 # Skipped text (using '...') can be suppressed as well
6677 >>> source = "lead in START relevant text END trailing text"
6678 >>> start_marker = Keyword("START")
6679 >>> end_marker = Keyword("END")
6680 >>> find_body = Suppress(...) + start_marker + ... + end_marker
6681 >>> print(find_body.parse_string(source))
6682 ['START', 'relevant text ', 'END']
6684 (See also :class:`DelimitedList`.)
6685 """
6687 def __init__(self, expr: Union[ParserElement, str], savelist: bool = False) -> None:
6688 if expr is ...:
6689 expr = _PendingSkip(NoMatch())
6690 super().__init__(expr)
6692 def __add__(self, other) -> ParserElement:
6693 if isinstance(self.expr, _PendingSkip):
6694 return Suppress(SkipTo(other)) + other
6696 return super().__add__(other)
6698 def __sub__(self, other) -> ParserElement:
6699 if isinstance(self.expr, _PendingSkip):
6700 return Suppress(SkipTo(other)) - other
6702 return super().__sub__(other)
6704 def postParse(self, instring, loc, tokenlist):
6705 return []
6707 def suppress(self) -> ParserElement:
6708 return self
6711# XXX: Example needs to be re-done for updated output
6712def trace_parse_action(f: ParseAction) -> ParseAction:
6713 """Decorator for debugging parse actions.
6715 When the parse action is called, this decorator will print
6716 ``">> entering method-name(line:<current_source_line>, <parse_location>, <matched_tokens>)"``.
6717 When the parse action completes, the decorator will print
6718 ``"<<"`` followed by the returned value, or any exception that the parse action raised.
6720 Example:
6722 .. testsetup:: stderr
6724 import sys
6725 sys.stderr = sys.stdout
6727 .. testcleanup:: stderr
6729 sys.stderr = sys.__stderr__
6731 .. testcode:: stderr
6733 wd = Word(alphas)
6735 @trace_parse_action
6736 def remove_duplicate_chars(tokens):
6737 return ''.join(sorted(set(''.join(tokens))))
6739 wds = wd[1, ...].set_parse_action(remove_duplicate_chars)
6740 print(wds.parse_string("slkdjs sld sldd sdlf sdljf"))
6742 prints:
6744 .. testoutput:: stderr
6745 :options: +NORMALIZE_WHITESPACE
6747 >>entering remove_duplicate_chars(line: 'slkdjs sld sldd sdlf sdljf',
6748 0, ParseResults(['slkdjs', 'sld', 'sldd', 'sdlf', 'sdljf'], {}))
6749 <<leaving remove_duplicate_chars (ret: 'dfjkls')
6750 ['dfjkls']
6752 .. versionchanged:: 3.1.0
6753 Exception type added to output
6754 """
6755 f = _trim_arity(f)
6757 def z(*paArgs):
6758 thisFunc = f.__name__
6759 s, l, t = paArgs[-3:]
6760 if len(paArgs) > 3:
6761 thisFunc = f"{type(paArgs[0]).__name__}.{thisFunc}"
6762 sys.stderr.write(f">>entering {thisFunc}(line: {line(l, s)!r}, {l}, {t!r})\n")
6763 try:
6764 ret = f(*paArgs)
6765 except Exception as exc:
6766 sys.stderr.write(
6767 f"<<leaving {thisFunc} (exception: {type(exc).__name__}: {exc})\n"
6768 )
6769 raise
6770 sys.stderr.write(f"<<leaving {thisFunc} (ret: {ret!r})\n")
6771 return ret
6773 z.__name__ = f.__name__
6774 return z
6777# convenience constants for positional expressions
6778empty = Empty().set_name("empty")
6779line_start = LineStart().set_name("line_start")
6780line_end = LineEnd().set_name("line_end")
6781string_start = StringStart().set_name("string_start")
6782string_end = StringEnd().set_name("string_end")
6784_escapedPunc = Regex(r"\\[\\[\]\/\-\*\.\$\+\^\?()~ ]").set_parse_action(
6785 lambda s, l, t: t[0][1]
6786)
6787_escapedHexChar = Regex(r"\\0?[xX][0-9a-fA-F]+").set_parse_action(
6788 lambda s, l, t: chr(int(t[0].lstrip(r"\0x"), 16))
6789)
6790_escapedOctChar = Regex(r"\\0[0-7]+").set_parse_action(
6791 lambda s, l, t: chr(int(t[0][1:], 8))
6792)
6793_singleChar = (
6794 _escapedPunc | _escapedHexChar | _escapedOctChar | CharsNotIn(r"\]", exact=1)
6795)
6796_charRange = Group(_singleChar + Suppress("-") + _singleChar)
6797_reBracketExpr = (
6798 Literal("[")
6799 + Opt("^").set_results_name("negate")
6800 + Group(OneOrMore(_charRange | _singleChar)).set_results_name("body")
6801 + Literal("]")
6802)
6805def srange(s: str) -> str:
6806 r"""Helper to easily define string ranges for use in :class:`Word`
6807 construction. Borrows syntax from regexp ``'[]'`` string range
6808 definitions::
6810 srange("[0-9]") -> "0123456789"
6811 srange("[a-z]") -> "abcdefghijklmnopqrstuvwxyz"
6812 srange("[a-z$_]") -> "abcdefghijklmnopqrstuvwxyz$_"
6814 The input string must be enclosed in []'s, and the returned string
6815 is the expanded character set joined into a single string. The
6816 values enclosed in the []'s may be:
6818 - a single character
6819 - an escaped character with a leading backslash (such as ``\-``
6820 or ``\]``)
6821 - an escaped hex character with a leading ``'\x'``
6822 (``\x21``, which is a ``'!'`` character) (``\0x##``
6823 is also supported for backwards compatibility)
6824 - an escaped octal character with a leading ``'\0'``
6825 (``\041``, which is a ``'!'`` character)
6826 - a range of any of the above, separated by a dash (``'a-z'``,
6827 etc.)
6828 - any combination of the above (``'aeiouy'``,
6829 ``'a-zA-Z0-9_$'``, etc.)
6830 """
6832 def _expanded(p):
6833 if isinstance(p, ParseResults):
6834 yield from (chr(c) for c in range(ord(p[0]), ord(p[1]) + 1))
6835 else:
6836 yield p
6838 try:
6839 return "".join(
6840 [c for part in _reBracketExpr.parse_string(s).body for c in _expanded(part)]
6841 )
6842 except Exception as e:
6843 return ""
6846def token_map(func, *args) -> ParseAction:
6847 """Helper to define a parse action by mapping a function to all
6848 elements of a :class:`ParseResults` list. If any additional args are passed,
6849 they are forwarded to the given function as additional arguments
6850 after the token, as in
6851 ``hex_integer = Word(hexnums).set_parse_action(token_map(int, 16))``,
6852 which will convert the parsed data to an integer using base 16.
6854 Example (compare the last to example in :class:`ParserElement.transform_string`::
6856 hex_ints = Word(hexnums)[1, ...].set_parse_action(token_map(int, 16))
6857 hex_ints.run_tests('''
6858 00 11 22 aa FF 0a 0d 1a
6859 ''')
6861 upperword = Word(alphas).set_parse_action(token_map(str.upper))
6862 upperword[1, ...].run_tests('''
6863 my kingdom for a horse
6864 ''')
6866 wd = Word(alphas).set_parse_action(token_map(str.title))
6867 wd[1, ...].set_parse_action(' '.join).run_tests('''
6868 now is the winter of our discontent made glorious summer by this sun of york
6869 ''')
6871 prints::
6873 00 11 22 aa FF 0a 0d 1a
6874 [0, 17, 34, 170, 255, 10, 13, 26]
6876 my kingdom for a horse
6877 ['MY', 'KINGDOM', 'FOR', 'A', 'HORSE']
6879 now is the winter of our discontent made glorious summer by this sun of york
6880 ['Now Is The Winter Of Our Discontent Made Glorious Summer By This Sun Of York']
6881 """
6883 def pa(s, l, t):
6884 return [func(tokn, *args) for tokn in t]
6886 func_name = getattr(func, "__name__", getattr(func, "__class__").__name__)
6887 pa.__name__ = func_name
6889 return pa
6892def autoname_elements() -> None:
6893 """
6894 Utility to simplify mass-naming of parser elements, for
6895 generating railroad diagram with named subdiagrams.
6896 """
6898 # guard against _getframe not being implemented in the current Python
6899 getframe_fn = getattr(sys, "_getframe", lambda _: None)
6900 calling_frame = getframe_fn(1)
6901 if calling_frame is None:
6902 return
6904 # find all locals in the calling frame that are ParserElements
6905 calling_frame = typing.cast(types.FrameType, calling_frame)
6906 for name, var in calling_frame.f_locals.items():
6907 # if no custom name defined, set the name to the var name
6908 if isinstance(var, ParserElement) and not var.customName:
6909 var.set_name(name)
6912dbl_quoted_string = Combine(
6913 Regex(r'"(?:[^"\n\r\\]|(?:"")|(?:\\(?:[^x]|x[0-9a-fA-F]+)))*') + '"'
6914).set_name("string enclosed in double quotes")
6916sgl_quoted_string = Combine(
6917 Regex(r"'(?:[^'\n\r\\]|(?:'')|(?:\\(?:[^x]|x[0-9a-fA-F]+)))*") + "'"
6918).set_name("string enclosed in single quotes")
6920quoted_string = Combine(
6921 (Regex(r'"(?:[^"\n\r\\]|(?:"")|(?:\\(?:[^x]|x[0-9a-fA-F]+)))*') + '"').set_name(
6922 "double quoted string"
6923 )
6924 | (Regex(r"'(?:[^'\n\r\\]|(?:'')|(?:\\(?:[^x]|x[0-9a-fA-F]+)))*") + "'").set_name(
6925 "single quoted string"
6926 )
6927).set_name("quoted string using single or double quotes")
6929# XXX: Is there some way to make this show up in API docs?
6930# .. versionadded:: 3.1.0
6931python_quoted_string = Combine(
6932 (Regex(r'"""(?:[^"\\]|""(?!")|"(?!"")|\\.)*', flags=re.MULTILINE) + '"""').set_name(
6933 "multiline double quoted string"
6934 )
6935 ^ (
6936 Regex(r"'''(?:[^'\\]|''(?!')|'(?!'')|\\.)*", flags=re.MULTILINE) + "'''"
6937 ).set_name("multiline single quoted string")
6938 ^ (Regex(r'"(?:[^"\n\r\\]|(?:\\")|(?:\\(?:[^x]|x[0-9a-fA-F]+)))*') + '"').set_name(
6939 "double quoted string"
6940 )
6941 ^ (Regex(r"'(?:[^'\n\r\\]|(?:\\')|(?:\\(?:[^x]|x[0-9a-fA-F]+)))*") + "'").set_name(
6942 "single quoted string"
6943 )
6944).set_name("Python quoted string")
6946unicode_string = Combine("u" + quoted_string.copy()).set_name("unicode string literal")
6949alphas8bit = srange(r"[\0xc0-\0xd6\0xd8-\0xf6\0xf8-\0xff]")
6950punc8bit = srange(r"[\0xa1-\0xbf\0xd7\0xf7]")
6952# build list of built-in expressions, for future reference if a global default value
6953# gets updated
6954_builtin_exprs: list[ParserElement] = [
6955 v for v in vars().values() if isinstance(v, ParserElement)
6956]
6958# Compatibility synonyms
6959# fmt: off
6960sglQuotedString = sgl_quoted_string
6961dblQuotedString = dbl_quoted_string
6962quotedString = quoted_string
6963unicodeString = unicode_string
6964lineStart = line_start
6965lineEnd = line_end
6966stringStart = string_start
6967stringEnd = string_end
6968nullDebugAction = replaced_by_pep8("nullDebugAction", null_debug_action)
6969traceParseAction = replaced_by_pep8("traceParseAction", trace_parse_action)
6970conditionAsParseAction = replaced_by_pep8("conditionAsParseAction", condition_as_parse_action)
6971tokenMap = replaced_by_pep8("tokenMap", token_map)
6972# fmt: on