Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/pyparsing/core.py: 46%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#
2# core.py
3#
4from __future__ import annotations
6import collections.abc
7from collections import deque
8import os
9import typing
10from typing import (
11 Any,
12 Callable,
13 Generator,
14 NamedTuple,
15 Sequence,
16 TextIO,
17 Union,
18 cast,
19)
20from abc import ABC, abstractmethod
21from enum import Enum
22import string
23import copy
24import warnings
25import re
26import sys
27from collections.abc import Iterable
28import traceback
29import types
30from operator import itemgetter
31from functools import wraps
32from threading import RLock
33from pathlib import Path
35from .warnings import PyparsingDeprecationWarning, PyparsingDiagnosticWarning
36from .util import (
37 _FifoCache,
38 _UnboundedCache,
39 __config_flags,
40 _collapse_string_to_ranges,
41 _convert_escaped_numerics_to_char,
42 _escape_regex_range_chars,
43 _flatten,
44 LRUMemo as _LRUMemo,
45 UnboundedMemo as _UnboundedMemo,
46 deprecate_argument,
47 replaced_by_pep8,
48)
49from .exceptions import *
50from .actions import *
51from .results import ParseResults, _ParseResultsWithOffset
52from .unicode import pyparsing_unicode
54_MAX_INT = sys.maxsize
55str_type: tuple[type, ...] = (str, bytes)
57#
58# Copyright (c) 2003-2022 Paul T. McGuire
59#
60# Permission is hereby granted, free of charge, to any person obtaining
61# a copy of this software and associated documentation files (the
62# "Software"), to deal in the Software without restriction, including
63# without limitation the rights to use, copy, modify, merge, publish,
64# distribute, sublicense, and/or sell copies of the Software, and to
65# permit persons to whom the Software is furnished to do so, subject to
66# the following conditions:
67#
68# The above copyright notice and this permission notice shall be
69# included in all copies or substantial portions of the Software.
70#
71# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
72# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
73# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
74# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
75# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
76# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
77# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
78#
80from functools import cached_property
83class __compat__(__config_flags):
84 """
85 A cross-version compatibility configuration for pyparsing features that will be
86 released in a future version. By setting values in this configuration to True,
87 those features can be enabled in prior versions for compatibility development
88 and testing.
90 - ``collect_all_And_tokens`` - flag to enable fix for Issue #63 that fixes erroneous grouping
91 of results names when an :class:`And` expression is nested within an :class:`Or` or :class:`MatchFirst`;
92 maintained for compatibility, but setting to ``False`` no longer restores pre-2.3.1
93 behavior
94 """
96 _type_desc = "compatibility"
98 collect_all_And_tokens = True
100 _all_names = [__ for __ in locals() if not __.startswith("_")]
101 _fixed_names = """
102 collect_all_And_tokens
103 """.split()
106class __diag__(__config_flags):
107 _type_desc = "diagnostic"
109 warn_multiple_tokens_in_named_alternation = False
110 warn_ungrouped_named_tokens_in_collection = False
111 warn_name_set_on_empty_Forward = False
112 warn_on_parse_using_empty_Forward = False
113 warn_on_assignment_to_Forward = False
114 warn_on_multiple_string_args_to_oneof = False
115 warn_on_match_first_with_lshift_operator = False
116 enable_debug_on_named_expressions = False
118 _all_names = [__ for __ in locals() if not __.startswith("_")]
119 _warning_names = [name for name in _all_names if name.startswith("warn")]
120 _debug_names = [name for name in _all_names if name.startswith("enable_debug")]
122 @classmethod
123 def enable_all_warnings(cls) -> None:
124 for name in cls._warning_names:
125 cls.enable(name)
128class Diagnostics(Enum):
129 """
130 Diagnostic configuration (all default to disabled)
132 - ``warn_multiple_tokens_in_named_alternation`` - flag to enable warnings when a results
133 name is defined on a :class:`MatchFirst` or :class:`Or` expression with one or more :class:`And` subexpressions
134 - ``warn_ungrouped_named_tokens_in_collection`` - flag to enable warnings when a results
135 name is defined on a containing expression with ungrouped subexpressions that also
136 have results names
137 - ``warn_name_set_on_empty_Forward`` - flag to enable warnings when a :class:`Forward` is defined
138 with a results name, but has no contents defined
139 - ``warn_on_parse_using_empty_Forward`` - flag to enable warnings when a :class:`Forward` is
140 defined in a grammar but has never had an expression attached to it
141 - ``warn_on_assignment_to_Forward`` - flag to enable warnings when a :class:`Forward` is defined
142 but is overwritten by assigning using ``'='`` instead of ``'<<='`` or ``'<<'``
143 - ``warn_on_multiple_string_args_to_oneof`` - flag to enable warnings when :class:`one_of` is
144 incorrectly called with multiple str arguments
145 - ``enable_debug_on_named_expressions`` - flag to auto-enable debug on all subsequent
146 calls to :class:`ParserElement.set_name`
148 Diagnostics are enabled/disabled by calling :class:`enable_diag` and :class:`disable_diag`.
149 All warnings can be enabled by calling :class:`enable_all_warnings`.
150 """
152 warn_multiple_tokens_in_named_alternation = 0
153 warn_ungrouped_named_tokens_in_collection = 1
154 warn_name_set_on_empty_Forward = 2
155 warn_on_parse_using_empty_Forward = 3
156 warn_on_assignment_to_Forward = 4
157 warn_on_multiple_string_args_to_oneof = 5
158 warn_on_match_first_with_lshift_operator = 6
159 enable_debug_on_named_expressions = 7
162def enable_diag(diag_enum: Diagnostics) -> None:
163 """
164 Enable a global pyparsing diagnostic flag (see :class:`Diagnostics`).
165 """
166 __diag__.enable(diag_enum.name)
169def disable_diag(diag_enum: Diagnostics) -> None:
170 """
171 Disable a global pyparsing diagnostic flag (see :class:`Diagnostics`).
172 """
173 __diag__.disable(diag_enum.name)
176def enable_all_warnings() -> None:
177 """
178 Enable all global pyparsing diagnostic warnings (see :class:`Diagnostics`).
179 """
180 __diag__.enable_all_warnings()
183# hide abstract class
184del __config_flags
187def _should_enable_warnings(
188 cmd_line_warn_options: typing.Iterable[str], warn_env_var: typing.Optional[str]
189) -> bool:
190 enable = bool(warn_env_var)
191 for warn_opt in cmd_line_warn_options:
192 w_action, w_message, w_category, w_module, w_line = (warn_opt + "::::").split(
193 ":"
194 )[:5]
195 if not w_action.lower().startswith("i") and (
196 not (w_message or w_category or w_module) or w_module == "pyparsing"
197 ):
198 enable = True
199 elif w_action.lower().startswith("i") and w_module in ("pyparsing", ""):
200 enable = False
201 return enable
204if _should_enable_warnings(
205 sys.warnoptions, os.environ.get("PYPARSINGENABLEALLWARNINGS")
206):
207 enable_all_warnings()
210# build list of single arg builtins, that can be used as parse actions
211# fmt: off
212_single_arg_builtins = {
213 sum, len, sorted, reversed, list, tuple, set, any, all, min, max
214}
215# fmt: on
217_generatorType = types.GeneratorType
218ParseImplReturnType = tuple[int, Any]
219PostParseReturnType = Union[ParseResults, Sequence[ParseResults]]
221ParseCondition = Union[
222 Callable[[], bool],
223 Callable[[ParseResults], bool],
224 Callable[[int, ParseResults], bool],
225 Callable[[str, int, ParseResults], bool],
226]
227ParseFailAction = Callable[[str, int, "ParserElement", Exception], None]
228DebugStartAction = Callable[[str, int, "ParserElement", bool], None]
229DebugSuccessAction = Callable[
230 [str, int, int, "ParserElement", ParseResults, bool], None
231]
232DebugExceptionAction = Callable[[str, int, "ParserElement", Exception, bool], None]
235alphas: str = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
236identchars: str = pyparsing_unicode.Latin1.identchars
237identbodychars: str = pyparsing_unicode.Latin1.identbodychars
238nums: str = "0123456789"
239hexnums: str = "0123456789ABCDEFabcdef"
240alphanums: str = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
241printables: str = (
242 '!"'
243 "#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ"
244 "[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~"
245)
248class _ParseActionIndexError(Exception):
249 """
250 Internal wrapper around IndexError so that IndexErrors raised inside
251 parse actions aren't misinterpreted as IndexErrors raised inside
252 ParserElement parseImpl methods.
253 """
255 def __init__(self, msg: str, exc: BaseException) -> None:
256 self.msg: str = msg
257 self.exc: BaseException = exc
260_trim_arity_call_line: traceback.StackSummary = None # type: ignore[assignment]
261pa_call_line_synth = ()
264def _trim_arity(func, max_limit=3):
265 """decorator to trim function calls to match the arity of the target"""
266 global _trim_arity_call_line, pa_call_line_synth
268 if func in _single_arg_builtins:
269 return lambda s, l, t: func(t)
271 limit = 0
272 found_arity = False
274 # synthesize what would be returned by traceback.extract_stack at the call to
275 # user's parse action 'func', so that we don't incur call penalty at parse time
277 # fmt: off
278 LINE_DIFF = 9
279 # IF ANY CODE CHANGES, EVEN JUST COMMENTS OR BLANK LINES, BETWEEN THE NEXT LINE AND
280 # THE CALL TO FUNC INSIDE WRAPPER, LINE_DIFF MUST BE MODIFIED!!!!
281 _trim_arity_call_line = _trim_arity_call_line or traceback.extract_stack(limit=2)[-1]
282 pa_call_line_synth = pa_call_line_synth or (_trim_arity_call_line[0], _trim_arity_call_line[1] + LINE_DIFF)
284 def wrapper(*args):
285 nonlocal found_arity, limit
286 if found_arity:
287 return func(*args[limit:])
288 while 1:
289 try:
290 ret = func(*args[limit:])
291 found_arity = True
292 return ret
293 except TypeError as te:
294 # re-raise TypeErrors if they did not come from our arity testing
295 if found_arity:
296 raise
297 else:
298 tb = te.__traceback__
299 frames = traceback.extract_tb(tb, limit=2)
300 frame_summary = frames[-1]
301 trim_arity_type_error = (
302 [frame_summary[:2]][-1][:2] == pa_call_line_synth
303 )
304 del tb
306 if trim_arity_type_error:
307 if limit < max_limit:
308 limit += 1
309 continue
311 raise
312 except IndexError as ie:
313 # wrap IndexErrors inside a _ParseActionIndexError
314 raise _ParseActionIndexError(
315 "IndexError raised in parse action", ie
316 ).with_traceback(None)
317 # fmt: on
319 # copy func name to wrapper for sensible debug output
320 # (can't use functools.wraps, since that messes with function signature)
321 func_name = getattr(func, "__name__", getattr(func, "__class__").__name__)
322 wrapper.__name__ = func_name
323 wrapper.__doc__ = func.__doc__
325 return wrapper
328def condition_as_parse_action(
329 fn: ParseCondition, message: typing.Optional[str] = None, fatal: bool = False
330) -> ParseAction:
331 """
332 Function to convert a simple predicate function that returns ``True`` or ``False``
333 into a parse action. Can be used in places when a parse action is required
334 and :meth:`ParserElement.add_condition` cannot be used (such as when adding a condition
335 to an operator level in :class:`infix_notation`).
337 Optional keyword arguments:
339 :param message: define a custom message to be used in the raised exception
340 :param fatal: if ``True``, will raise :class:`ParseFatalException`
341 to stop parsing immediately;
342 otherwise will raise :class:`ParseException`
344 """
345 msg = message if message is not None else "failed user-defined condition"
346 exc_type = ParseFatalException if fatal else ParseException
347 fn = _trim_arity(fn)
349 @wraps(fn)
350 def pa(s, l, t):
351 if not bool(fn(s, l, t)):
352 raise exc_type(s, l, msg)
354 return pa
357# control characters escaped so they can't corrupt the printed debug line
358# (for example a stray '\r' would otherwise return the cursor to column 0)
359_debug_control_char_map = {c: repr(chr(c))[1:-1] for c in (*range(0x20), 0x7F)}
362def _default_start_debug_action(
363 instring: str, loc: int, expr: ParserElement, cache_hit: bool = False
364):
365 cache_hit_str = "*" if cache_hit else ""
366 current_col = col(loc, instring)
367 current_line = line(loc, instring)
368 escaped_line = current_line.translate(_debug_control_char_map)
369 # keep the caret under the match location after escaping widens the line
370 caret_col = (
371 len(current_line[: current_col - 1].translate(_debug_control_char_map)) + 1
372 )
373 print(
374 (
375 f"{cache_hit_str}Match {expr} at loc {loc}({lineno(loc, instring)},{current_col})\n"
376 f" {escaped_line}\n"
377 f" {'^':>{caret_col}}"
378 )
379 )
382def _default_success_debug_action(
383 instring: str,
384 startloc: int,
385 endloc: int,
386 expr: ParserElement,
387 toks: ParseResults,
388 cache_hit: bool = False,
389):
390 cache_hit_str = "*" if cache_hit else ""
391 print(f"{cache_hit_str}Matched {expr} -> {toks.as_list()}")
394def _default_exception_debug_action(
395 instring: str,
396 loc: int,
397 expr: ParserElement,
398 exc: Exception,
399 cache_hit: bool = False,
400):
401 cache_hit_str = "*" if cache_hit else ""
402 print(f"{cache_hit_str}Match {expr} failed, {type(exc).__name__} raised: {exc}")
405def null_debug_action(*args):
406 """'Do-nothing' debug action, to suppress debugging output during parsing."""
409class ParserElement(ABC):
410 """Abstract base level parser element class."""
412 DEFAULT_WHITE_CHARS: str = " \n\t\r"
413 verbose_stacktrace: bool = False
414 _literalStringClass: type = None # type: ignore[assignment]
416 @staticmethod
417 def set_default_whitespace_chars(chars: str) -> None:
418 r"""
419 Overrides the default whitespace chars
421 Example:
423 .. doctest::
425 # default whitespace chars are space, <TAB> and newline
426 >>> Word(alphas)[1, ...].parse_string("abc def\nghi jkl")
427 ParseResults(['abc', 'def', 'ghi', 'jkl'], {})
429 # change to just treat newline as significant
430 >>> ParserElement.set_default_whitespace_chars(" \t")
431 >>> Word(alphas)[1, ...].parse_string("abc def\nghi jkl")
432 ParseResults(['abc', 'def'], {})
434 # Reset to default
435 >>> ParserElement.set_default_whitespace_chars(" \n\t\r")
436 """
437 ParserElement.DEFAULT_WHITE_CHARS = chars
439 # update whitespace all parse expressions defined in this module
440 for expr in _builtin_exprs:
441 if expr.copyDefaultWhiteChars:
442 expr.whiteChars = set(chars)
444 @staticmethod
445 def inline_literals_using(cls: type) -> None:
446 """
447 Set class to be used for inclusion of string literals into a parser.
449 Example:
451 .. doctest::
452 :options: +NORMALIZE_WHITESPACE
454 # default literal class used is Literal
455 >>> integer = Word(nums)
456 >>> date_str = (
457 ... integer("year") + '/'
458 ... + integer("month") + '/'
459 ... + integer("day")
460 ... )
462 >>> date_str.parse_string("1999/12/31")
463 ParseResults(['1999', '/', '12', '/', '31'],
464 {'year': '1999', 'month': '12', 'day': '31'})
466 # change to Suppress
467 >>> ParserElement.inline_literals_using(Suppress)
468 >>> date_str = (
469 ... integer("year") + '/'
470 ... + integer("month") + '/'
471 ... + integer("day")
472 ... )
474 >>> date_str.parse_string("1999/12/31")
475 ParseResults(['1999', '12', '31'],
476 {'year': '1999', 'month': '12', 'day': '31'})
478 # Reset
479 >>> ParserElement.inline_literals_using(Literal)
480 """
481 ParserElement._literalStringClass = cls
483 @classmethod
484 def using_each(cls, seq, **class_kwargs):
485 """
486 Yields a sequence of ``class(obj, **class_kwargs)`` for obj in seq.
488 Example:
490 .. testcode::
492 LPAR, RPAR, LBRACE, RBRACE, SEMI = Suppress.using_each("(){};")
494 .. versionadded:: 3.1.0
495 """
496 yield from (cls(obj, **class_kwargs) for obj in seq)
498 class DebugActions(NamedTuple):
499 debug_try: typing.Optional[DebugStartAction]
500 debug_match: typing.Optional[DebugSuccessAction]
501 debug_fail: typing.Optional[DebugExceptionAction]
503 def __init__(self, savelist: bool = False) -> None:
504 self.parseAction: list[ParseAction] = list()
505 self.failAction: typing.Optional[ParseFailAction] = None
506 self.customName: str = None # type: ignore[assignment]
507 self._defaultName: typing.Optional[str] = None
508 self.resultsName: str = None # type: ignore[assignment]
509 self.saveAsList: bool = savelist
510 self.skipWhitespace: bool = True
511 self.whiteChars: set[str] = set(ParserElement.DEFAULT_WHITE_CHARS)
512 self.copyDefaultWhiteChars: bool = True
513 # used when checking for left-recursion
514 self._may_return_empty: bool = False
515 self.keepTabs: bool = False
516 self.ignoreExprs: list[ParserElement] = list()
517 self.debug: bool = False
518 self.streamlined: bool = False
519 # optimize exception handling for subclasses that don't advance parse index
520 self.mayIndexError: bool = True
521 self.errmsg: Union[str, None] = ""
522 # mark results names as modal (report only last) or cumulative (list all)
523 self.modalResults: bool = True
524 # custom debug actions
525 self.debugActions = self.DebugActions(None, None, None)
526 # avoid redundant calls to preParse
527 self.callPreparse: bool = True
528 self.callDuringTry: bool = False
529 self.suppress_warnings_: list[Diagnostics] = []
530 self.show_in_diagram: bool = True
532 @property
533 def mayReturnEmpty(self) -> bool:
534 """
535 .. deprecated:: 3.3.0
536 use _may_return_empty instead.
537 """
538 return self._may_return_empty
540 @mayReturnEmpty.setter
541 def mayReturnEmpty(self, value) -> None:
542 """
543 .. deprecated:: 3.3.0
544 use _may_return_empty instead.
545 """
546 self._may_return_empty = value
548 def suppress_warning(self, warning_type: Diagnostics) -> ParserElement:
549 """
550 Suppress warnings emitted for a particular diagnostic on this expression.
552 Example:
554 .. doctest::
556 >>> label = pp.Word(pp.alphas)
558 # Normally using an empty Forward in a grammar
559 # would print a warning, but we can suppress that
560 >>> base = pp.Forward().suppress_warning(
561 ... pp.Diagnostics.warn_on_parse_using_empty_Forward)
563 >>> grammar = base | label
564 >>> print(grammar.parse_string("x"))
565 ['x']
566 """
567 self.suppress_warnings_.append(warning_type)
568 return self
570 def visit_all(self):
571 """General-purpose method to yield all expressions and sub-expressions
572 in a grammar. Typically just for internal use.
573 """
574 to_visit = deque([self])
575 seen = set()
576 while to_visit:
577 cur = to_visit.popleft()
579 # guard against looping forever through recursive grammars
580 if cur in seen:
581 continue
582 seen.add(cur)
584 to_visit.extend(cur.recurse())
585 yield cur
587 def copy(self) -> ParserElement:
588 """
589 Make a copy of this :class:`ParserElement`. Useful for defining
590 different parse actions for the same parsing pattern, using copies of
591 the original parse element.
593 Example:
595 .. testcode::
597 integer = Word(nums).set_parse_action(
598 lambda toks: int(toks[0]))
599 integerK = integer.copy().add_parse_action(
600 lambda toks: toks[0] * 1024) + Suppress("K")
601 integerM = integer.copy().add_parse_action(
602 lambda toks: toks[0] * 1024 * 1024) + Suppress("M")
604 print(
605 (integerK | integerM | integer)[1, ...].parse_string(
606 "5K 100 640K 256M")
607 )
609 prints:
611 .. testoutput::
613 [5120, 100, 655360, 268435456]
615 Equivalent form of ``expr.copy()`` is just ``expr()``:
617 .. testcode::
619 integerM = integer().add_parse_action(
620 lambda toks: toks[0] * 1024 * 1024) + Suppress("M")
621 """
622 cpy = copy.copy(self)
623 cpy.parseAction = self.parseAction[:]
624 cpy.ignoreExprs = self.ignoreExprs[:]
625 if self.copyDefaultWhiteChars:
626 cpy.whiteChars = set(ParserElement.DEFAULT_WHITE_CHARS)
627 return cpy
629 def set_results_name(
630 self, name: str, list_all_matches: bool = False, **kwargs
631 ) -> ParserElement:
632 """
633 Define name for referencing matching tokens as a nested attribute
634 of the returned parse results.
636 Normally, results names are assigned as you would assign keys in a dict:
637 any existing value is overwritten by later values. If it is necessary to
638 keep all values captured for a particular results name, call ``set_results_name``
639 with ``list_all_matches`` = True.
641 NOTE: ``set_results_name`` returns a *copy* of the original :class:`ParserElement` object;
642 this is so that the client can define a basic element, such as an
643 integer, and reference it in multiple places with different names.
645 You can also set results names using the abbreviated syntax,
646 ``expr("name")`` in place of ``expr.set_results_name("name")``
647 - see :meth:`__call__`. If ``list_all_matches`` is required, use
648 ``expr("name*")``.
650 Example:
652 .. testcode::
654 integer = Word(nums)
655 date_str = (integer.set_results_name("year") + '/'
656 + integer.set_results_name("month") + '/'
657 + integer.set_results_name("day"))
659 # equivalent form:
660 date_str = integer("year") + '/' + integer("month") + '/' + integer("day")
661 """
662 listAllMatches: bool = deprecate_argument(kwargs, "listAllMatches", False)
664 list_all_matches = listAllMatches or list_all_matches
665 return self._setResultsName(name, list_all_matches)
667 def _setResultsName(self, name, list_all_matches=False) -> ParserElement:
668 if name is None:
669 return self
670 newself = self.copy()
671 if name.endswith("*"):
672 name = name[:-1]
673 list_all_matches = True
674 newself.resultsName = name
675 newself.modalResults = not list_all_matches
676 return newself
678 def set_break(self, break_flag: bool = True) -> ParserElement:
679 """
680 Method to invoke the Python pdb debugger when this element is
681 about to be parsed. Set ``break_flag`` to ``True`` to enable, ``False`` to
682 disable.
683 """
684 if break_flag:
685 _parseMethod = self._parse
687 def breaker(instring, loc, do_actions=True, callPreParse=True):
688 # this call to breakpoint() is intentional, not a checkin error
689 breakpoint()
690 return _parseMethod(instring, loc, do_actions, callPreParse)
692 breaker._originalParseMethod = _parseMethod # type: ignore [attr-defined]
693 self._parse = breaker # type: ignore [method-assign]
694 elif hasattr(self._parse, "_originalParseMethod"):
695 self._parse = self._parse._originalParseMethod # type: ignore [method-assign]
696 return self
698 def set_parse_action(
699 self, *fns: ParseAction, call_during_try: bool = False, **kwargs: Any
700 ) -> ParserElement:
701 """
702 Define one or more actions to perform when successfully matching parse element definition.
704 Parse actions can be called to perform data conversions, do extra validation,
705 update external data structures, or enhance or replace the parsed tokens.
706 Each parse action ``fn`` is a callable method with 0-3 arguments, called as
707 ``fn(s, loc, toks)`` , ``fn(loc, toks)`` , ``fn(toks)`` , or just ``fn()`` , where:
709 - ``s`` = the original string being parsed (see note below)
710 - ``loc`` = the location of the matching substring
711 - ``toks`` = a list of the matched tokens, packaged as a :class:`ParseResults` object
713 The parsed tokens are passed to the parse action as ParseResults. They can be
714 modified in place using list-style append, extend, and pop operations to update
715 the parsed list elements; and with dictionary-style item set and del operations
716 to add, update, or remove any named results. If the tokens are modified in place,
717 it is not necessary to return them with a return statement.
719 Parse actions can also completely replace the given tokens, with another ``ParseResults``
720 object, or with some entirely different object (common for parse actions that perform data
721 conversions). A convenient way to build a new parse result is to define the values
722 using a dict, and then create the return value using :class:`ParseResults.from_dict`.
724 If None is passed as the ``fn`` parse action, all previously added parse actions for this
725 expression are cleared.
727 Optional keyword arguments:
729 :param call_during_try: (default= ``False``) indicate if parse action
730 should be run during lookaheads and alternate
731 testing. For parse actions that have side
732 effects, it is important to only call the parse
733 action once it is determined that it is being
734 called as part of a successful parse.
735 For parse actions that perform additional
736 validation, then ``call_during_try`` should
737 be passed as True, so that the validation code
738 is included in the preliminary "try" parses.
740 .. Note::
741 The default parsing behavior is to expand tabs in the input string
742 before starting the parsing process.
743 See :meth:`parse_string` for more information on parsing strings
744 containing ``<TAB>`` s, and suggested methods to maintain a
745 consistent view of the parsed string, the parse location, and
746 line and column positions within the parsed string.
748 Example: Parse dates in the form ``YYYY/MM/DD``
749 -----------------------------------------------
751 Setup code:
753 .. testcode::
755 def convert_to_int(toks):
756 '''a parse action to convert toks from str to int
757 at parse time'''
758 return int(toks[0])
760 def is_valid_date(instring, loc, toks):
761 '''a parse action to verify that the date is a valid date'''
762 from datetime import date
763 year, month, day = toks[::2]
764 try:
765 date(year, month, day)
766 except ValueError:
767 raise ParseException(instring, loc, "invalid date given")
769 integer = Word(nums)
770 date_str = integer + '/' + integer + '/' + integer
772 # add parse actions
773 integer.set_parse_action(convert_to_int)
774 date_str.set_parse_action(is_valid_date)
776 Successful parse - note that integer fields are converted to ints:
778 .. testcode::
780 print(date_str.parse_string("1999/12/31"))
782 prints:
784 .. testoutput::
786 [1999, '/', 12, '/', 31]
788 Failure - invalid date:
790 .. testcode::
792 date_str.parse_string("1999/13/31")
794 prints:
796 .. testoutput::
798 Traceback (most recent call last):
799 ParseException: invalid date given, found '1999' ...
800 """
801 callDuringTry: bool = deprecate_argument(kwargs, "callDuringTry", False)
803 if list(fns) == [None]:
804 self.parseAction.clear()
805 return self
807 if not all(callable(fn) for fn in fns):
808 raise TypeError("parse actions must be callable")
809 self.parseAction[:] = [_trim_arity(fn) for fn in fns]
810 self.callDuringTry = self.callDuringTry or call_during_try or callDuringTry
812 return self
814 def add_parse_action(
815 self, *fns: ParseAction, call_during_try: bool = False, **kwargs: Any
816 ) -> ParserElement:
817 """
818 Add one or more parse actions to expression's list of parse actions. See :class:`set_parse_action`.
820 See examples in :class:`copy`.
821 """
822 callDuringTry: bool = deprecate_argument(kwargs, "callDuringTry", False)
824 self.parseAction += [_trim_arity(fn) for fn in fns]
825 self.callDuringTry = self.callDuringTry or callDuringTry or call_during_try
826 return self
828 def add_condition(
829 self, *fns: ParseCondition, call_during_try: bool = False, **kwargs: Any
830 ) -> ParserElement:
831 """Add a boolean predicate function to expression's list of parse actions. See
832 :class:`set_parse_action` for function call signatures. Unlike ``set_parse_action``,
833 functions passed to ``add_condition`` need to return boolean success/fail of the condition.
835 Optional keyword arguments:
837 - ``message`` = define a custom message to be used in the raised exception
838 - ``fatal`` = if True, will raise ParseFatalException to stop parsing immediately; otherwise will raise
839 ParseException
840 - ``call_during_try`` = boolean to indicate if this method should be called during internal tryParse calls,
841 default=False
843 Example:
845 .. doctest::
846 :options: +NORMALIZE_WHITESPACE
848 >>> integer = Word(nums).set_parse_action(lambda toks: int(toks[0]))
849 >>> year_int = integer.copy().add_condition(
850 ... lambda toks: toks[0] >= 2000,
851 ... message="Only support years 2000 and later")
852 >>> date_str = year_int + '/' + integer + '/' + integer
854 >>> result = date_str.parse_string("1999/12/31")
855 Traceback (most recent call last):
856 ParseException: Only support years 2000 and later...
857 """
858 callDuringTry: bool = deprecate_argument(kwargs, "callDuringTry", False)
860 for fn in fns:
861 self.parseAction.append(
862 condition_as_parse_action(
863 fn,
864 message=str(kwargs.get("message")),
865 fatal=bool(kwargs.get("fatal", False)),
866 )
867 )
869 self.callDuringTry = self.callDuringTry or call_during_try or callDuringTry
870 return self
872 def set_fail_action(self, fn: ParseFailAction) -> ParserElement:
873 """
874 Define action to perform if parsing fails at this expression.
875 Fail acton fn is a callable function that takes the arguments
876 ``fn(s, loc, expr, err)`` where:
878 - ``s`` = string being parsed
879 - ``loc`` = location where expression match was attempted and failed
880 - ``expr`` = the parse expression that failed
881 - ``err`` = the exception thrown
883 The function returns no value. It may throw :class:`ParseFatalException`
884 if it is desired to stop parsing immediately."""
885 self.failAction = fn
886 return self
888 def _skipIgnorables(self, instring: str, loc: int) -> int:
889 if not self.ignoreExprs:
890 return loc
891 exprsFound = True
892 ignore_expr_fns = [e._parse for e in self.ignoreExprs]
893 last_loc = loc
894 while exprsFound:
895 exprsFound = False
896 for ignore_fn in ignore_expr_fns:
897 try:
898 while 1:
899 loc, dummy = ignore_fn(instring, loc)
900 exprsFound = True
901 except ParseException:
902 pass
903 # check if all ignore exprs matched but didn't actually advance the parse location
904 if loc == last_loc:
905 break
906 last_loc = loc
907 return loc
909 def preParse(self, instring: str, loc: int) -> int:
910 if self.ignoreExprs:
911 loc = self._skipIgnorables(instring, loc)
913 if self.skipWhitespace:
914 instrlen = len(instring)
915 white_chars = self.whiteChars
916 while loc < instrlen and instring[loc] in white_chars:
917 loc += 1
919 return loc
921 def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType:
922 return loc, []
924 def postParse(self, instring, loc, tokenlist):
925 return tokenlist
927 # @profile
928 def _parseNoCache(
929 self, instring, loc, do_actions=True, callPreParse=True
930 ) -> tuple[int, ParseResults]:
931 debugging = self.debug # and do_actions)
932 len_instring = len(instring)
934 if debugging or self.failAction:
935 # print("Match {} at loc {}({}, {})".format(self, loc, lineno(loc, instring), col(loc, instring)))
936 try:
937 if callPreParse and self.callPreparse:
938 pre_loc = self.preParse(instring, loc)
939 else:
940 pre_loc = loc
941 tokens_start = pre_loc
942 if self.debugActions.debug_try:
943 self.debugActions.debug_try(instring, tokens_start, self, False)
944 if self.mayIndexError or pre_loc >= len_instring:
945 try:
946 loc, tokens = self.parseImpl(instring, pre_loc, do_actions)
947 except IndexError:
948 raise ParseException(instring, len_instring, self.errmsg, self)
949 else:
950 loc, tokens = self.parseImpl(instring, pre_loc, do_actions)
951 except Exception as err:
952 # print("Exception raised:", err)
953 if self.debugActions.debug_fail:
954 self.debugActions.debug_fail(
955 instring, tokens_start, self, err, False
956 )
957 if self.failAction:
958 self.failAction(instring, tokens_start, self, err)
959 raise
960 else:
961 if callPreParse and self.callPreparse:
962 pre_loc = self.preParse(instring, loc)
963 else:
964 pre_loc = loc
965 tokens_start = pre_loc
966 if self.mayIndexError or pre_loc >= len_instring:
967 try:
968 loc, tokens = self.parseImpl(instring, pre_loc, do_actions)
969 except IndexError:
970 raise ParseException(instring, len_instring, self.errmsg, self)
971 else:
972 loc, tokens = self.parseImpl(instring, pre_loc, do_actions)
974 tokens = self.postParse(instring, loc, tokens)
976 ret_tokens = ParseResults(
977 tokens, self.resultsName, aslist=self.saveAsList, modal=self.modalResults
978 )
979 if self.parseAction and (do_actions or self.callDuringTry):
980 if debugging:
981 try:
982 for fn in self.parseAction:
983 try:
984 tokens = fn(instring, tokens_start, ret_tokens) # type: ignore [call-arg, arg-type]
985 except IndexError as parse_action_exc:
986 exc = ParseException("exception raised in parse action")
987 raise exc from parse_action_exc
989 if tokens is not None and tokens is not ret_tokens:
990 ret_tokens = ParseResults(
991 tokens,
992 self.resultsName,
993 aslist=self.saveAsList
994 and isinstance(tokens, (ParseResults, list)),
995 modal=self.modalResults,
996 )
997 except Exception as err:
998 # print "Exception raised in user parse action:", err
999 if self.debugActions.debug_fail:
1000 self.debugActions.debug_fail(
1001 instring, tokens_start, self, err, False
1002 )
1003 raise
1004 else:
1005 for fn in self.parseAction:
1006 try:
1007 tokens = fn(instring, tokens_start, ret_tokens) # type: ignore [call-arg, arg-type]
1008 except IndexError as parse_action_exc:
1009 exc = ParseException("exception raised in parse action")
1010 raise exc from parse_action_exc
1012 if tokens is not None and tokens is not ret_tokens:
1013 ret_tokens = ParseResults(
1014 tokens,
1015 self.resultsName,
1016 aslist=self.saveAsList
1017 and isinstance(tokens, (ParseResults, list)),
1018 modal=self.modalResults,
1019 )
1020 if debugging:
1021 # print("Matched", self, "->", ret_tokens.as_list())
1022 if self.debugActions.debug_match:
1023 self.debugActions.debug_match(
1024 instring, tokens_start, loc, self, ret_tokens, False
1025 )
1027 return loc, ret_tokens
1029 def try_parse(
1030 self,
1031 instring: str,
1032 loc: int,
1033 *,
1034 raise_fatal: bool = False,
1035 do_actions: bool = False,
1036 ) -> int:
1037 try:
1038 return self._parse(instring, loc, do_actions=do_actions)[0]
1039 except ParseFatalException:
1040 if raise_fatal:
1041 raise
1042 raise ParseException(instring, loc, self.errmsg, self)
1044 def can_parse_next(self, instring: str, loc: int, do_actions: bool = False) -> bool:
1045 try:
1046 self.try_parse(instring, loc, do_actions=do_actions)
1047 except (ParseException, IndexError):
1048 return False
1049 else:
1050 return True
1052 # cache for left-recursion in Forward references
1053 recursion_lock = RLock()
1054 recursion_memos: collections.abc.MutableMapping[
1055 tuple[int, Forward, bool], tuple[int, Union[ParseResults, Exception]]
1056 ] = {}
1058 class _CacheType(typing.Protocol):
1059 """
1060 Class to be used for packrat and left-recursion cacheing of results
1061 and exceptions.
1062 """
1064 not_in_cache: bool
1066 def get(self, *args) -> typing.Any: ...
1068 def set(self, *args) -> None: ...
1070 def clear(self) -> None: ...
1072 class NullCache(dict):
1073 """
1074 A null cache type for initialization of the packrat_cache class variable.
1075 If/when enable_packrat() is called, this null cache will be replaced by a
1076 proper _CacheType class instance.
1077 """
1079 not_in_cache: bool = True
1081 def get(self, *args) -> typing.Any: ...
1083 def set(self, *args) -> None: ...
1085 def clear(self) -> None: ...
1087 # class-level argument cache for optimizing repeated calls when backtracking
1088 # through recursive expressions
1089 packrat_cache: _CacheType = NullCache()
1090 packrat_cache_lock = RLock()
1091 packrat_cache_stats = [0, 0]
1093 # this method gets repeatedly called during backtracking with the same arguments -
1094 # we can cache these arguments and save ourselves the trouble of re-parsing the contained expression
1095 def _parseCache(
1096 self, instring, loc, do_actions=True, callPreParse=True
1097 ) -> tuple[int, ParseResults]:
1098 HIT, MISS = 0, 1
1099 lookup = (self, instring, loc, callPreParse, do_actions)
1100 with ParserElement.packrat_cache_lock:
1101 cache = ParserElement.packrat_cache
1102 value = cache.get(lookup)
1103 if value is cache.not_in_cache:
1104 ParserElement.packrat_cache_stats[MISS] += 1
1105 try:
1106 value = self._parseNoCache(instring, loc, do_actions, callPreParse)
1107 except ParseBaseException as pe:
1108 # cache a copy of the exception, without the traceback
1109 cache.set(lookup, pe.__class__(*pe.args))
1110 raise
1111 else:
1112 cache.set(lookup, (value[0], value[1].copy(), loc))
1113 return value
1114 else:
1115 ParserElement.packrat_cache_stats[HIT] += 1
1116 if self.debug and self.debugActions.debug_try:
1117 try:
1118 self.debugActions.debug_try(instring, loc, self, cache_hit=True) # type: ignore [call-arg]
1119 except TypeError:
1120 pass
1121 if isinstance(value, Exception):
1122 if self.debug and self.debugActions.debug_fail:
1123 try:
1124 self.debugActions.debug_fail(
1125 instring, loc, self, value, cache_hit=True # type: ignore [call-arg]
1126 )
1127 except TypeError:
1128 pass
1129 raise value
1131 value = cast(tuple[int, ParseResults, int], value)
1132 loc_, result, endloc = value[0], value[1].copy(), value[2]
1133 if self.debug and self.debugActions.debug_match:
1134 try:
1135 self.debugActions.debug_match(
1136 instring, loc_, endloc, self, result, cache_hit=True # type: ignore [call-arg]
1137 )
1138 except TypeError:
1139 pass
1141 return loc_, result
1143 _parse = _parseNoCache
1145 @staticmethod
1146 def reset_cache() -> None:
1147 """
1148 Clears caches used by packrat and left-recursion.
1149 """
1150 with ParserElement.packrat_cache_lock:
1151 ParserElement.packrat_cache.clear()
1152 ParserElement.packrat_cache_stats[:] = [0] * len(
1153 ParserElement.packrat_cache_stats
1154 )
1155 ParserElement.recursion_memos.clear()
1157 # class attributes to keep caching status
1158 _packratEnabled = False
1159 _left_recursion_enabled = False
1161 @staticmethod
1162 def disable_memoization() -> None:
1163 """
1164 Disables active Packrat or Left Recursion parsing and their memoization
1166 This method also works if neither Packrat nor Left Recursion are enabled.
1167 This makes it safe to call before activating Packrat nor Left Recursion
1168 to clear any previous settings.
1169 """
1170 with ParserElement.packrat_cache_lock:
1171 ParserElement.reset_cache()
1172 ParserElement._left_recursion_enabled = False
1173 ParserElement._packratEnabled = False
1174 ParserElement._parse = ParserElement._parseNoCache
1176 @staticmethod
1177 def enable_left_recursion(
1178 cache_size_limit: typing.Optional[int] = None, *, force=False
1179 ) -> None:
1180 """
1181 Enables "bounded recursion" parsing, which allows for both direct and indirect
1182 left-recursion. During parsing, left-recursive :class:`Forward` elements are
1183 repeatedly matched with a fixed recursion depth that is gradually increased
1184 until finding the longest match.
1186 Example:
1188 .. testcode::
1190 import pyparsing as pp
1191 pp.ParserElement.enable_left_recursion()
1193 E = pp.Forward("E")
1194 num = pp.Word(pp.nums)
1196 # match `num`, or `num '+' num`, or `num '+' num '+' num`, ...
1197 E <<= E + '+' - num | num
1199 print(E.parse_string("1+2+3+4"))
1201 prints:
1203 .. testoutput::
1205 ['1', '+', '2', '+', '3', '+', '4']
1207 Recursion search naturally memoizes matches of ``Forward`` elements and may
1208 thus skip reevaluation of parse actions during backtracking. This may break
1209 programs with parse actions which rely on strict ordering of side-effects.
1211 Parameters:
1213 - ``cache_size_limit`` - (default=``None``) - memoize at most this many
1214 ``Forward`` elements during matching; if ``None`` (the default),
1215 memoize all ``Forward`` elements.
1217 Bounded Recursion parsing works similar but not identical to Packrat parsing,
1218 thus the two cannot be used together. Use ``force=True`` to disable any
1219 previous, conflicting settings.
1220 """
1221 with ParserElement.packrat_cache_lock:
1222 if force:
1223 ParserElement.disable_memoization()
1224 elif ParserElement._packratEnabled:
1225 raise RuntimeError("Packrat and Bounded Recursion are not compatible")
1226 if cache_size_limit is None:
1227 ParserElement.recursion_memos = _UnboundedMemo()
1228 elif cache_size_limit > 0:
1229 ParserElement.recursion_memos = _LRUMemo(capacity=cache_size_limit) # type: ignore[assignment]
1230 else:
1231 raise NotImplementedError(f"Memo size of {cache_size_limit}")
1232 ParserElement._left_recursion_enabled = True
1234 @staticmethod
1235 def enable_packrat(
1236 cache_size_limit: Union[int, None] = 128, *, force: bool = False
1237 ) -> None:
1238 """
1239 Enables "packrat" parsing, which adds memoizing to the parsing logic.
1240 Repeated parse attempts at the same string location (which happens
1241 often in many complex grammars) can immediately return a cached value,
1242 instead of re-executing parsing/validating code. Memoizing is done of
1243 both valid results and parsing exceptions.
1245 Parameters:
1247 - ``cache_size_limit`` - (default= ``128``) - if an integer value is provided
1248 will limit the size of the packrat cache; if None is passed, then
1249 the cache size will be unbounded; if 0 is passed, the cache will
1250 be effectively disabled.
1252 This speedup may break existing programs that use parse actions that
1253 have side-effects. For this reason, packrat parsing is disabled when
1254 you first import pyparsing. To activate the packrat feature, your
1255 program must call the class method :class:`ParserElement.enable_packrat`.
1256 For best results, call ``enable_packrat()`` immediately after
1257 importing pyparsing.
1259 .. Can't really be doctested, alas
1261 Example::
1263 import pyparsing
1264 pyparsing.ParserElement.enable_packrat()
1266 Packrat parsing works similar but not identical to Bounded Recursion parsing,
1267 thus the two cannot be used together. Use ``force=True`` to disable any
1268 previous, conflicting settings.
1269 """
1270 with ParserElement.packrat_cache_lock:
1271 if force:
1272 ParserElement.disable_memoization()
1273 elif ParserElement._left_recursion_enabled:
1274 raise RuntimeError("Packrat and Bounded Recursion are not compatible")
1276 if ParserElement._packratEnabled:
1277 return
1279 ParserElement._packratEnabled = True
1280 if cache_size_limit is None:
1281 ParserElement.packrat_cache = _UnboundedCache()
1282 else:
1283 ParserElement.packrat_cache = _FifoCache(cache_size_limit)
1284 ParserElement._parse = ParserElement._parseCache
1286 def parse_string(
1287 self, instring: str, parse_all: bool = False, **kwargs
1288 ) -> ParseResults:
1289 """
1290 Parse a string with respect to the parser definition. This function is intended as the primary interface to the
1291 client code.
1293 :param instring: The input string to be parsed.
1294 :param parse_all: If set, the entire input string must match the grammar.
1295 :param parseAll: retained for pre-PEP8 compatibility, will be removed in a future release.
1296 :raises ParseException: Raised if ``parse_all`` is set and the input string does not match the whole grammar.
1297 :returns: the parsed data as a :class:`ParseResults` object, which may be accessed as a `list`, a `dict`, or
1298 an object with attributes if the given parser includes results names.
1300 If the input string is required to match the entire grammar, ``parse_all`` flag must be set to ``True``. This
1301 is also equivalent to ending the grammar with :class:`StringEnd`\\ ().
1303 To report proper column numbers, ``parse_string`` operates on a copy of the input string where all tabs are
1304 converted to spaces (8 spaces per tab, as per the default in ``string.expandtabs``). If the input string
1305 contains tabs and the grammar uses parse actions that use the ``loc`` argument to index into the string
1306 being parsed, one can ensure a consistent view of the input string by doing one of the following:
1308 - calling ``parse_with_tabs`` on your grammar before calling ``parse_string`` (see :class:`parse_with_tabs`),
1309 - define your parse action using the full ``(s,loc,toks)`` signature, and reference the input string using the
1310 parse action's ``s`` argument, or
1311 - explicitly expand the tabs in your input string before calling ``parse_string``.
1313 Examples:
1315 By default, partial matches are OK.
1317 .. doctest::
1319 >>> res = Word('a').parse_string('aaaaabaaa')
1320 >>> print(res)
1321 ['aaaaa']
1323 The parsing behavior varies by the inheriting class of this abstract class. Please refer to the children
1324 directly to see more examples.
1326 It raises an exception if parse_all flag is set and instring does not match the whole grammar.
1328 .. doctest::
1330 >>> res = Word('a').parse_string('aaaaabaaa', parse_all=True)
1331 Traceback (most recent call last):
1332 ParseException: Expected end of text, found 'b' ...
1333 """
1334 parseAll: bool = deprecate_argument(kwargs, "parseAll", False)
1336 parse_all = parse_all or parseAll
1338 ParserElement.reset_cache()
1339 if not self.streamlined:
1340 self.streamline()
1341 for e in self.ignoreExprs:
1342 e.streamline()
1343 if not self.keepTabs:
1344 instring = instring.expandtabs()
1345 try:
1346 loc, tokens = self._parse(instring, 0)
1347 if parse_all:
1348 loc = self.preParse(instring, loc)
1349 se = Empty() + StringEnd().set_debug(False)
1350 se._parse(instring, loc)
1351 except _ParseActionIndexError as pa_exc:
1352 raise pa_exc.exc
1353 except ParseBaseException as exc:
1354 if ParserElement.verbose_stacktrace:
1355 raise
1357 # catch and re-raise exception from here, clearing out pyparsing internal stack trace
1358 raise exc.with_traceback(None)
1359 else:
1360 return tokens
1362 def scan_string(
1363 self,
1364 instring: str,
1365 max_matches: int = _MAX_INT,
1366 overlap: bool = False,
1367 always_skip_whitespace=True,
1368 *,
1369 debug: bool = False,
1370 **kwargs,
1371 ) -> Generator[tuple[ParseResults, int, int], None, None]:
1372 """
1373 Scan the input string for expression matches. Each match will return the
1374 matching tokens, start location, and end location. May be called with optional
1375 ``max_matches`` argument, to clip scanning after 'n' matches are found. If
1376 ``overlap`` is specified, then overlapping matches will be reported.
1378 Note that the start and end locations are reported relative to the string
1379 being parsed. See :class:`parse_string` for more information on parsing
1380 strings with embedded tabs.
1382 Example:
1384 .. testcode::
1386 source = "sldjf123lsdjjkf345sldkjf879lkjsfd987"
1387 print(source)
1388 for tokens, start, end in Word(alphas).scan_string(source):
1389 print(' '*start + '^'*(end-start))
1390 print(' '*start + tokens[0])
1392 prints:
1394 .. testoutput::
1396 sldjf123lsdjjkf345sldkjf879lkjsfd987
1397 ^^^^^
1398 sldjf
1399 ^^^^^^^
1400 lsdjjkf
1401 ^^^^^^
1402 sldkjf
1403 ^^^^^^
1404 lkjsfd
1405 """
1406 maxMatches: int = deprecate_argument(kwargs, "maxMatches", _MAX_INT)
1408 max_matches = min(maxMatches, max_matches)
1409 if not self.streamlined:
1410 self.streamline()
1411 for e in self.ignoreExprs:
1412 e.streamline()
1414 if not self.keepTabs:
1415 instring = str(instring).expandtabs()
1416 instrlen = len(instring)
1417 loc = 0
1418 if always_skip_whitespace:
1419 preparser = Empty()
1420 preparser.ignoreExprs = self.ignoreExprs
1421 preparser.whiteChars = self.whiteChars
1422 preparseFn = preparser.preParse
1423 else:
1424 preparseFn = self.preParse
1425 parseFn = self._parse
1426 ParserElement.reset_cache()
1427 matches = 0
1428 try:
1429 while loc <= instrlen and matches < max_matches:
1430 try:
1431 preloc: int = preparseFn(instring, loc)
1432 nextLoc: int
1433 tokens: ParseResults
1434 nextLoc, tokens = parseFn(instring, preloc, callPreParse=False)
1435 except ParseException:
1436 loc = preloc + 1
1437 else:
1438 if nextLoc > loc:
1439 matches += 1
1440 if debug:
1441 print(
1442 {
1443 "tokens": tokens.as_list(),
1444 "start": preloc,
1445 "end": nextLoc,
1446 }
1447 )
1448 yield tokens, preloc, nextLoc
1449 if overlap:
1450 nextloc = preparseFn(instring, loc)
1451 if nextloc > loc:
1452 loc = nextLoc
1453 else:
1454 loc += 1
1455 else:
1456 loc = nextLoc
1457 else:
1458 loc = preloc + 1
1459 except ParseBaseException as exc:
1460 if ParserElement.verbose_stacktrace:
1461 raise
1463 # catch and re-raise exception from here, clears out pyparsing internal stack trace
1464 raise exc.with_traceback(None)
1466 def transform_string(self, instring: str, *, debug: bool = False) -> str:
1467 """
1468 Extension to :class:`scan_string`, to modify matching text with modified tokens that may
1469 be returned from a parse action. To use ``transform_string``, define a grammar and
1470 attach a parse action to it that modifies the returned token list.
1471 Invoking ``transform_string()`` on a target string will then scan for matches,
1472 and replace the matched text patterns according to the logic in the parse
1473 action. ``transform_string()`` returns the resulting transformed string.
1475 Example:
1477 .. testcode::
1479 quote = '''now is the winter of our discontent,
1480 made glorious summer by this sun of york.'''
1482 wd = Word(alphas)
1483 wd.set_parse_action(lambda toks: toks[0].title())
1485 print(wd.transform_string(quote))
1487 prints:
1489 .. testoutput::
1491 Now Is The Winter Of Our Discontent,
1492 Made Glorious Summer By This Sun Of York.
1493 """
1494 out: list[str] = []
1495 lastE = 0
1496 # force preservation of <TAB>s, to minimize unwanted transformation of string, and to
1497 # keep string locs straight between transform_string and scan_string
1498 self.keepTabs = True
1499 try:
1500 for t, s, e in self.scan_string(instring, debug=debug):
1501 if s > lastE:
1502 out.append(instring[lastE:s])
1503 lastE = e
1505 if not t:
1506 continue
1508 if isinstance(t, ParseResults):
1509 out += t.as_list()
1510 elif isinstance(t, Iterable) and not isinstance(t, str_type):
1511 out.extend(t)
1512 else:
1513 out.append(t)
1515 out.append(instring[lastE:])
1516 out = [o for o in out if o]
1517 return "".join([str(s) for s in _flatten(out)])
1518 except ParseBaseException as exc:
1519 if ParserElement.verbose_stacktrace:
1520 raise
1522 # catch and re-raise exception from here, clears out pyparsing internal stack trace
1523 raise exc.with_traceback(None)
1525 def search_string(
1526 self,
1527 instring: str,
1528 max_matches: int = _MAX_INT,
1529 *,
1530 debug: bool = False,
1531 **kwargs,
1532 ) -> ParseResults:
1533 """
1534 Another extension to :class:`scan_string`, simplifying the access to the tokens found
1535 to match the given parse expression. May be called with optional
1536 ``max_matches`` argument, to clip searching after 'n' matches are found.
1538 Example:
1540 .. testcode::
1542 quote = '''More than Iron, more than Lead,
1543 more than Gold I need Electricity'''
1545 # a capitalized word starts with an uppercase letter,
1546 # followed by zero or more lowercase letters
1547 cap_word = Word(alphas.upper(), alphas.lower())
1549 print(cap_word.search_string(quote))
1551 # the sum() builtin can be used to merge results
1552 # into a single ParseResults object
1553 print(sum(cap_word.search_string(quote)))
1555 prints:
1557 .. testoutput::
1559 [['More'], ['Iron'], ['Lead'], ['Gold'], ['I'], ['Electricity']]
1560 ['More', 'Iron', 'Lead', 'Gold', 'I', 'Electricity']
1561 """
1562 maxMatches: int = deprecate_argument(kwargs, "maxMatches", _MAX_INT)
1564 max_matches = min(maxMatches, max_matches)
1565 try:
1566 return ParseResults(
1567 [
1568 t
1569 for t, s, e in self.scan_string(
1570 instring,
1571 max_matches=max_matches,
1572 always_skip_whitespace=False,
1573 debug=debug,
1574 )
1575 ]
1576 )
1577 except ParseBaseException as exc:
1578 if ParserElement.verbose_stacktrace:
1579 raise
1581 # catch and re-raise exception from here, clears out pyparsing internal stack trace
1582 raise exc.with_traceback(None)
1584 def split(
1585 self,
1586 instring: str,
1587 maxsplit: int = _MAX_INT,
1588 include_separators: bool = False,
1589 **kwargs,
1590 ) -> Generator[str, None, None]:
1591 """
1592 Generator method to split a string using the given expression as a separator.
1593 May be called with optional ``maxsplit`` argument, to limit the number of splits;
1594 and the optional ``include_separators`` argument (default= ``False``), if the separating
1595 matching text should be included in the split results.
1597 Example:
1599 .. testcode::
1601 punc = one_of(list(".,;:/-!?"))
1602 print(list(punc.split(
1603 "This, this?, this sentence, is badly punctuated!")))
1605 prints:
1607 .. testoutput::
1609 ['This', ' this', '', ' this sentence', ' is badly punctuated', '']
1610 """
1611 includeSeparators: bool = deprecate_argument(kwargs, "includeSeparators", False)
1613 include_separators = includeSeparators or include_separators
1614 last = 0
1615 for t, s, e in self.scan_string(instring, max_matches=maxsplit):
1616 yield instring[last:s]
1617 if include_separators:
1618 yield t[0]
1619 last = e
1620 yield instring[last:]
1622 def __add__(self, other) -> ParserElement:
1623 """
1624 Implementation of ``+`` operator - returns :class:`And`. Adding strings to a :class:`ParserElement`
1625 converts them to :class:`Literal`\\ s by default.
1627 Example:
1629 .. testcode::
1631 greet = Word(alphas) + "," + Word(alphas) + "!"
1632 hello = "Hello, World!"
1633 print(hello, "->", greet.parse_string(hello))
1635 prints:
1637 .. testoutput::
1639 Hello, World! -> ['Hello', ',', 'World', '!']
1641 ``...`` may be used as a parse expression as a short form of :class:`SkipTo`:
1643 .. testcode::
1645 Literal('start') + ... + Literal('end')
1647 is equivalent to:
1649 .. testcode::
1651 Literal('start') + SkipTo('end')("_skipped*") + Literal('end')
1653 Note that the skipped text is returned with '_skipped' as a results name,
1654 and to support having multiple skips in the same parser, the value returned is
1655 a list of all skipped text.
1656 """
1657 if other is Ellipsis:
1658 return _PendingSkip(self)
1660 if isinstance(other, str_type):
1661 other = self._literalStringClass(other)
1662 if not isinstance(other, ParserElement):
1663 return NotImplemented
1664 return And([self, other])
1666 def __radd__(self, other) -> ParserElement:
1667 """
1668 Implementation of ``+`` operator when left operand is not a :class:`ParserElement`
1669 """
1670 if other is Ellipsis:
1671 return SkipTo(self)("_skipped*") + self
1673 if isinstance(other, str_type):
1674 other = self._literalStringClass(other)
1675 if not isinstance(other, ParserElement):
1676 return NotImplemented
1677 return other + self
1679 def __sub__(self, other) -> ParserElement:
1680 """
1681 Implementation of ``-`` operator, returns :class:`And` with error stop
1682 """
1683 if isinstance(other, str_type):
1684 other = self._literalStringClass(other)
1685 if not isinstance(other, ParserElement):
1686 return NotImplemented
1687 return self + And._ErrorStop() + other
1689 def __rsub__(self, other) -> ParserElement:
1690 """
1691 Implementation of ``-`` operator when left operand is not a :class:`ParserElement`
1692 """
1693 if isinstance(other, str_type):
1694 other = self._literalStringClass(other)
1695 if not isinstance(other, ParserElement):
1696 return NotImplemented
1697 return other - self
1699 def __mul__(self, other) -> ParserElement:
1700 """
1701 Implementation of ``*`` operator, allows use of ``expr * 3`` in place of
1702 ``expr + expr + expr``. Expressions may also be multiplied by a 2-integer
1703 tuple, similar to ``{min, max}`` multipliers in regular expressions. Tuples
1704 may also include ``None`` as in:
1706 - ``expr*(n, None)`` or ``expr*(n, )`` is equivalent
1707 to ``expr*n + ZeroOrMore(expr)``
1708 (read as "at least n instances of ``expr``")
1709 - ``expr*(None, n)`` is equivalent to ``expr*(0, n)``
1710 (read as "0 to n instances of ``expr``")
1711 - ``expr*(None, None)`` is equivalent to ``ZeroOrMore(expr)``
1712 - ``expr*(1, None)`` is equivalent to ``OneOrMore(expr)``
1714 Note that ``expr*(None, n)`` does not raise an exception if
1715 more than n exprs exist in the input stream; that is,
1716 ``expr*(None, n)`` does not enforce a maximum number of expr
1717 occurrences. If this behavior is desired, then write
1718 ``expr*(None, n) + ~expr``
1719 """
1720 if other is Ellipsis:
1721 other = (0, None)
1722 elif isinstance(other, tuple) and other[:1] == (Ellipsis,):
1723 other = (0, *other[1:2], None)[:2]
1725 if not isinstance(other, (int, tuple)):
1726 return NotImplemented
1728 if isinstance(other, int):
1729 minElements, optElements = other, 0
1730 else:
1731 other = tuple(o if o is not Ellipsis else None for o in other)
1732 other = (*other, None, None)[:2]
1733 if other[0] is None:
1734 other = (0, other[1])
1735 if isinstance(other[0], int) and other[1] is None:
1736 if other[0] == 0:
1737 return ZeroOrMore(self)
1738 if other[0] == 1:
1739 return OneOrMore(self)
1740 else:
1741 return self * other[0] + ZeroOrMore(self)
1742 elif isinstance(other[0], int) and isinstance(other[1], int):
1743 minElements, optElements = other
1744 optElements -= minElements
1745 else:
1746 return NotImplemented
1748 if minElements < 0:
1749 raise ValueError("cannot multiply ParserElement by negative value")
1750 if optElements < 0:
1751 raise ValueError(
1752 "second tuple value must be greater or equal to first tuple value"
1753 )
1754 if minElements == optElements == 0:
1755 return And([])
1757 if optElements:
1758 # Build the optional tail as a bounded ``ZeroOrMore`` instead of a
1759 # deeply nested ``Opt(self + Opt(self + ...))`` chain. The nested
1760 # form recursed ``optElements`` levels deep, which raised
1761 # RecursionError for large upper bounds (e.g. ``expr[..., 1000]``)
1762 # -- see issue #332. ``ZeroOrMore(..., max=optElements)`` is a flat
1763 # loop that still *exits early* (it stops at the first non-match,
1764 # just like the recursive form), so it preserves the original
1765 # early-exit behavior while no longer scaling the call stack with
1766 # the upper bound.
1767 optionalTail = ZeroOrMore(self, max=optElements)
1769 if minElements:
1770 if minElements == 1:
1771 ret = self + optionalTail
1772 else:
1773 ret = And([self] * minElements) + optionalTail
1774 else:
1775 ret = optionalTail
1776 else:
1777 if minElements == 1:
1778 ret = self
1779 else:
1780 ret = And([self] * minElements)
1781 return ret
1783 def __rmul__(self, other) -> ParserElement:
1784 return self.__mul__(other)
1786 def __or__(self, other) -> ParserElement:
1787 """
1788 Implementation of ``|`` operator - returns :class:`MatchFirst`
1790 .. versionchanged:: 3.1.0
1791 Support ``expr | ""`` as a synonym for ``Optional(expr)``.
1792 """
1793 if other is Ellipsis:
1794 return _PendingSkip(self, must_skip=True)
1796 if isinstance(other, str_type):
1797 # `expr | ""` is equivalent to `Opt(expr)`
1798 if other == "":
1799 return Opt(self)
1800 other = self._literalStringClass(other)
1801 if not isinstance(other, ParserElement):
1802 return NotImplemented
1803 return MatchFirst([self, other])
1805 def __ror__(self, other) -> ParserElement:
1806 """
1807 Implementation of ``|`` operator when left operand is not a :class:`ParserElement`
1808 """
1809 if isinstance(other, str_type):
1810 other = self._literalStringClass(other)
1811 if not isinstance(other, ParserElement):
1812 return NotImplemented
1813 return other | self
1815 def __xor__(self, other) -> ParserElement:
1816 """
1817 Implementation of ``^`` operator - returns :class:`Or`
1818 """
1819 if isinstance(other, str_type):
1820 other = self._literalStringClass(other)
1821 if not isinstance(other, ParserElement):
1822 return NotImplemented
1823 return Or([self, other])
1825 def __rxor__(self, other) -> ParserElement:
1826 """
1827 Implementation of ``^`` operator when left operand is not a :class:`ParserElement`
1828 """
1829 if isinstance(other, str_type):
1830 other = self._literalStringClass(other)
1831 if not isinstance(other, ParserElement):
1832 return NotImplemented
1833 return other ^ self
1835 def __and__(self, other) -> ParserElement:
1836 """
1837 Implementation of ``&`` operator - returns :class:`Each`
1838 """
1839 if isinstance(other, str_type):
1840 other = self._literalStringClass(other)
1841 if not isinstance(other, ParserElement):
1842 return NotImplemented
1843 return Each([self, other])
1845 def __rand__(self, other) -> ParserElement:
1846 """
1847 Implementation of ``&`` operator when left operand is not a :class:`ParserElement`
1848 """
1849 if isinstance(other, str_type):
1850 other = self._literalStringClass(other)
1851 if not isinstance(other, ParserElement):
1852 return NotImplemented
1853 return other & self
1855 def __invert__(self) -> ParserElement:
1856 """
1857 Implementation of ``~`` operator - returns :class:`NotAny`
1858 """
1859 return NotAny(self)
1861 # disable __iter__ to override legacy use of sequential access to __getitem__ to
1862 # iterate over a sequence
1863 __iter__ = None
1865 def __getitem__(self, key):
1866 """
1867 use ``[]`` indexing notation as a short form for expression repetition:
1869 - ``expr[n]`` is equivalent to ``expr*n``
1870 - ``expr[m, n]`` is equivalent to ``expr*(m, n)``
1871 - ``expr[n, ...]`` or ``expr[n,]`` is equivalent
1872 to ``expr*n + ZeroOrMore(expr)``
1873 (read as "at least n instances of ``expr``")
1874 - ``expr[..., n]`` is equivalent to ``expr*(0, n)``
1875 (read as "0 to n instances of ``expr``")
1876 - ``expr[...]`` and ``expr[0, ...]`` are equivalent to ``ZeroOrMore(expr)``
1877 - ``expr[1, ...]`` is equivalent to ``OneOrMore(expr)``
1879 ``None`` may be used in place of ``...``.
1881 Note that ``expr[..., n]`` and ``expr[m, n]`` do not raise an exception
1882 if more than ``n`` ``expr``\\ s exist in the input stream. If this behavior is
1883 desired, then write ``expr[..., n] + ~expr``.
1885 For repetition with a stop_on expression, use slice notation:
1887 - ``expr[...: end_expr]`` and ``expr[0, ...: end_expr]`` are equivalent to ``ZeroOrMore(expr, stop_on=end_expr)``
1888 - ``expr[1, ...: end_expr]`` is equivalent to ``OneOrMore(expr, stop_on=end_expr)``
1890 .. versionchanged:: 3.1.0
1891 Support for slice notation.
1892 """
1894 stop_on_defined = False
1895 stop_on = NoMatch()
1896 if isinstance(key, slice):
1897 key, stop_on = key.start, key.stop
1898 if key is None:
1899 key = ...
1900 stop_on_defined = True
1901 elif isinstance(key, tuple) and isinstance(key[-1], slice):
1902 key, stop_on = (key[0], key[1].start), key[1].stop
1903 stop_on_defined = True
1905 # convert single arg keys to tuples
1906 if isinstance(key, str_type):
1907 key = (key,)
1908 try:
1909 iter(key)
1910 except TypeError:
1911 key = (key, key)
1913 if len(key) > 2:
1914 raise TypeError(
1915 f"only 1 or 2 index arguments supported ({key[:5]}{f'... [{len(key)}]' if len(key) > 5 else ''})"
1916 )
1918 # clip to 2 elements
1919 ret = self * tuple(key[:2])
1920 ret = typing.cast(_MultipleMatch, ret)
1922 if stop_on_defined:
1923 ret.stopOn(stop_on)
1925 return ret
1927 def __call__(self, name: typing.Optional[str] = None) -> ParserElement:
1928 """
1929 Shortcut for :class:`set_results_name`, with ``list_all_matches=False``.
1931 If ``name`` is given with a trailing ``'*'`` character, then ``list_all_matches`` will be
1932 passed as ``True``.
1934 If ``name`` is omitted, same as calling :class:`copy`.
1936 Example:
1938 .. testcode::
1940 # these are equivalent
1941 userdata = (
1942 Word(alphas).set_results_name("name")
1943 + Word(nums + "-").set_results_name("socsecno")
1944 )
1946 userdata = Word(alphas)("name") + Word(nums + "-")("socsecno")
1947 """
1948 if name is not None:
1949 return self._setResultsName(name)
1951 return self.copy()
1953 def suppress(self) -> ParserElement:
1954 """
1955 Suppresses the output of this :class:`ParserElement`; useful to keep punctuation from
1956 cluttering up returned output.
1957 """
1958 return Suppress(self)
1960 def ignore_whitespace(self, recursive: bool = True) -> ParserElement:
1961 """
1962 Enables the skipping of whitespace before matching the characters in the
1963 :class:`ParserElement`'s defined pattern.
1965 :param recursive: If ``True`` (the default), also enable whitespace skipping in child elements (if any)
1966 """
1967 self.skipWhitespace = True
1968 return self
1970 def leave_whitespace(self, recursive: bool = True) -> ParserElement:
1971 """
1972 Disables the skipping of whitespace before matching the characters in the
1973 :class:`ParserElement`'s defined pattern. This is normally only used internally by
1974 the pyparsing module, but may be needed in some whitespace-sensitive grammars.
1976 :param recursive: If true (the default), also disable whitespace skipping in child elements (if any)
1977 """
1978 self.skipWhitespace = False
1979 return self
1981 def set_whitespace_chars(
1982 self, chars: Union[set[str], str], copy_defaults: bool = False
1983 ) -> ParserElement:
1984 """
1985 Overrides the default whitespace chars
1986 """
1987 self.skipWhitespace = True
1988 self.whiteChars = set(chars)
1989 self.copyDefaultWhiteChars = copy_defaults
1990 return self
1992 def parse_with_tabs(self) -> ParserElement:
1993 """
1994 Overrides default behavior to expand ``<TAB>`` s to spaces before parsing the input string.
1995 Must be called before ``parse_string`` when the input grammar contains elements that
1996 match ``<TAB>`` characters.
1997 """
1998 self.keepTabs = True
1999 return self
2001 def ignore(self, other: ParserElement) -> ParserElement:
2002 """
2003 Define expression to be ignored (e.g., comments) while doing pattern
2004 matching; may be called repeatedly, to define multiple comment or other
2005 ignorable patterns.
2007 Example:
2009 .. doctest::
2011 >>> patt = Word(alphas)[...]
2012 >>> print(patt.parse_string('ablaj /* comment */ lskjd'))
2013 ['ablaj']
2015 >>> patt = Word(alphas)[...].ignore(c_style_comment)
2016 >>> print(patt.parse_string('ablaj /* comment */ lskjd'))
2017 ['ablaj', 'lskjd']
2018 """
2019 if isinstance(other, str_type):
2020 other = Suppress(other)
2022 if isinstance(other, Suppress):
2023 if other not in self.ignoreExprs:
2024 self.ignoreExprs.append(other)
2025 else:
2026 self.ignoreExprs.append(Suppress(other.copy()))
2027 return self
2029 def set_debug_actions(
2030 self,
2031 start_action: DebugStartAction,
2032 success_action: DebugSuccessAction,
2033 exception_action: DebugExceptionAction,
2034 ) -> ParserElement:
2035 """
2036 Customize display of debugging messages while doing pattern matching:
2038 :param start_action: method to be called when an expression is about to be parsed;
2039 should have the signature::
2041 fn(input_string: str,
2042 location: int,
2043 expression: ParserElement,
2044 cache_hit: bool)
2046 :param success_action: method to be called when an expression has successfully parsed;
2047 should have the signature::
2049 fn(input_string: str,
2050 start_location: int,
2051 end_location: int,
2052 expression: ParserELement,
2053 parsed_tokens: ParseResults,
2054 cache_hit: bool)
2056 :param exception_action: method to be called when expression fails to parse;
2057 should have the signature::
2059 fn(input_string: str,
2060 location: int,
2061 expression: ParserElement,
2062 exception: Exception,
2063 cache_hit: bool)
2064 """
2065 self.debugActions = self.DebugActions(
2066 start_action or _default_start_debug_action, # type: ignore[truthy-function]
2067 success_action or _default_success_debug_action, # type: ignore[truthy-function]
2068 exception_action or _default_exception_debug_action, # type: ignore[truthy-function]
2069 )
2070 self.debug = any(self.debugActions)
2071 return self
2073 def set_debug(self, flag: bool = True, recurse: bool = False) -> ParserElement:
2074 """
2075 Enable display of debugging messages while doing pattern matching.
2076 Set ``flag`` to ``True`` to enable, ``False`` to disable.
2077 Set ``recurse`` to ``True`` to set the debug flag on this expression and all sub-expressions.
2079 Example:
2081 .. testcode::
2083 wd = Word(alphas).set_name("alphaword")
2084 integer = Word(nums).set_name("numword")
2085 term = wd | integer
2087 # turn on debugging for wd
2088 wd.set_debug()
2090 term[1, ...].parse_string("abc 123 xyz 890")
2092 prints:
2094 .. testoutput::
2095 :options: +NORMALIZE_WHITESPACE
2097 Match alphaword at loc 0(1,1)
2098 abc 123 xyz 890
2099 ^
2100 Matched alphaword -> ['abc']
2101 Match alphaword at loc 4(1,5)
2102 abc 123 xyz 890
2103 ^
2104 Match alphaword failed, ParseException raised: Expected alphaword, ...
2105 Match alphaword at loc 8(1,9)
2106 abc 123 xyz 890
2107 ^
2108 Matched alphaword -> ['xyz']
2109 Match alphaword at loc 12(1,13)
2110 abc 123 xyz 890
2111 ^
2112 Match alphaword failed, ParseException raised: Expected alphaword, ...
2113 abc 123 xyz 890
2114 ^
2115 Match alphaword failed, ParseException raised: Expected alphaword, found end of text ...
2117 The output shown is that produced by the default debug actions - custom debug actions can be
2118 specified using :meth:`set_debug_actions`. Prior to attempting
2119 to match the ``wd`` expression, the debugging message ``"Match <exprname> at loc <n>(<line>,<col>)"``
2120 is shown. Then if the parse succeeds, a ``"Matched"`` message is shown, or an ``"Exception raised"``
2121 message is shown. Also note the use of :meth:`set_name` to assign a human-readable name to the expression,
2122 which makes debugging and exception messages easier to understand - for instance, the default
2123 name created for the :class:`Word` expression without calling :meth:`set_name` is ``"W:(A-Za-z)"``.
2125 .. versionchanged:: 3.1.0
2126 ``recurse`` argument added.
2127 """
2128 if recurse:
2129 for expr in self.visit_all():
2130 expr.set_debug(flag, recurse=False)
2131 return self
2133 if flag:
2134 self.set_debug_actions(
2135 _default_start_debug_action,
2136 _default_success_debug_action,
2137 _default_exception_debug_action,
2138 )
2139 else:
2140 self.debug = False
2141 return self
2143 @property
2144 def default_name(self) -> str:
2145 if self._defaultName is None:
2146 self._defaultName = self._generateDefaultName()
2147 return self._defaultName
2149 @abstractmethod
2150 def _generateDefaultName(self) -> str:
2151 """
2152 Child classes must define this method, which defines how the ``default_name`` is set.
2153 """
2155 def set_name(self, name: typing.Optional[str]) -> ParserElement:
2156 """
2157 Define name for this expression, makes debugging and exception messages clearer. If
2158 `__diag__.enable_debug_on_named_expressions` is set to True, setting a name will also
2159 enable debug for this expression.
2161 If `name` is None, clears any custom name for this expression, and clears the
2162 debug flag is it was enabled via `__diag__.enable_debug_on_named_expressions`.
2164 Example:
2166 .. doctest::
2168 >>> integer = Word(nums)
2169 >>> integer.parse_string("ABC")
2170 Traceback (most recent call last):
2171 ParseException: Expected W:(0-9) (at char 0), (line:1, col:1)
2173 >>> integer.set_name("integer")
2174 integer
2175 >>> integer.parse_string("ABC")
2176 Traceback (most recent call last):
2177 ParseException: Expected integer (at char 0), (line:1, col:1)
2179 .. versionchanged:: 3.1.0
2180 Accept ``None`` as the ``name`` argument.
2181 """
2182 self.customName = name # type: ignore[assignment]
2183 self.errmsg = f"Expected {str(self)}"
2185 if __diag__.enable_debug_on_named_expressions:
2186 self.set_debug(name is not None)
2188 return self
2190 @property
2191 def name(self) -> str:
2192 """
2193 Returns a user-defined name if available, but otherwise defaults back to the auto-generated name
2194 """
2195 return self.customName if self.customName is not None else self.default_name
2197 @name.setter
2198 def name(self, new_name) -> None:
2199 self.set_name(new_name)
2201 def __str__(self) -> str:
2202 return self.name
2204 def __repr__(self) -> str:
2205 return str(self)
2207 def streamline(self) -> ParserElement:
2208 self.streamlined = True
2209 self._defaultName = None
2210 return self
2212 def recurse(self) -> list[ParserElement]:
2213 return []
2215 def _checkRecursion(self, parseElementList):
2216 subRecCheckList = parseElementList[:] + [self]
2217 for e in self.recurse():
2218 e._checkRecursion(subRecCheckList)
2220 def validate(self, validateTrace=None) -> None:
2221 """
2222 .. deprecated:: 3.0.0
2223 Do not use to check for left recursion.
2225 Check defined expressions for valid structure, check for infinite recursive definitions.
2227 """
2228 warnings.warn(
2229 "ParserElement.validate() is deprecated, and should not be used to check for left recursion",
2230 PyparsingDeprecationWarning,
2231 stacklevel=2,
2232 )
2233 self._checkRecursion([])
2235 def parse_file(
2236 self,
2237 file_or_filename: Union[str, Path, TextIO],
2238 encoding: str = "utf-8",
2239 parse_all: bool = False,
2240 **kwargs,
2241 ) -> ParseResults:
2242 """
2243 Execute the parse expression on the given file or filename.
2244 If a filename is specified (instead of a file object),
2245 the entire file is opened, read, and closed before parsing.
2246 """
2247 parseAll: bool = deprecate_argument(kwargs, "parseAll", False)
2249 parse_all = parse_all or parseAll
2250 try:
2251 file_or_filename = typing.cast(TextIO, file_or_filename)
2252 file_contents = file_or_filename.read()
2253 except AttributeError:
2254 file_or_filename = typing.cast(str, file_or_filename)
2255 with open(file_or_filename, "r", encoding=encoding) as f:
2256 file_contents = f.read()
2257 try:
2258 return self.parse_string(file_contents, parse_all)
2259 except ParseBaseException as exc:
2260 if ParserElement.verbose_stacktrace:
2261 raise
2263 # catch and re-raise exception from here, clears out pyparsing internal stack trace
2264 raise exc.with_traceback(None)
2266 def __eq__(self, other):
2267 if self is other:
2268 return True
2269 elif isinstance(other, str_type):
2270 return self.matches(other, parse_all=True)
2271 elif isinstance(other, ParserElement):
2272 return vars(self) == vars(other)
2273 return False
2275 def __hash__(self):
2276 return id(self)
2278 def matches(self, test_string: str, parse_all: bool = True, **kwargs) -> bool:
2279 """
2280 Method for quick testing of a parser against a test string. Good for simple
2281 inline microtests of sub expressions while building up larger parser.
2283 :param test_string: to test against this expression for a match
2284 :param parse_all: flag to pass to :meth:`parse_string` when running tests
2286 Example:
2288 .. doctest::
2290 >>> expr = Word(nums)
2291 >>> expr.matches("100")
2292 True
2293 """
2294 parseAll: bool = deprecate_argument(kwargs, "parseAll", True)
2296 parse_all = parse_all and parseAll
2297 try:
2298 self.parse_string(str(test_string), parse_all=parse_all)
2299 return True
2300 except ParseBaseException:
2301 return False
2303 def run_tests(
2304 self,
2305 tests: Union[str, list[str]],
2306 parse_all: bool = True,
2307 comment: typing.Optional[Union[ParserElement, str]] = "#",
2308 full_dump: bool = True,
2309 print_results: bool = True,
2310 failure_tests: bool = False,
2311 post_parse: typing.Optional[
2312 Callable[[str, ParseResults], typing.Optional[str]]
2313 ] = None,
2314 file: typing.Optional[TextIO] = None,
2315 with_line_numbers: bool = False,
2316 *,
2317 parseAll: bool = True,
2318 fullDump: bool = True,
2319 printResults: bool = True,
2320 failureTests: bool = False,
2321 postParse: typing.Optional[
2322 Callable[[str, ParseResults], typing.Optional[str]]
2323 ] = None,
2324 ) -> tuple[bool, list[tuple[str, Union[ParseResults, Exception]]]]:
2325 """
2326 Execute the parse expression on a series of test strings, showing each
2327 test, the parsed results or where the parse failed. Quick and easy way to
2328 run a parse expression against a list of sample strings.
2330 Parameters:
2332 - ``tests`` - a list of separate test strings, or a multiline string of test strings
2333 - ``parse_all`` - (default= ``True``) - flag to pass to :class:`parse_string` when running tests
2334 - ``comment`` - (default= ``'#'``) - expression for indicating embedded comments in the test
2335 string; pass None to disable comment filtering
2336 - ``full_dump`` - (default= ``True``) - dump results as list followed by results names in nested outline;
2337 if False, only dump nested list
2338 - ``print_results`` - (default= ``True``) prints test output to stdout
2339 - ``failure_tests`` - (default= ``False``) indicates if these tests are expected to fail parsing
2340 - ``post_parse`` - (default= ``None``) optional callback for successful parse results; called as
2341 `fn(test_string, parse_results)` and returns a string to be added to the test output
2342 - ``file`` - (default= ``None``) optional file-like object to which test output will be written;
2343 if None, will default to ``sys.stdout``
2344 - ``with_line_numbers`` - default= ``False``) show test strings with line and column numbers
2346 Returns: a (success, results) tuple, where success indicates that all tests succeeded
2347 (or failed if ``failure_tests`` is True), and the results contain a list of lines of each
2348 test's output
2350 Passing example:
2352 .. testcode::
2354 number_expr = pyparsing_common.number.copy()
2356 result = number_expr.run_tests('''
2357 # unsigned integer
2358 100
2359 # negative integer
2360 -100
2361 # float with scientific notation
2362 6.02e23
2363 # integer with scientific notation
2364 1e-12
2365 # negative decimal number without leading digit
2366 -.100
2367 ''')
2368 print("Success" if result[0] else "Failed!")
2370 prints:
2372 .. testoutput::
2373 :options: +NORMALIZE_WHITESPACE
2376 # unsigned integer
2377 100
2378 [100]
2380 # negative integer
2381 -100
2382 [-100]
2384 # float with scientific notation
2385 6.02e23
2386 [6.02e+23]
2388 # integer with scientific notation
2389 1e-12
2390 [1e-12]
2392 # negative decimal number without leading digit
2393 -.100
2394 [-0.1]
2395 Success
2397 Failure-test example:
2399 .. testcode::
2401 result = number_expr.run_tests('''
2402 # stray character
2403 100Z
2404 # too many '.'
2405 3.14.159
2406 ''', failure_tests=True)
2407 print("Success" if result[0] else "Failed!")
2409 prints:
2411 .. testoutput::
2412 :options: +NORMALIZE_WHITESPACE
2415 # stray character
2416 100Z
2417 100Z
2418 ^
2419 ParseException: Expected end of text, found 'Z' ...
2421 # too many '.'
2422 3.14.159
2423 3.14.159
2424 ^
2425 ParseException: Expected end of text, found '.' ...
2426 FAIL: Expected end of text, found '.' ...
2427 Success
2429 Each test string must be on a single line. If you want to test a string that spans multiple
2430 lines, create a test like this:
2432 .. testcode::
2434 expr = Word(alphanums)[1,...]
2435 expr.run_tests(r"this is a test\\n of strings that spans \\n 3 lines")
2437 .. testoutput::
2438 :options: +NORMALIZE_WHITESPACE
2439 :hide:
2442 this is a test\\n of strings that spans \\n 3 lines
2443 ['this', 'is', 'a', 'test', 'of', 'strings', 'that', 'spans', '3', 'lines']
2445 (Note that this is a raw string literal, you must include the leading ``'r'``.)
2446 """
2447 from .testing import pyparsing_test
2449 parseAll = parseAll and parse_all
2450 fullDump = fullDump and full_dump
2451 printResults = printResults and print_results
2452 failureTests = failureTests or failure_tests
2453 postParse = postParse or post_parse
2454 if isinstance(tests, str_type):
2455 tests = typing.cast(str, tests)
2456 line_strip = type(tests).strip
2457 tests = [line_strip(test_line) for test_line in tests.rstrip().splitlines()]
2458 comment_specified = comment is not None
2459 if comment_specified:
2460 if isinstance(comment, str_type):
2461 comment = typing.cast(str, comment)
2462 comment = Literal(comment)
2463 comment = typing.cast(ParserElement, comment)
2464 if file is None:
2465 file = sys.stdout
2466 print_ = file.write
2468 result: Union[ParseResults, Exception]
2469 allResults: list[tuple[str, Union[ParseResults, Exception]]] = []
2470 comments: list[str] = []
2471 success = True
2472 NL = Literal(r"\n").add_parse_action(replace_with("\n")).ignore(quoted_string)
2473 BOM = "\ufeff"
2474 nlstr = "\n"
2475 for t in tests:
2476 if comment_specified and comment.matches(t, False) or comments and not t:
2477 comments.append(
2478 pyparsing_test.with_line_numbers(t) if with_line_numbers else t
2479 )
2480 continue
2481 if not t:
2482 continue
2483 out = [
2484 f"{nlstr}{nlstr.join(comments) if comments else ''}",
2485 pyparsing_test.with_line_numbers(t) if with_line_numbers else t,
2486 ]
2487 comments.clear()
2488 try:
2489 # convert newline marks to actual newlines, and strip leading BOM if present
2490 t = NL.transform_string(t.lstrip(BOM))
2491 result = self.parse_string(t, parse_all=parse_all)
2492 except ParseBaseException as pe:
2493 fatal = "(FATAL) " if isinstance(pe, ParseFatalException) else ""
2494 out.append(pe.explain())
2495 out.append(f"FAIL: {fatal}{pe}")
2496 if ParserElement.verbose_stacktrace:
2497 out.extend(traceback.format_tb(pe.__traceback__))
2498 success = success and failureTests
2499 result = pe
2500 except Exception as exc:
2501 tag = "FAIL-EXCEPTION"
2503 # see if this exception was raised in a parse action
2504 tb = exc.__traceback__
2505 it = iter(traceback.walk_tb(tb))
2506 for f, line in it:
2507 if (f.f_code.co_filename, line) == pa_call_line_synth:
2508 next_f = next(it)[0]
2509 tag += f" (raised in parse action {next_f.f_code.co_name!r})"
2510 break
2512 out.append(f"{tag}: {type(exc).__name__}: {exc}")
2513 if ParserElement.verbose_stacktrace:
2514 out.extend(traceback.format_tb(exc.__traceback__))
2515 success = success and failureTests
2516 result = exc
2517 else:
2518 success = success and not failureTests
2519 if postParse is not None:
2520 try:
2521 pp_value = postParse(t, result)
2522 if pp_value is not None:
2523 if isinstance(pp_value, ParseResults):
2524 out.append(pp_value.dump())
2525 else:
2526 out.append(str(pp_value))
2527 else:
2528 out.append(result.dump())
2529 except Exception as e:
2530 out.append(result.dump(full=fullDump))
2531 out.append(
2532 f"{postParse.__name__} failed: {type(e).__name__}: {e}"
2533 )
2534 else:
2535 out.append(result.dump(full=fullDump))
2536 out.append("")
2538 if printResults:
2539 print_("\n".join(out))
2541 allResults.append((t, result))
2543 return success, allResults
2545 def create_diagram(
2546 self,
2547 output_html: Union[TextIO, Path, str],
2548 vertical: int = 3,
2549 show_results_names: bool = False,
2550 show_groups: bool = False,
2551 embed: bool = False,
2552 show_hidden: bool = False,
2553 **kwargs,
2554 ) -> None:
2555 """
2556 Create a railroad diagram for the parser.
2558 Parameters:
2560 - ``output_html`` (str or file-like object) - output target for generated
2561 diagram HTML
2562 - ``vertical`` (int) - threshold for formatting multiple alternatives vertically
2563 instead of horizontally (default=3)
2564 - ``show_results_names`` - bool flag whether diagram should show annotations for
2565 defined results names
2566 - ``show_groups`` - bool flag whether groups should be highlighted with an unlabeled surrounding box
2567 - ``show_hidden`` - bool flag to show diagram elements for internal elements that are usually hidden
2568 - ``embed`` - bool flag whether generated HTML should omit <HEAD>, <BODY>, and <DOCTYPE> tags to embed
2569 the resulting HTML in an enclosing HTML source
2570 - ``head`` - str containing additional HTML to insert into the <HEAD> section of the generated code;
2571 can be used to insert custom CSS styling
2572 - ``body`` - str containing additional HTML to insert at the beginning of the <BODY> section of the
2573 generated code
2575 Additional diagram-formatting keyword arguments can also be included;
2576 see railroad.Diagram class.
2578 .. versionchanged:: 3.1.0
2579 ``embed`` argument added.
2580 """
2582 try:
2583 from .diagram import to_railroad, railroad_to_html
2584 except ImportError as ie:
2585 raise Exception(
2586 "must ``pip install pyparsing[diagrams]`` to generate parser railroad diagrams"
2587 ) from ie
2589 self.streamline()
2591 railroad = to_railroad(
2592 self,
2593 vertical=vertical,
2594 show_results_names=show_results_names,
2595 show_groups=show_groups,
2596 show_hidden=show_hidden,
2597 diagram_kwargs=kwargs,
2598 )
2599 if not isinstance(output_html, (str, Path)):
2600 # we were passed a file-like object, just write to it
2601 output_html.write(railroad_to_html(railroad, embed=embed, **kwargs))
2602 return
2604 with open(output_html, "w", encoding="utf-8") as diag_file:
2605 diag_file.write(railroad_to_html(railroad, embed=embed, **kwargs))
2607 # Compatibility synonyms
2608 # fmt: off
2609 inlineLiteralsUsing = staticmethod(replaced_by_pep8("inlineLiteralsUsing", inline_literals_using))
2610 setDefaultWhitespaceChars = staticmethod(replaced_by_pep8(
2611 "setDefaultWhitespaceChars", set_default_whitespace_chars
2612 ))
2613 disableMemoization = staticmethod(replaced_by_pep8("disableMemoization", disable_memoization))
2614 enableLeftRecursion = staticmethod(replaced_by_pep8("enableLeftRecursion", enable_left_recursion))
2615 enablePackrat = staticmethod(replaced_by_pep8("enablePackrat", enable_packrat))
2616 resetCache = staticmethod(replaced_by_pep8("resetCache", reset_cache))
2618 setResultsName = replaced_by_pep8("setResultsName", set_results_name)
2619 setBreak = replaced_by_pep8("setBreak", set_break)
2620 setParseAction = replaced_by_pep8("setParseAction", set_parse_action)
2621 addParseAction = replaced_by_pep8("addParseAction", add_parse_action)
2622 addCondition = replaced_by_pep8("addCondition", add_condition)
2623 setFailAction = replaced_by_pep8("setFailAction", set_fail_action)
2624 tryParse = replaced_by_pep8("tryParse", try_parse)
2625 parseString = replaced_by_pep8("parseString", parse_string)
2626 scanString = replaced_by_pep8("scanString", scan_string)
2627 transformString = replaced_by_pep8("transformString", transform_string)
2628 searchString = replaced_by_pep8("searchString", search_string)
2629 ignoreWhitespace = replaced_by_pep8("ignoreWhitespace", ignore_whitespace)
2630 leaveWhitespace = replaced_by_pep8("leaveWhitespace", leave_whitespace)
2631 setWhitespaceChars = replaced_by_pep8("setWhitespaceChars", set_whitespace_chars)
2632 parseWithTabs = replaced_by_pep8("parseWithTabs", parse_with_tabs)
2633 setDebugActions = replaced_by_pep8("setDebugActions", set_debug_actions)
2634 setDebug = replaced_by_pep8("setDebug", set_debug)
2635 setName = replaced_by_pep8("setName", set_name)
2636 parseFile = replaced_by_pep8("parseFile", parse_file)
2637 runTests = replaced_by_pep8("runTests", run_tests)
2638 canParseNext = replaced_by_pep8("canParseNext", can_parse_next)
2639 defaultName = default_name
2640 # fmt: on
2643class _PendingSkip(ParserElement):
2644 # internal placeholder class to hold a place were '...' is added to a parser element,
2645 # once another ParserElement is added, this placeholder will be replaced with a SkipTo
2646 def __init__(self, expr: ParserElement, must_skip: bool = False) -> None:
2647 super().__init__()
2648 self.anchor = expr
2649 self.must_skip = must_skip
2651 def _generateDefaultName(self) -> str:
2652 return str(self.anchor + Empty()).replace("Empty", "...")
2654 def __add__(self, other) -> ParserElement:
2655 skipper = SkipTo(other).set_name("...")("_skipped*")
2656 if self.must_skip:
2658 def must_skip(t):
2659 if not t._skipped or t._skipped.as_list() == [""]:
2660 del t[0]
2661 t.pop("_skipped", None)
2663 def show_skip(t):
2664 if t._skipped.as_list()[-1:] == [""]:
2665 t.pop("_skipped")
2666 t["_skipped"] = f"missing <{self.anchor!r}>"
2668 return (
2669 self.anchor + skipper().add_parse_action(must_skip)
2670 | skipper().add_parse_action(show_skip)
2671 ) + other
2673 return self.anchor + skipper + other
2675 def __repr__(self):
2676 return self.defaultName
2678 def parseImpl(self, *args) -> ParseImplReturnType:
2679 raise Exception(
2680 "use of `...` expression without following SkipTo target expression"
2681 )
2684class Token(ParserElement):
2685 """Abstract :class:`ParserElement` subclass, for defining atomic
2686 matching patterns.
2687 """
2689 def __init__(self) -> None:
2690 super().__init__(savelist=False)
2692 def _generateDefaultName(self) -> str:
2693 return type(self).__name__
2696class NoMatch(Token):
2697 """
2698 A token that will never match.
2699 """
2701 def __init__(self) -> None:
2702 super().__init__()
2703 self._may_return_empty = True
2704 self.mayIndexError = False
2705 self.errmsg = "Unmatchable token"
2707 def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType:
2708 raise ParseException(instring, loc, self.errmsg, self)
2711class Literal(Token):
2712 """
2713 Token to exactly match a specified string.
2715 Example:
2717 .. doctest::
2719 >>> Literal('abc').parse_string('abc')
2720 ParseResults(['abc'], {})
2721 >>> Literal('abc').parse_string('abcdef')
2722 ParseResults(['abc'], {})
2723 >>> Literal('abc').parse_string('ab')
2724 Traceback (most recent call last):
2725 ParseException: Expected 'abc', found 'ab' (at char 0), (line: 1, col: 1)
2727 For case-insensitive matching, use :class:`CaselessLiteral`.
2729 For keyword matching (force word break before and after the matched string),
2730 use :class:`Keyword` or :class:`CaselessKeyword`.
2731 """
2733 def __new__(cls, match_string: str = "", **kwargs):
2734 # Performance tuning: select a subclass with optimized parseImpl
2735 if cls is Literal:
2736 matchString: str = deprecate_argument(kwargs, "matchString", "")
2738 match_string = matchString or match_string
2739 if not match_string:
2740 return super().__new__(Empty)
2741 if len(match_string) == 1:
2742 return super().__new__(_SingleCharLiteral)
2744 # Default behavior
2745 return super().__new__(cls)
2747 # Needed to make copy.copy() work correctly if we customize __new__
2748 def __getnewargs__(self):
2749 return (self.match,)
2751 def __init__(self, match_string: str = "", **kwargs) -> None:
2752 matchString: str = deprecate_argument(kwargs, "matchString", "")
2754 super().__init__()
2755 match_string = matchString or match_string
2756 self.match = match_string
2757 self.matchLen = len(match_string)
2758 self.firstMatchChar = match_string[:1]
2759 self.errmsg = f"Expected {self.name}"
2760 self._may_return_empty = False
2761 self.mayIndexError = False
2763 def _generateDefaultName(self) -> str:
2764 return repr(self.match)
2766 def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType:
2767 if instring[loc] == self.firstMatchChar and instring.startswith(
2768 self.match, loc
2769 ):
2770 return loc + self.matchLen, self.match
2771 raise ParseException(instring, loc, self.errmsg, self)
2774class Empty(Literal):
2775 """
2776 An empty token, will always match.
2777 """
2779 def __init__(self, match_string="", *, matchString="") -> None:
2780 super().__init__("")
2781 self._may_return_empty = True
2782 self.mayIndexError = False
2784 def _generateDefaultName(self) -> str:
2785 return "Empty"
2787 def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType:
2788 return loc, []
2791class _SingleCharLiteral(Literal):
2792 def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType:
2793 if instring[loc] == self.firstMatchChar:
2794 return loc + 1, self.match
2795 raise ParseException(instring, loc, self.errmsg, self)
2798ParserElement._literalStringClass = Literal
2801class Keyword(Token):
2802 """
2803 Token to exactly match a specified string as a keyword, that is,
2804 it must be immediately preceded and followed by whitespace or
2805 non-keyword characters. Compare with :class:`Literal`:
2807 - ``Literal("if")`` will match the leading ``'if'`` in
2808 ``'ifAndOnlyIf'``.
2809 - ``Keyword("if")`` will not; it will only match the leading
2810 ``'if'`` in ``'if x=1'``, or ``'if(y==2)'``
2812 Accepts two optional constructor arguments in addition to the
2813 keyword string:
2815 - ``ident_chars`` is a string of characters that would be valid
2816 identifier characters, defaulting to all alphanumerics + "_" and
2817 "$"
2818 - ``caseless`` allows case-insensitive matching, default is ``False``.
2820 Example:
2822 .. doctest::
2823 :options: +NORMALIZE_WHITESPACE
2825 >>> Keyword("start").parse_string("start")
2826 ParseResults(['start'], {})
2827 >>> Keyword("start").parse_string("starting")
2828 Traceback (most recent call last):
2829 ParseException: Expected Keyword 'start', keyword was immediately
2830 followed by keyword character, found 'ing' (at char 5), (line:1, col:6)
2832 .. doctest::
2833 :options: +NORMALIZE_WHITESPACE
2835 >>> Keyword("start").parse_string("starting").debug()
2836 Traceback (most recent call last):
2837 ParseException: Expected Keyword "start", keyword was immediately
2838 followed by keyword character, found 'ing' ...
2840 For case-insensitive matching, use :class:`CaselessKeyword`.
2841 """
2843 DEFAULT_KEYWORD_CHARS = f"{alphanums}_$"
2845 def __init__(
2846 self,
2847 match_string: str = "",
2848 ident_chars: typing.Optional[str] = None,
2849 caseless: bool = False,
2850 **kwargs,
2851 ) -> None:
2852 matchString = deprecate_argument(kwargs, "matchString", "")
2853 identChars = deprecate_argument(kwargs, "identChars", None)
2855 super().__init__()
2856 identChars = identChars or ident_chars
2857 if identChars is None:
2858 identChars = Keyword.DEFAULT_KEYWORD_CHARS
2859 match_string = matchString or match_string
2860 self.match = match_string
2861 self.matchLen = len(match_string)
2862 self.firstMatchChar = match_string[:1]
2863 if not self.firstMatchChar:
2864 raise ValueError("null string passed to Keyword; use Empty() instead")
2865 self.errmsg = f"Expected {type(self).__name__} {self.name}"
2866 self._may_return_empty = False
2867 self.mayIndexError = False
2868 self.caseless = caseless
2869 if caseless:
2870 self.caselessmatch = match_string.upper()
2871 identChars = identChars.upper()
2872 self.ident_chars = set(identChars)
2874 @property
2875 def identChars(self) -> set[str]:
2876 """
2877 .. deprecated:: 3.3.0
2878 use ident_chars instead.
2880 Property returning the characters being used as keyword characters for this expression.
2881 """
2882 return self.ident_chars
2884 def _generateDefaultName(self) -> str:
2885 return repr(self.match)
2887 def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType:
2888 errmsg = self.errmsg or ""
2889 errloc = loc
2890 if self.caseless:
2891 if instring[loc : loc + self.matchLen].upper() == self.caselessmatch:
2892 if loc == 0 or instring[loc - 1].upper() not in self.identChars:
2893 if (
2894 loc >= len(instring) - self.matchLen
2895 or instring[loc + self.matchLen].upper() not in self.identChars
2896 ):
2897 return loc + self.matchLen, self.match
2899 # followed by keyword char
2900 errmsg += ", was immediately followed by keyword character"
2901 errloc = loc + self.matchLen
2902 else:
2903 # preceded by keyword char
2904 errmsg += ", keyword was immediately preceded by keyword character"
2905 errloc = loc - 1
2906 # else no match just raise plain exception
2908 elif (
2909 instring[loc] == self.firstMatchChar
2910 and self.matchLen == 1
2911 or instring.startswith(self.match, loc)
2912 ):
2913 if loc == 0 or instring[loc - 1] not in self.identChars:
2914 if (
2915 loc >= len(instring) - self.matchLen
2916 or instring[loc + self.matchLen] not in self.identChars
2917 ):
2918 return loc + self.matchLen, self.match
2920 # followed by keyword char
2921 errmsg += ", keyword was immediately followed by keyword character"
2922 errloc = loc + self.matchLen
2923 else:
2924 # preceded by keyword char
2925 errmsg += ", keyword was immediately preceded by keyword character"
2926 errloc = loc - 1
2927 # else no match just raise plain exception
2929 raise ParseException(instring, errloc, errmsg, self)
2931 @staticmethod
2932 def set_default_keyword_chars(chars) -> None:
2933 """
2934 Overrides the default characters used by :class:`Keyword` expressions.
2935 """
2936 Keyword.DEFAULT_KEYWORD_CHARS = chars
2938 # Compatibility synonyms
2939 setDefaultKeywordChars = staticmethod(
2940 replaced_by_pep8("setDefaultKeywordChars", set_default_keyword_chars)
2941 )
2944class CaselessLiteral(Literal):
2945 """
2946 Token to match a specified string, ignoring case of letters.
2947 Note: the matched results will always be in the case of the given
2948 match string, NOT the case of the input text.
2950 Example:
2952 .. doctest::
2954 >>> CaselessLiteral("CMD")[1, ...].parse_string("cmd CMD Cmd10")
2955 ParseResults(['CMD', 'CMD', 'CMD'], {})
2957 (Contrast with example for :class:`CaselessKeyword`.)
2958 """
2960 def __init__(self, match_string: str = "", **kwargs) -> None:
2961 matchString: str = deprecate_argument(kwargs, "matchString", "")
2963 match_string = matchString or match_string
2964 super().__init__(match_string.upper())
2965 # Preserve the defining literal.
2966 self.returnString = match_string
2967 self.errmsg = f"Expected {self.name}"
2969 def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType:
2970 if instring[loc : loc + self.matchLen].upper() == self.match:
2971 return loc + self.matchLen, self.returnString
2972 raise ParseException(instring, loc, self.errmsg, self)
2975class CaselessKeyword(Keyword):
2976 """
2977 Caseless version of :class:`Keyword`.
2979 Example:
2981 .. doctest::
2983 >>> CaselessKeyword("CMD")[1, ...].parse_string("cmd CMD Cmd10")
2984 ParseResults(['CMD', 'CMD'], {})
2986 (Contrast with example for :class:`CaselessLiteral`.)
2987 """
2989 def __init__(
2990 self, match_string: str = "", ident_chars: typing.Optional[str] = None, **kwargs
2991 ) -> None:
2992 matchString: str = deprecate_argument(kwargs, "matchString", "")
2993 identChars: typing.Optional[str] = deprecate_argument(
2994 kwargs, "identChars", None
2995 )
2997 identChars = identChars or ident_chars
2998 match_string = matchString or match_string
2999 super().__init__(match_string, identChars, caseless=True)
3002class CloseMatch(Token):
3003 """A variation on :class:`Literal` which matches "close" matches,
3004 that is, strings with at most 'n' mismatching characters.
3005 :class:`CloseMatch` takes parameters:
3007 - ``match_string`` - string to be matched
3008 - ``caseless`` - a boolean indicating whether to ignore casing when comparing characters
3009 - ``max_mismatches`` - (``default=1``) maximum number of
3010 mismatches allowed to count as a match
3012 The results from a successful parse will contain the matched text
3013 from the input string and the following named results:
3015 - ``mismatches`` - a list of the positions within the
3016 match_string where mismatches were found
3017 - ``original`` - the original match_string used to compare
3018 against the input string
3020 If ``mismatches`` is an empty list, then the match was an exact
3021 match.
3023 Example:
3025 .. doctest::
3026 :options: +NORMALIZE_WHITESPACE
3028 >>> patt = CloseMatch("ATCATCGAATGGA")
3029 >>> patt.parse_string("ATCATCGAAXGGA")
3030 ParseResults(['ATCATCGAAXGGA'],
3031 {'original': 'ATCATCGAATGGA', 'mismatches': [9]})
3033 >>> patt.parse_string("ATCAXCGAAXGGA")
3034 Traceback (most recent call last):
3035 ParseException: Expected 'ATCATCGAATGGA' (with up to 1 mismatches),
3036 found 'ATCAXCGAAXGGA' (at char 0), (line:1, col:1)
3038 # exact match
3039 >>> patt.parse_string("ATCATCGAATGGA")
3040 ParseResults(['ATCATCGAATGGA'],
3041 {'original': 'ATCATCGAATGGA', 'mismatches': []})
3043 # close match allowing up to 2 mismatches
3044 >>> patt = CloseMatch("ATCATCGAATGGA", max_mismatches=2)
3045 >>> patt.parse_string("ATCAXCGAAXGGA")
3046 ParseResults(['ATCAXCGAAXGGA'],
3047 {'original': 'ATCATCGAATGGA', 'mismatches': [4, 9]})
3048 """
3050 def __init__(
3051 self,
3052 match_string: str,
3053 max_mismatches: typing.Optional[int] = None,
3054 *,
3055 caseless=False,
3056 **kwargs,
3057 ) -> None:
3058 maxMismatches: int = deprecate_argument(kwargs, "maxMismatches", 1)
3060 maxMismatches = max_mismatches if max_mismatches is not None else maxMismatches
3061 super().__init__()
3062 self.match_string = match_string
3063 self.maxMismatches = maxMismatches
3064 self.errmsg = f"Expected {self.match_string!r} (with up to {self.maxMismatches} mismatches)"
3065 self.caseless = caseless
3066 self.mayIndexError = False
3067 self._may_return_empty = False
3069 def _generateDefaultName(self) -> str:
3070 return f"{type(self).__name__}:{self.match_string!r}"
3072 def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType:
3073 start = loc
3074 instrlen = len(instring)
3075 maxloc = start + len(self.match_string)
3077 if maxloc <= instrlen:
3078 match_string = self.match_string
3079 match_stringloc = 0
3080 mismatches = []
3081 maxMismatches = self.maxMismatches
3083 for match_stringloc, s_m in enumerate(
3084 zip(instring[loc:maxloc], match_string)
3085 ):
3086 src, mat = s_m
3087 if self.caseless:
3088 src, mat = src.lower(), mat.lower()
3090 if src != mat:
3091 mismatches.append(match_stringloc)
3092 if len(mismatches) > maxMismatches:
3093 break
3094 else:
3095 loc = start + match_stringloc + 1
3096 results = ParseResults([instring[start:loc]])
3097 results["original"] = match_string
3098 results["mismatches"] = mismatches
3099 return loc, results
3101 raise ParseException(instring, loc, self.errmsg, self)
3104class Word(Token):
3105 """Token for matching words composed of allowed character sets.
3107 Parameters:
3109 - ``init_chars`` - string of all characters that should be used to
3110 match as a word; "ABC" will match "AAA", "ABAB", "CBAC", etc.;
3111 if ``body_chars`` is also specified, then this is the string of
3112 initial characters
3113 - ``body_chars`` - string of characters that
3114 can be used for matching after a matched initial character as
3115 given in ``init_chars``; if omitted, same as the initial characters
3116 (default=``None``)
3117 - ``min`` - minimum number of characters to match (default=1)
3118 - ``max`` - maximum number of characters to match (default=0)
3119 - ``exact`` - exact number of characters to match (default=0)
3120 - ``as_keyword`` - match as a keyword (default=``False``)
3121 - ``exclude_chars`` - characters that might be
3122 found in the input ``body_chars`` string but which should not be
3123 accepted for matching ;useful to define a word of all
3124 printables except for one or two characters, for instance
3125 (default=``None``)
3127 :class:`srange` is useful for defining custom character set strings
3128 for defining :class:`Word` expressions, using range notation from
3129 regular expression character sets.
3131 A common mistake is to use :class:`Word` to match a specific literal
3132 string, as in ``Word("Address")``. Remember that :class:`Word`
3133 uses the string argument to define *sets* of matchable characters.
3134 This expression would match "Add", "AAA", "dAred", or any other word
3135 made up of the characters 'A', 'd', 'r', 'e', and 's'. To match an
3136 exact literal string, use :class:`Literal` or :class:`Keyword`.
3138 pyparsing includes helper strings for building Words:
3140 - :attr:`alphas`
3141 - :attr:`nums`
3142 - :attr:`alphanums`
3143 - :attr:`hexnums`
3144 - :attr:`alphas8bit` (alphabetic characters in ASCII range 128-255
3145 - accented, tilded, umlauted, etc.)
3146 - :attr:`punc8bit` (non-alphabetic characters in ASCII range
3147 128-255 - currency, symbols, superscripts, diacriticals, etc.)
3148 - :attr:`printables` (any non-whitespace character)
3150 ``alphas``, ``nums``, and ``printables`` are also defined in several
3151 Unicode sets - see :class:`pyparsing_unicode`.
3153 Example:
3155 .. testcode::
3157 # a word composed of digits
3158 integer = Word(nums)
3159 # Two equivalent alternate forms:
3160 Word("0123456789")
3161 Word(srange("[0-9]"))
3163 # a word with a leading capital, and zero or more lowercase
3164 capitalized_word = Word(alphas.upper(), alphas.lower())
3166 # hostnames are alphanumeric, with leading alpha, and '-'
3167 hostname = Word(alphas, alphanums + '-')
3169 # roman numeral
3170 # (not a strict parser, accepts invalid mix of characters)
3171 roman = Word("IVXLCDM")
3173 # any string of non-whitespace characters, except for ','
3174 csv_value = Word(printables, exclude_chars=",")
3176 :raises ValueError: If ``min`` and ``max`` are both specified
3177 and the test ``min <= max`` fails.
3179 .. versionchanged:: 3.1.0
3180 Raises :exc:`ValueError` if ``min`` > ``max``.
3181 """
3183 def __init__(
3184 self,
3185 init_chars: str = "",
3186 body_chars: typing.Optional[str] = None,
3187 min: int = 1,
3188 max: int = 0,
3189 exact: int = 0,
3190 as_keyword: bool = False,
3191 exclude_chars: typing.Optional[str] = None,
3192 **kwargs,
3193 ) -> None:
3194 initChars: typing.Optional[str] = deprecate_argument(kwargs, "initChars", None)
3195 bodyChars: typing.Optional[str] = deprecate_argument(kwargs, "bodyChars", None)
3196 asKeyword: bool = deprecate_argument(kwargs, "asKeyword", False)
3197 excludeChars: typing.Optional[str] = deprecate_argument(
3198 kwargs, "excludeChars", None
3199 )
3201 initChars = initChars or init_chars
3202 bodyChars = bodyChars or body_chars
3203 asKeyword = asKeyword or as_keyword
3204 excludeChars = excludeChars or exclude_chars
3205 super().__init__()
3206 if not initChars:
3207 raise ValueError(
3208 f"invalid {type(self).__name__}, initChars cannot be empty string"
3209 )
3211 initChars_set = set(initChars)
3212 if excludeChars:
3213 excludeChars_set = set(excludeChars)
3214 initChars_set -= excludeChars_set
3215 if bodyChars:
3216 bodyChars = "".join(set(bodyChars) - excludeChars_set)
3217 self.init_chars = initChars_set
3218 self.initCharsOrig = "".join(sorted(initChars_set))
3220 if bodyChars:
3221 self.bodyChars = set(bodyChars)
3222 self.bodyCharsOrig = "".join(sorted(bodyChars))
3223 else:
3224 self.bodyChars = initChars_set
3225 self.bodyCharsOrig = self.initCharsOrig
3227 self.maxSpecified = max > 0
3229 if min < 1:
3230 raise ValueError(
3231 "cannot specify a minimum length < 1; use Opt(Word()) if zero-length word is permitted"
3232 )
3234 if self.maxSpecified and min > max:
3235 raise ValueError(
3236 f"invalid args, if min and max both specified min must be <= max (min={min}, max={max})"
3237 )
3239 self.minLen = min
3241 if max > 0:
3242 self.maxLen = max
3243 else:
3244 self.maxLen = _MAX_INT
3246 if exact > 0:
3247 min = max = exact
3248 self.maxLen = exact
3249 self.minLen = exact
3251 self.errmsg = f"Expected {self.name}"
3252 self.mayIndexError = False
3253 self.asKeyword = asKeyword
3254 if self.asKeyword:
3255 self.errmsg += " as a keyword"
3257 # see if we can make a regex for this Word
3258 if " " not in (self.initChars | self.bodyChars):
3259 if len(self.initChars) == 1:
3260 re_leading_fragment = re.escape(self.initCharsOrig)
3261 else:
3262 re_leading_fragment = f"[{_collapse_string_to_ranges(self.initChars)}]"
3264 if self.bodyChars == self.initChars:
3265 if max == 0 and self.minLen == 1:
3266 repeat = "+"
3267 elif max == 1:
3268 repeat = ""
3269 else:
3270 if self.minLen != self.maxLen:
3271 repeat = f"{{{self.minLen},{'' if self.maxLen == _MAX_INT else self.maxLen}}}"
3272 else:
3273 repeat = f"{{{self.minLen}}}"
3274 self.reString = f"{re_leading_fragment}{repeat}"
3275 else:
3276 if max == 1:
3277 re_body_fragment = ""
3278 repeat = ""
3279 else:
3280 re_body_fragment = f"[{_collapse_string_to_ranges(self.bodyChars)}]"
3281 if max == 0 and self.minLen == 1:
3282 repeat = "*"
3283 elif max == 2:
3284 repeat = "?" if min <= 1 else ""
3285 else:
3286 if min != max:
3287 repeat = f"{{{min - 1 if min > 0 else ''},{max - 1 if max > 0 else ''}}}"
3288 else:
3289 repeat = f"{{{min - 1 if min > 0 else ''}}}"
3291 self.reString = f"{re_leading_fragment}{re_body_fragment}{repeat}"
3293 if self.asKeyword:
3294 self.reString = rf"\b{self.reString}\b"
3296 try:
3297 self.re = re.compile(self.reString)
3298 except re.error:
3299 self.re = None # type: ignore[assignment]
3300 else:
3301 self.re_match = self.re.match
3302 self.parseImpl = self.parseImpl_regex # type: ignore[method-assign]
3304 @property
3305 def initChars(self) -> set[str]:
3306 """
3307 .. deprecated:: 3.3.0
3308 use `init_chars` instead.
3310 Property returning the initial chars to be used when matching this
3311 Word expression. If no body chars were specified, the initial characters
3312 will also be the body characters.
3313 """
3314 return set(self.init_chars)
3316 def copy(self) -> Word:
3317 """
3318 Returns a copy of this expression.
3320 Generally only used internally by pyparsing.
3321 """
3322 ret: Word = cast(Word, super().copy())
3323 if hasattr(self, "re_match"):
3324 ret.re_match = self.re_match
3325 ret.parseImpl = ret.parseImpl_regex # type: ignore[method-assign]
3326 return ret
3328 def _generateDefaultName(self) -> str:
3329 def charsAsStr(s):
3330 max_repr_len = 16
3331 s = _collapse_string_to_ranges(s, re_escape=False)
3333 if len(s) > max_repr_len:
3334 return f"{s[:max_repr_len - 3]}..."
3336 return s
3338 if self.initChars != self.bodyChars:
3339 base = f"W:({charsAsStr(self.initChars)}, {charsAsStr(self.bodyChars)})"
3340 else:
3341 base = f"W:({charsAsStr(self.initChars)})"
3343 # add length specification
3344 if self.minLen > 1 or self.maxLen != _MAX_INT:
3345 if self.minLen == self.maxLen:
3346 if self.minLen == 1:
3347 return base[2:]
3348 else:
3349 return base + f"{{{self.minLen}}}"
3350 elif self.maxLen == _MAX_INT:
3351 return base + f"{{{self.minLen},...}}"
3352 else:
3353 return base + f"{{{self.minLen},{self.maxLen}}}"
3354 return base
3356 def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType:
3357 if instring[loc] not in self.initChars:
3358 raise ParseException(instring, loc, self.errmsg, self)
3360 start = loc
3361 loc += 1
3362 instrlen = len(instring)
3363 body_chars: set[str] = self.bodyChars
3364 maxloc = start + self.maxLen
3365 maxloc = min(maxloc, instrlen)
3366 while loc < maxloc and instring[loc] in body_chars:
3367 loc += 1
3369 throw_exception = False
3370 if loc - start < self.minLen:
3371 throw_exception = True
3372 elif self.asKeyword and (
3373 (start > 0 and instring[start - 1] in body_chars)
3374 or (loc < instrlen and instring[loc] in body_chars)
3375 ):
3376 throw_exception = True
3378 if throw_exception:
3379 raise ParseException(instring, loc, self.errmsg, self)
3381 return loc, instring[start:loc]
3383 def parseImpl_regex(self, instring, loc, do_actions=True) -> ParseImplReturnType:
3384 result = self.re_match(instring, loc)
3385 if not result:
3386 raise ParseException(instring, loc, self.errmsg, self)
3388 loc = result.end()
3389 return loc, result[0]
3392class Char(Word):
3393 """A short-cut class for defining :class:`Word` ``(characters, exact=1)``,
3394 when defining a match of any single character in a string of
3395 characters.
3396 """
3398 def __init__(
3399 self,
3400 charset: str,
3401 as_keyword: bool = False,
3402 exclude_chars: typing.Optional[str] = None,
3403 **kwargs,
3404 ) -> None:
3405 asKeyword: bool = deprecate_argument(kwargs, "asKeyword", False)
3406 excludeChars: typing.Optional[str] = deprecate_argument(
3407 kwargs, "excludeChars", None
3408 )
3410 asKeyword = asKeyword or as_keyword
3411 excludeChars = excludeChars or exclude_chars
3412 super().__init__(
3413 charset, exact=1, as_keyword=asKeyword, exclude_chars=excludeChars
3414 )
3417class Regex(Token):
3418 r"""Token for matching strings that match a given regular
3419 expression. Defined with string specifying the regular expression in
3420 a form recognized by the stdlib Python `re module <https://docs.python.org/3/library/re.html>`_.
3421 If the given regex contains named groups (defined using ``(?P<name>...)``),
3422 these will be preserved as named :class:`ParseResults`.
3424 If instead of the Python stdlib ``re`` module you wish to use a different RE module
3425 (such as the ``regex`` module), you can do so by building your ``Regex`` object with
3426 a compiled RE that was compiled using ``regex``.
3428 The parameters ``pattern`` and ``flags`` are passed
3429 to the ``re.compile()`` function as-is. See the Python
3430 `re module <https://docs.python.org/3/library/re.html>`_ module for an
3431 explanation of the acceptable patterns and flags.
3433 Example:
3435 .. testcode::
3437 realnum = Regex(r"[+-]?\d+\.\d*")
3438 # ref: https://stackoverflow.com/questions/267399/how-do-you-match-only-valid-roman-numerals-with-a-regular-expression
3439 roman = Regex(r"M{0,4}(CM|CD|D?{0,3})(XC|XL|L?X{0,3})(IX|IV|V?I{0,3})")
3441 # named fields in a regex will be returned as named results
3442 date = Regex(r'(?P<year>\d{4})-(?P<month>\d\d?)-(?P<day>\d\d?)')
3444 # the Regex class will accept regular expressions compiled using the
3445 # re module
3446 import re
3447 parser = pp.Regex(re.compile(r'[0-9]'))
3448 """
3450 def __init__(
3451 self,
3452 pattern: Any,
3453 flags: Union[re.RegexFlag, int] = 0,
3454 as_group_list: bool = False,
3455 as_match: bool = False,
3456 **kwargs,
3457 ) -> None:
3458 super().__init__()
3459 asGroupList: bool = deprecate_argument(kwargs, "asGroupList", False)
3460 asMatch: bool = deprecate_argument(kwargs, "asMatch", False)
3462 asGroupList = asGroupList or as_group_list
3463 asMatch = asMatch or as_match
3465 if isinstance(pattern, str_type):
3466 if not pattern:
3467 raise ValueError("null string passed to Regex; use Empty() instead")
3469 self._re = None
3470 self._may_return_empty = None # type: ignore [assignment]
3471 self.reString = self.pattern = pattern
3473 elif hasattr(pattern, "pattern") and hasattr(pattern, "match"):
3474 self._re = pattern
3475 self._may_return_empty = None # type: ignore [assignment]
3476 self.pattern = self.reString = pattern.pattern
3478 elif callable(pattern):
3479 # defer creating this pattern until we really need it
3480 self.pattern = pattern
3481 self._may_return_empty = None # type: ignore [assignment]
3482 self._re = None
3484 else:
3485 raise TypeError(
3486 "Regex may only be constructed with a string or a compiled RE object,"
3487 " or a callable that takes no arguments and returns a string or a"
3488 " compiled RE object"
3489 )
3491 self.flags = flags
3492 self.errmsg = f"Expected {self.name}"
3493 self.mayIndexError = False
3494 self.asGroupList = asGroupList
3495 self.asMatch = asMatch
3496 if self.asGroupList:
3497 self.parseImpl = self.parseImplAsGroupList # type: ignore [method-assign]
3498 if self.asMatch:
3499 self.parseImpl = self.parseImplAsMatch # type: ignore [method-assign]
3501 def copy(self) -> Regex:
3502 """
3503 Returns a copy of this expression.
3505 Generally only used internally by pyparsing.
3506 """
3507 ret: Regex = cast(Regex, super().copy())
3508 if self.asGroupList:
3509 ret.parseImpl = ret.parseImplAsGroupList # type: ignore [method-assign]
3510 if self.asMatch:
3511 ret.parseImpl = ret.parseImplAsMatch # type: ignore [method-assign]
3512 return ret
3514 @cached_property
3515 def re(self) -> re.Pattern:
3516 """
3517 Property returning the compiled regular expression for this Regex.
3519 Generally only used internally by pyparsing.
3520 """
3521 if self._re:
3522 return self._re
3524 if callable(self.pattern):
3525 # replace self.pattern with the string returned by calling self.pattern()
3526 self.pattern = cast(Callable[[], str], self.pattern)()
3528 # see if we got a compiled RE back instead of a str - if so, we're done
3529 if hasattr(self.pattern, "pattern") and hasattr(self.pattern, "match"):
3530 self._re = cast(re.Pattern[str], self.pattern)
3531 self.pattern = self.reString = self._re.pattern
3532 return self._re
3534 try:
3535 self._re = re.compile(self.pattern, self.flags)
3536 except re.error:
3537 raise ValueError(f"invalid pattern ({self.pattern!r}) passed to Regex")
3538 else:
3539 self._may_return_empty = self.re.match("", pos=0) is not None
3540 return self._re
3542 @cached_property
3543 def re_match(self) -> Callable[[str, int], Any]:
3544 return self.re.match
3546 @property
3547 def mayReturnEmpty(self):
3548 if self._may_return_empty is None:
3549 # force compile of regex pattern, to set may_return_empty flag
3550 self.re # noqa
3551 return self._may_return_empty
3553 @mayReturnEmpty.setter
3554 def mayReturnEmpty(self, value):
3555 self._may_return_empty = value
3557 def _generateDefaultName(self) -> str:
3558 unescaped = repr(self.pattern).replace("\\\\", "\\")
3559 return f"Re:({unescaped})"
3561 def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType:
3562 # explicit check for matching past the length of the string;
3563 # this is done because the re module will not complain about
3564 # a match with `pos > len(instring)`, it will just return ""
3565 if loc > len(instring) and self.mayReturnEmpty:
3566 raise ParseException(instring, loc, self.errmsg, self)
3568 result = self.re_match(instring, loc)
3569 if not result:
3570 raise ParseException(instring, loc, self.errmsg, self)
3572 loc = result.end()
3573 ret = ParseResults(result[0])
3574 d = result.groupdict()
3576 for k, v in d.items():
3577 ret[k] = v
3579 return loc, ret
3581 def parseImplAsGroupList(self, instring, loc, do_actions=True):
3582 if loc > len(instring) and self.mayReturnEmpty:
3583 raise ParseException(instring, loc, self.errmsg, self)
3585 result = self.re_match(instring, loc)
3586 if not result:
3587 raise ParseException(instring, loc, self.errmsg, self)
3589 loc = result.end()
3590 ret = result.groups()
3591 return loc, ret
3593 def parseImplAsMatch(self, instring, loc, do_actions=True):
3594 if loc > len(instring) and self.mayReturnEmpty:
3595 raise ParseException(instring, loc, self.errmsg, self)
3597 result = self.re_match(instring, loc)
3598 if not result:
3599 raise ParseException(instring, loc, self.errmsg, self)
3601 loc = result.end()
3602 ret = result
3603 return loc, ret
3605 def sub(self, repl: str) -> ParserElement:
3606 r"""
3607 Return :class:`Regex` with an attached parse action to transform the parsed
3608 result as if called using `re.sub(expr, repl, string) <https://docs.python.org/3/library/re.html#re.sub>`_.
3610 Example:
3612 .. testcode::
3614 make_html = Regex(r"(\w+):(.*?):").sub(r"<\1>\2</\1>")
3615 print(make_html.transform_string("h1:main title:"))
3617 .. testoutput::
3619 <h1>main title</h1>
3620 """
3621 if self.asGroupList:
3622 raise TypeError("cannot use sub() with Regex(as_group_list=True)")
3624 if self.asMatch and callable(repl):
3625 raise TypeError(
3626 "cannot use sub() with a callable with Regex(as_match=True)"
3627 )
3629 if self.asMatch:
3631 def pa(tokens):
3632 return tokens[0].expand(repl)
3634 else:
3636 def pa(tokens):
3637 return self.re.sub(repl, tokens[0])
3639 return self.add_parse_action(pa)
3642class QuotedString(Token):
3643 r"""
3644 Token for matching strings that are delimited by quoting characters.
3646 Defined with the following parameters:
3648 - ``quote_char`` - string of one or more characters defining the
3649 quote delimiting string
3650 - ``esc_char`` - character to re_escape quotes, typically backslash
3651 (default= ``None``)
3652 - ``esc_quote`` - special quote sequence to re_escape an embedded quote
3653 string (such as SQL's ``""`` to re_escape an embedded ``"``)
3654 (default= ``None``)
3655 - ``multiline`` - boolean indicating whether quotes can span
3656 multiple lines (default= ``False``)
3657 - ``unquote_results`` - boolean indicating whether the matched text
3658 should be unquoted (default= ``True``)
3659 - ``end_quote_char`` - string of one or more characters defining the
3660 end of the quote delimited string (default= ``None`` => same as
3661 quote_char)
3662 - ``convert_whitespace_escapes`` - convert escaped whitespace
3663 (``'\t'``, ``'\n'``, etc.) to actual whitespace
3664 (default= ``True``)
3666 .. caution:: ``convert_whitespace_escapes`` has no effect if
3667 ``unquote_results`` is ``False``.
3669 Example:
3671 .. doctest::
3673 >>> qs = QuotedString('"')
3674 >>> print(qs.search_string('lsjdf "This is the quote" sldjf'))
3675 [['This is the quote']]
3676 >>> complex_qs = QuotedString('{{', end_quote_char='}}')
3677 >>> print(complex_qs.search_string(
3678 ... 'lsjdf {{This is the "quote"}} sldjf'))
3679 [['This is the "quote"']]
3680 >>> sql_qs = QuotedString('"', esc_quote='""')
3681 >>> print(sql_qs.search_string(
3682 ... 'lsjdf "This is the quote with ""embedded"" quotes" sldjf'))
3683 [['This is the quote with "embedded" quotes']]
3684 """
3686 ws_map = dict(((r"\t", "\t"), (r"\n", "\n"), (r"\f", "\f"), (r"\r", "\r")))
3688 def __init__(
3689 self,
3690 quote_char: str = "",
3691 esc_char: typing.Optional[str] = None,
3692 esc_quote: typing.Optional[str] = None,
3693 multiline: bool = False,
3694 unquote_results: bool = True,
3695 end_quote_char: typing.Optional[str] = None,
3696 convert_whitespace_escapes: bool = True,
3697 **kwargs,
3698 ) -> None:
3699 super().__init__()
3700 quoteChar: str = deprecate_argument(kwargs, "quoteChar", "")
3701 escChar: str = deprecate_argument(kwargs, "escChar", None)
3702 escQuote: str = deprecate_argument(kwargs, "escQuote", None)
3703 unquoteResults: bool = deprecate_argument(kwargs, "unquoteResults", True)
3704 endQuoteChar: typing.Optional[str] = deprecate_argument(
3705 kwargs, "endQuoteChar", None
3706 )
3707 convertWhitespaceEscapes: bool = deprecate_argument(
3708 kwargs, "convertWhitespaceEscapes", True
3709 )
3711 esc_char = escChar or esc_char
3712 esc_quote = escQuote or esc_quote
3713 unquote_results = unquoteResults and unquote_results
3714 end_quote_char = endQuoteChar or end_quote_char
3715 convert_whitespace_escapes = (
3716 convertWhitespaceEscapes and convert_whitespace_escapes
3717 )
3718 quote_char = quoteChar or quote_char
3720 # reject empty or whitespace-only quote chars, but preserve any
3721 # whitespace that is part of a valid quote delimiter (e.g. a leading
3722 # newline in a multiline quote such as "\n;")
3723 if not quote_char.strip():
3724 raise ValueError("quote_char cannot be the empty string")
3726 if end_quote_char is None:
3727 end_quote_char = quote_char
3728 elif not end_quote_char.strip():
3729 raise ValueError("end_quote_char cannot be the empty string")
3731 self.quote_char: str = quote_char
3732 self.quote_char_len: int = len(quote_char)
3733 self.first_quote_char: str = quote_char[0]
3734 self.end_quote_char: str = end_quote_char
3735 self.end_quote_char_len: int = len(end_quote_char)
3736 self.esc_char: str = esc_char or ""
3737 self.has_esc_char: bool = esc_char is not None
3738 self.esc_quote: str = esc_quote or ""
3739 self.unquote_results: bool = unquote_results
3740 self.convert_whitespace_escapes: bool = convert_whitespace_escapes
3741 self.multiline = multiline
3742 self.re_flags = re.RegexFlag(0)
3744 # fmt: off
3745 # build up re pattern for the content between the quote delimiters
3746 inner_pattern: list[str] = []
3748 if esc_quote:
3749 inner_pattern.append(rf"(?:{re.escape(esc_quote)})")
3751 if esc_char:
3752 inner_pattern.append(rf"(?:{re.escape(esc_char)}.)")
3754 if len(self.end_quote_char) > 1:
3755 inner_pattern.append(
3756 "(?:"
3757 + "|".join(
3758 f"(?:{re.escape(self.end_quote_char[:i])}(?!{re.escape(self.end_quote_char[i:])}))"
3759 for i in range(len(self.end_quote_char) - 1, 0, -1)
3760 )
3761 + ")"
3762 )
3764 if self.multiline:
3765 self.re_flags |= re.MULTILINE | re.DOTALL
3766 inner_pattern.append(
3767 rf"(?:[^{_escape_regex_range_chars(self.end_quote_char[0])}"
3768 rf"{(_escape_regex_range_chars(self.esc_char) if self.has_esc_char else '')}])"
3769 )
3770 else:
3771 inner_pattern.append(
3772 rf"(?:[^{_escape_regex_range_chars(self.end_quote_char[0])}\n\r"
3773 rf"{(_escape_regex_range_chars(self.esc_char) if self.has_esc_char else '')}])"
3774 )
3776 self.pattern = "".join(
3777 [
3778 re.escape(self.quote_char),
3779 "(?:",
3780 '|'.join(inner_pattern),
3781 ")*",
3782 re.escape(self.end_quote_char),
3783 ]
3784 )
3786 if self.unquote_results:
3787 if self.convert_whitespace_escapes:
3788 self.unquote_scan_re = re.compile(
3789 rf"({'|'.join(re.escape(k) for k in self.ws_map)})"
3790 rf"|(\\[0-7]{3}|\\0|\\x[0-9a-fA-F]{2}|\\u[0-9a-fA-F]{4})"
3791 rf"|({re.escape(self.esc_char)}.)"
3792 rf"|(\n|.)",
3793 flags=self.re_flags,
3794 )
3795 else:
3796 self.unquote_scan_re = re.compile(
3797 rf"({re.escape(self.esc_char)}.)"
3798 rf"|(\n|.)",
3799 flags=self.re_flags
3800 )
3801 # fmt: on
3803 try:
3804 self.re = re.compile(self.pattern, self.re_flags)
3805 self.reString = self.pattern
3806 self.re_match = self.re.match
3807 except re.error:
3808 raise ValueError(f"invalid pattern {self.pattern!r} passed to Regex")
3810 self.errmsg = f"Expected {self.name}"
3811 self.mayIndexError = False
3812 self._may_return_empty = True
3814 def _generateDefaultName(self) -> str:
3815 if self.quote_char == self.end_quote_char and isinstance(
3816 self.quote_char, str_type
3817 ):
3818 return f"string enclosed in {self.quote_char!r}"
3820 return f"quoted string, starting with {self.quote_char} ending with {self.end_quote_char}"
3822 def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType:
3823 # check first character of opening quote to see if that is a match
3824 # before doing the more complicated regex match
3825 result = (
3826 instring[loc] == self.first_quote_char
3827 and self.re_match(instring, loc)
3828 or None
3829 )
3830 if not result:
3831 raise ParseException(instring, loc, self.errmsg, self)
3833 # get ending loc and matched string from regex matching result
3834 loc = result.end()
3835 ret = result[0]
3837 if self.unquote_results:
3838 # strip off quotes
3839 ret = ret[self.quote_char_len : -self.end_quote_char_len]
3841 if isinstance(ret, str_type):
3842 # fmt: off
3843 if self.convert_whitespace_escapes:
3844 # as we iterate over matches in the input string,
3845 # collect from whichever match group of the unquote_scan_re
3846 # regex matches (only 1 group will match at any given time)
3847 ret = "".join(
3848 # match group 1 matches \t, \n, etc.
3849 self.ws_map[g] if (g := match[1])
3850 # match group 2 matches escaped octal, null, hex, and Unicode
3851 # sequences
3852 else _convert_escaped_numerics_to_char(g[1:]) if (g := match[2])
3853 # match group 3 matches escaped characters
3854 else g[-1] if (g := match[3])
3855 # match group 4 matches any character
3856 else match[4]
3857 for match in self.unquote_scan_re.finditer(ret)
3858 )
3859 else:
3860 ret = "".join(
3861 # match group 1 matches escaped characters
3862 g[-1] if (g := match[1])
3863 # match group 2 matches any character
3864 else match[2]
3865 for match in self.unquote_scan_re.finditer(ret)
3866 )
3867 # fmt: on
3869 # replace escaped quotes
3870 if self.esc_quote:
3871 ret = ret.replace(self.esc_quote, self.end_quote_char)
3873 return loc, ret
3876class CharsNotIn(Token):
3877 """Token for matching words composed of characters *not* in a given
3878 set (will include whitespace in matched characters if not listed in
3879 the provided exclusion set - see example). Defined with string
3880 containing all disallowed characters, and an optional minimum,
3881 maximum, and/or exact length. The default value for ``min`` is
3882 1 (a minimum value < 1 is not valid); the default values for
3883 ``max`` and ``exact`` are 0, meaning no maximum or exact
3884 length restriction.
3886 Example:
3888 .. testcode::
3890 # define a comma-separated-value as anything that is not a ','
3891 csv_value = CharsNotIn(',')
3892 print(
3893 DelimitedList(csv_value).parse_string(
3894 "dkls,lsdkjf,s12 34,@!#,213"
3895 )
3896 )
3898 prints:
3900 .. testoutput::
3902 ['dkls', 'lsdkjf', 's12 34', '@!#', '213']
3903 """
3905 def __init__(
3906 self, not_chars: str = "", min: int = 1, max: int = 0, exact: int = 0, **kwargs
3907 ) -> None:
3908 super().__init__()
3909 notChars: str = deprecate_argument(kwargs, "notChars", "")
3911 self.skipWhitespace = False
3912 self.notChars = not_chars or notChars
3913 self.notCharsSet = set(self.notChars)
3915 if min < 1:
3916 raise ValueError(
3917 "cannot specify a minimum length < 1; use"
3918 " Opt(CharsNotIn()) if zero-length char group is permitted"
3919 )
3921 self.minLen = min
3923 if max > 0:
3924 self.maxLen = max
3925 else:
3926 self.maxLen = _MAX_INT
3928 if exact > 0:
3929 self.maxLen = exact
3930 self.minLen = exact
3932 self.errmsg = f"Expected {self.name}"
3933 self._may_return_empty = self.minLen == 0
3934 self.mayIndexError = False
3936 def _generateDefaultName(self) -> str:
3937 not_chars_str = _collapse_string_to_ranges(self.notChars)
3938 if len(not_chars_str) > 16:
3939 return f"!W:({self.notChars[: 16 - 3]}...)"
3940 else:
3941 return f"!W:({self.notChars})"
3943 def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType:
3944 notchars = self.notCharsSet
3945 if instring[loc] in notchars:
3946 raise ParseException(instring, loc, self.errmsg, self)
3948 start = loc
3949 loc += 1
3950 maxlen = min(start + self.maxLen, len(instring))
3951 while loc < maxlen and instring[loc] not in notchars:
3952 loc += 1
3954 if loc - start < self.minLen:
3955 raise ParseException(instring, loc, self.errmsg, self)
3957 return loc, instring[start:loc]
3960class White(Token):
3961 """Special matching class for matching whitespace. Normally,
3962 whitespace is ignored by pyparsing grammars. This class is included
3963 when some whitespace structures are significant. Define with
3964 a string containing the whitespace characters to be matched; default
3965 is ``" \\t\\r\\n"``. Also takes optional ``min``,
3966 ``max``, and ``exact`` arguments, as defined for the
3967 :class:`Word` class.
3968 """
3970 whiteStrs = {
3971 " ": "<SP>",
3972 "\t": "<TAB>",
3973 "\n": "<LF>",
3974 "\r": "<CR>",
3975 "\f": "<FF>",
3976 "\u00a0": "<NBSP>",
3977 "\u1680": "<OGHAM_SPACE_MARK>",
3978 "\u180e": "<MONGOLIAN_VOWEL_SEPARATOR>",
3979 "\u2000": "<EN_QUAD>",
3980 "\u2001": "<EM_QUAD>",
3981 "\u2002": "<EN_SPACE>",
3982 "\u2003": "<EM_SPACE>",
3983 "\u2004": "<THREE-PER-EM_SPACE>",
3984 "\u2005": "<FOUR-PER-EM_SPACE>",
3985 "\u2006": "<SIX-PER-EM_SPACE>",
3986 "\u2007": "<FIGURE_SPACE>",
3987 "\u2008": "<PUNCTUATION_SPACE>",
3988 "\u2009": "<THIN_SPACE>",
3989 "\u200a": "<HAIR_SPACE>",
3990 "\u200b": "<ZERO_WIDTH_SPACE>",
3991 "\u202f": "<NNBSP>",
3992 "\u205f": "<MMSP>",
3993 "\u3000": "<IDEOGRAPHIC_SPACE>",
3994 }
3996 def __init__(
3997 self, ws: str = " \t\r\n", min: int = 1, max: int = 0, exact: int = 0
3998 ) -> None:
3999 super().__init__()
4000 self.matchWhite = ws
4001 self.set_whitespace_chars(
4002 "".join(c for c in self.whiteStrs if c not in self.matchWhite),
4003 copy_defaults=True,
4004 )
4005 # self.leave_whitespace()
4006 self._may_return_empty = True
4007 self.errmsg = f"Expected {self.name}"
4009 self.minLen = min
4011 if max > 0:
4012 self.maxLen = max
4013 else:
4014 self.maxLen = _MAX_INT
4016 if exact > 0:
4017 self.maxLen = exact
4018 self.minLen = exact
4020 def _generateDefaultName(self) -> str:
4021 return "".join(White.whiteStrs[c] for c in self.matchWhite)
4023 def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType:
4024 if instring[loc] not in self.matchWhite:
4025 raise ParseException(instring, loc, self.errmsg, self)
4026 start = loc
4027 loc += 1
4028 maxloc = start + self.maxLen
4029 maxloc = min(maxloc, len(instring))
4030 while loc < maxloc and instring[loc] in self.matchWhite:
4031 loc += 1
4033 if loc - start < self.minLen:
4034 raise ParseException(instring, loc, self.errmsg, self)
4036 return loc, instring[start:loc]
4039class PositionToken(Token):
4040 def __init__(self) -> None:
4041 super().__init__()
4042 self._may_return_empty = True
4043 self.mayIndexError = False
4046class GoToColumn(PositionToken):
4047 """Token to advance to a specific column of input text; useful for
4048 tabular report scraping.
4049 """
4051 def __init__(self, colno: int) -> None:
4052 super().__init__()
4053 self.col = colno
4055 def preParse(self, instring: str, loc: int) -> int:
4056 if col(loc, instring) == self.col:
4057 return loc
4059 instrlen = len(instring)
4060 if self.ignoreExprs:
4061 loc = self._skipIgnorables(instring, loc)
4062 while (
4063 loc < instrlen
4064 and instring[loc].isspace()
4065 and col(loc, instring) != self.col
4066 ):
4067 loc += 1
4069 return loc
4071 def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType:
4072 thiscol = col(loc, instring)
4073 if thiscol > self.col:
4074 raise ParseException(instring, loc, "Text not in expected column", self)
4075 newloc = loc + self.col - thiscol
4076 ret = instring[loc:newloc]
4077 return newloc, ret
4080class LineStart(PositionToken):
4081 r"""Matches if current position is at the logical beginning of a line (after skipping whitespace)
4082 within the parse string
4084 Example:
4086 .. testcode::
4088 test = '''\
4089 AAA this line
4090 AAA and this line
4091 AAA and even this line
4092 B AAA but definitely not this line
4093 '''
4095 for t in (LineStart() + 'AAA' + rest_of_line).search_string(test):
4096 print(t)
4098 prints:
4100 .. testoutput::
4102 ['AAA', ' this line']
4103 ['AAA', ' and this line']
4104 ['AAA', ' and even this line']
4106 """
4108 def __init__(self) -> None:
4109 super().__init__()
4110 self.leave_whitespace()
4111 self.orig_whiteChars = set() | self.whiteChars
4112 self.whiteChars.discard("\n")
4113 self.skipper = Empty().set_whitespace_chars(self.whiteChars)
4114 self.set_name("start of line")
4116 def preParse(self, instring: str, loc: int) -> int:
4117 if loc == 0:
4118 return loc
4120 ret = self.skipper.preParse(instring, loc)
4122 if "\n" in self.orig_whiteChars:
4123 while instring[ret : ret + 1] == "\n":
4124 ret = self.skipper.preParse(instring, ret + 1)
4126 return ret
4128 def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType:
4129 if col(loc, instring) == 1:
4130 return loc, []
4131 raise ParseException(instring, loc, self.errmsg, self)
4134class LineEnd(PositionToken):
4135 """Matches if current position is at the end of a line within the
4136 parse string
4137 """
4139 def __init__(self) -> None:
4140 super().__init__()
4141 self.whiteChars.discard("\n")
4142 self.set_whitespace_chars(self.whiteChars, copy_defaults=False)
4143 self.set_name("end of line")
4145 def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType:
4146 if loc < len(instring):
4147 if instring[loc] == "\n":
4148 return loc + 1, "\n"
4149 else:
4150 raise ParseException(instring, loc, self.errmsg, self)
4151 elif loc == len(instring):
4152 return loc + 1, []
4153 else:
4154 raise ParseException(instring, loc, self.errmsg, self)
4157class StringStart(PositionToken):
4158 """Matches if current position is at the beginning of the parse
4159 string
4160 """
4162 def __init__(self) -> None:
4163 super().__init__()
4164 self.set_name("start of text")
4166 def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType:
4167 # see if entire string up to here is just whitespace and ignoreables
4168 if loc != 0 and loc != self.preParse(instring, 0):
4169 raise ParseException(instring, loc, self.errmsg, self)
4171 return loc, []
4174class StringEnd(PositionToken):
4175 """
4176 Matches if current position is at the end of the parse string
4177 """
4179 def __init__(self) -> None:
4180 super().__init__()
4181 self.set_name("end of text")
4183 def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType:
4184 if loc < len(instring):
4185 raise ParseException(instring, loc, self.errmsg, self)
4186 if loc == len(instring):
4187 return loc + 1, []
4188 if loc > len(instring):
4189 return loc, []
4191 raise ParseException(instring, loc, self.errmsg, self)
4194class WordStart(PositionToken):
4195 """Matches if the current position is at the beginning of a
4196 :class:`Word`, and is not preceded by any character in a given
4197 set of ``word_chars`` (default= ``printables``). To emulate the
4198 ``\b`` behavior of regular expressions, use
4199 ``WordStart(alphanums)``. ``WordStart`` will also match at
4200 the beginning of the string being parsed, or at the beginning of
4201 a line.
4202 """
4204 def __init__(self, word_chars: str = printables, **kwargs) -> None:
4205 wordChars: str = deprecate_argument(kwargs, "wordChars", printables)
4207 wordChars = word_chars if wordChars == printables else wordChars
4208 super().__init__()
4209 self.wordChars = set(wordChars)
4210 self.set_name("start of a word")
4212 def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType:
4213 if loc != 0:
4214 if (
4215 instring[loc - 1] in self.wordChars
4216 or instring[loc] not in self.wordChars
4217 ):
4218 raise ParseException(instring, loc, self.errmsg, self)
4219 return loc, []
4222class WordEnd(PositionToken):
4223 """Matches if the current position is at the end of a :class:`Word`,
4224 and is not followed by any character in a given set of ``word_chars``
4225 (default= ``printables``). To emulate the ``\b`` behavior of
4226 regular expressions, use ``WordEnd(alphanums)``. ``WordEnd``
4227 will also match at the end of the string being parsed, or at the end
4228 of a line.
4229 """
4231 def __init__(self, word_chars: str = printables, **kwargs) -> None:
4232 wordChars: str = deprecate_argument(kwargs, "wordChars", printables)
4234 wordChars = word_chars if wordChars == printables else wordChars
4235 super().__init__()
4236 self.wordChars = set(wordChars)
4237 self.skipWhitespace = False
4238 self.set_name("end of a word")
4240 def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType:
4241 instrlen = len(instring)
4242 if instrlen > 0 and loc < instrlen:
4243 if (
4244 instring[loc] in self.wordChars
4245 or instring[loc - 1] not in self.wordChars
4246 ):
4247 raise ParseException(instring, loc, self.errmsg, self)
4248 return loc, []
4251class Tag(Token):
4252 """
4253 A meta-element for inserting a named result into the parsed
4254 tokens that may be checked later in a parse action or while
4255 processing the parsed results. Accepts an optional tag value,
4256 defaulting to `True`.
4258 Example:
4260 .. doctest::
4262 >>> end_punc = "." | ("!" + Tag("enthusiastic"))
4263 >>> greeting = "Hello," + Word(alphas) + end_punc
4265 >>> result = greeting.parse_string("Hello, World.")
4266 >>> print(result.dump())
4267 ['Hello,', 'World', '.']
4269 >>> result = greeting.parse_string("Hello, World!")
4270 >>> print(result.dump())
4271 ['Hello,', 'World', '!']
4272 - enthusiastic: True
4274 .. versionadded:: 3.1.0
4275 """
4277 def __init__(self, tag_name: str, value: Any = True) -> None:
4278 super().__init__()
4279 self._may_return_empty = True
4280 self.mayIndexError = False
4281 self.leave_whitespace()
4282 self.tag_name = tag_name
4283 self.tag_value = value
4284 self.add_parse_action(self._add_tag)
4285 self.show_in_diagram = False
4287 def _add_tag(self, tokens: ParseResults):
4288 tokens[self.tag_name] = self.tag_value
4290 def _generateDefaultName(self) -> str:
4291 return f"{type(self).__name__}:{self.tag_name}={self.tag_value!r}"
4294class ParseExpression(ParserElement):
4295 """Abstract subclass of ParserElement, for combining and
4296 post-processing parsed tokens.
4297 """
4299 def __init__(
4300 self, exprs: typing.Iterable[ParserElement], savelist: bool = False
4301 ) -> None:
4302 super().__init__(savelist)
4303 self.exprs: list[ParserElement]
4304 if isinstance(exprs, _generatorType):
4305 exprs = list(exprs)
4307 if isinstance(exprs, str_type):
4308 self.exprs = [self._literalStringClass(exprs)]
4309 elif isinstance(exprs, ParserElement):
4310 self.exprs = [exprs]
4311 elif isinstance(exprs, Iterable):
4312 exprs = list(exprs)
4313 # if sequence of strings provided, wrap with Literal
4314 if any(isinstance(expr, str_type) for expr in exprs):
4315 exprs = (
4316 self._literalStringClass(e) if isinstance(e, str_type) else e
4317 for e in exprs
4318 )
4319 self.exprs = list(exprs)
4320 else:
4321 try:
4322 self.exprs = list(exprs)
4323 except TypeError:
4324 self.exprs = [exprs]
4325 self.callPreparse = False
4327 def recurse(self) -> list[ParserElement]:
4328 return self.exprs[:]
4330 def append(self, other) -> ParserElement:
4331 """
4332 Add an expression to the list of expressions related to this ParseExpression instance.
4333 """
4334 self.exprs.append(other)
4335 self._defaultName = None
4336 return self
4338 def leave_whitespace(self, recursive: bool = True) -> ParserElement:
4339 """
4340 Extends ``leave_whitespace`` defined in base class, and also invokes ``leave_whitespace`` on
4341 all contained expressions.
4342 """
4343 super().leave_whitespace(recursive)
4345 if recursive:
4346 self.exprs = [e.copy() for e in self.exprs]
4347 for e in self.exprs:
4348 e.leave_whitespace(recursive)
4349 return self
4351 def ignore_whitespace(self, recursive: bool = True) -> ParserElement:
4352 """
4353 Extends ``ignore_whitespace`` defined in base class, and also invokes ``ignore_whitespace`` on
4354 all contained expressions.
4355 """
4356 super().ignore_whitespace(recursive)
4357 if recursive:
4358 self.exprs = [e.copy() for e in self.exprs]
4359 for e in self.exprs:
4360 e.ignore_whitespace(recursive)
4361 return self
4363 def ignore(self, other) -> ParserElement:
4364 """
4365 Define expression to be ignored (e.g., comments) while doing pattern
4366 matching; may be called repeatedly, to define multiple comment or other
4367 ignorable patterns.
4368 """
4369 if isinstance(other, Suppress):
4370 if other not in self.ignoreExprs:
4371 super().ignore(other)
4372 for e in self.exprs:
4373 e.ignore(self.ignoreExprs[-1])
4374 else:
4375 super().ignore(other)
4376 for e in self.exprs:
4377 e.ignore(self.ignoreExprs[-1])
4378 return self
4380 def _generateDefaultName(self) -> str:
4381 return f"{type(self).__name__}:({self.exprs})"
4383 def streamline(self) -> ParserElement:
4384 if self.streamlined:
4385 return self
4387 super().streamline()
4389 for e in self.exprs:
4390 e.streamline()
4392 # collapse nested :class:`And`'s of the form ``And(And(And(a, b), c), d)`` to ``And(a, b, c, d)``
4393 # but only if there are no parse actions or resultsNames on the nested And's
4394 # (likewise for :class:`Or`'s and :class:`MatchFirst`'s)
4395 if len(self.exprs) == 2:
4396 first, second = self.exprs
4397 if (
4398 isinstance(first, self.__class__)
4399 and not first.parseAction
4400 and first.resultsName is None
4401 and not first.debug
4402 ):
4403 self.exprs[:] = (*first.exprs, second)
4404 self._defaultName = None
4405 self._may_return_empty |= first.mayReturnEmpty
4406 self.mayIndexError |= first.mayIndexError
4408 last = self.exprs[-1]
4409 if (
4410 isinstance(last, self.__class__)
4411 and not last.parseAction
4412 and last.resultsName is None
4413 and not last.debug
4414 ):
4415 self.exprs[-1:] = last.exprs
4416 self._defaultName = None
4417 self._may_return_empty |= last.mayReturnEmpty
4418 self.mayIndexError |= last.mayIndexError
4420 self.errmsg = f"Expected {self}"
4422 return self
4424 def validate(self, validateTrace=None) -> None:
4425 warnings.warn(
4426 "ParserElement.validate() is deprecated, and should not be used to check for left recursion",
4427 PyparsingDeprecationWarning,
4428 stacklevel=2,
4429 )
4430 tmp = (validateTrace if validateTrace is not None else [])[:] + [self]
4431 for e in self.exprs:
4432 e.validate(tmp)
4433 self._checkRecursion([])
4435 def copy(self) -> ParserElement:
4436 """
4437 Returns a copy of this expression.
4439 Generally only used internally by pyparsing.
4440 """
4441 ret = super().copy()
4442 ret = typing.cast(ParseExpression, ret)
4443 ret.exprs = [e.copy() for e in self.exprs]
4444 return ret
4446 def _setResultsName(self, name, list_all_matches=False) -> ParserElement:
4447 if not (
4448 __diag__.warn_ungrouped_named_tokens_in_collection
4449 and Diagnostics.warn_ungrouped_named_tokens_in_collection
4450 not in self.suppress_warnings_
4451 ):
4452 return super()._setResultsName(name, list_all_matches)
4454 for e in self.exprs:
4455 if (
4456 isinstance(e, ParserElement)
4457 and e.resultsName
4458 and (
4459 Diagnostics.warn_ungrouped_named_tokens_in_collection
4460 not in e.suppress_warnings_
4461 )
4462 ):
4463 warning = (
4464 "warn_ungrouped_named_tokens_in_collection:"
4465 f" setting results name {name!r} on {type(self).__name__} expression"
4466 f" collides with {e.resultsName!r} on contained expression"
4467 )
4468 warnings.warn(warning, PyparsingDiagnosticWarning, stacklevel=3)
4469 break
4471 return super()._setResultsName(name, list_all_matches)
4473 # Compatibility synonyms
4474 # fmt: off
4475 leaveWhitespace = replaced_by_pep8("leaveWhitespace", leave_whitespace)
4476 ignoreWhitespace = replaced_by_pep8("ignoreWhitespace", ignore_whitespace)
4477 # fmt: on
4480class And(ParseExpression):
4481 """
4482 Requires all given :class:`ParserElement` s to be found in the given order.
4483 Expressions may be separated by whitespace.
4484 May be constructed using the ``'+'`` operator.
4485 May also be constructed using the ``'-'`` operator, which will
4486 suppress backtracking.
4488 Example:
4490 .. testcode::
4492 integer = Word(nums)
4493 name_expr = Word(alphas)[1, ...]
4495 expr = And([integer("id"), name_expr("name"), integer("age")])
4496 # more easily written as:
4497 expr = integer("id") + name_expr("name") + integer("age")
4498 """
4500 class _ErrorStop(Empty):
4501 def __init__(self, *args, **kwargs) -> None:
4502 super().__init__(*args, **kwargs)
4503 self.leave_whitespace()
4505 def _generateDefaultName(self) -> str:
4506 return "-"
4508 def __init__(
4509 self,
4510 exprs_arg: typing.Iterable[Union[ParserElement, str]],
4511 savelist: bool = True,
4512 ) -> None:
4513 # instantiate exprs as a list, converting strs to ParserElements
4514 exprs: list[ParserElement] = [
4515 self._literalStringClass(e) if isinstance(e, str) else e for e in exprs_arg
4516 ]
4518 # convert any Ellipsis elements to SkipTo
4519 if Ellipsis in exprs:
4521 # Ellipsis cannot be the last element
4522 if exprs[-1] is Ellipsis:
4523 raise Exception("cannot construct And with sequence ending in ...")
4525 tmp: list[ParserElement] = []
4526 for cur_expr, next_expr in zip(exprs, exprs[1:]):
4527 if cur_expr is Ellipsis:
4528 tmp.append(SkipTo(next_expr)("_skipped*"))
4529 else:
4530 tmp.append(cur_expr)
4532 exprs[:-1] = tmp
4534 super().__init__(exprs, savelist)
4535 if self.exprs:
4536 self._may_return_empty = all(e.mayReturnEmpty for e in self.exprs)
4537 if not isinstance(self.exprs[0], White):
4538 self.set_whitespace_chars(
4539 self.exprs[0].whiteChars,
4540 copy_defaults=self.exprs[0].copyDefaultWhiteChars,
4541 )
4542 self.skipWhitespace = self.exprs[0].skipWhitespace
4543 else:
4544 self.skipWhitespace = False
4545 else:
4546 self._may_return_empty = True
4547 self.callPreparse = True
4549 def streamline(self) -> ParserElement:
4550 """
4551 Collapse `And` expressions like `And(And(And(A, B), C), D)`
4552 to `And(A, B, C, D)`.
4554 .. doctest::
4556 >>> expr = Word("A") + Word("B") + Word("C") + Word("D")
4557 >>> # Using '+' operator creates nested And expression
4558 >>> expr
4559 {{{W:(A) W:(B)} W:(C)} W:(D)}
4560 >>> # streamline simplifies to a single And with multiple expressions
4561 >>> expr.streamline()
4562 {W:(A) W:(B) W:(C) W:(D)}
4564 Guards against collapsing out expressions that have special features,
4565 such as results names or parse actions.
4567 Resolves pending Skip commands defined using `...` terms.
4568 """
4569 # collapse any _PendingSkip's
4570 if self.exprs and any(
4571 isinstance(e, ParseExpression)
4572 and e.exprs
4573 and isinstance(e.exprs[-1], _PendingSkip)
4574 for e in self.exprs[:-1]
4575 ):
4576 deleted_expr_marker = NoMatch()
4577 for i, e in enumerate(self.exprs[:-1]):
4578 if e is deleted_expr_marker:
4579 continue
4580 if (
4581 isinstance(e, ParseExpression)
4582 and e.exprs
4583 and isinstance(e.exprs[-1], _PendingSkip)
4584 ):
4585 e.exprs[-1] = e.exprs[-1] + self.exprs[i + 1]
4586 self.exprs[i + 1] = deleted_expr_marker
4587 self.exprs = [e for e in self.exprs if e is not deleted_expr_marker]
4589 super().streamline()
4591 # link any IndentedBlocks to the prior expression
4592 prev: ParserElement
4593 cur: ParserElement
4594 for prev, cur in zip(self.exprs, self.exprs[1:]):
4595 # traverse cur or any first embedded expr of cur looking for an IndentedBlock
4596 # (but watch out for recursive grammar)
4597 seen = set()
4598 while True:
4599 if id(cur) in seen:
4600 break
4601 seen.add(id(cur))
4602 if isinstance(cur, IndentedBlock):
4603 prev.add_parse_action(
4604 lambda s, l, t, cur_=cur: setattr(
4605 cur_, "parent_anchor", col(l, s)
4606 )
4607 )
4608 break
4609 subs = cur.recurse()
4610 next_first = next(iter(subs), None)
4611 if next_first is None:
4612 break
4613 cur = typing.cast(ParserElement, next_first)
4615 self._may_return_empty = all(e.mayReturnEmpty for e in self.exprs)
4616 return self
4618 def parseImpl(self, instring, loc, do_actions=True):
4620 # if no exprs defined, assume we contain a single Empty
4621 # (consistent with behavior of `all([])` returning True)
4622 exprs = iter(self.exprs or (Empty(),))
4624 # pass False as callPreParse arg to _parse for first element, since we already
4625 # pre-parsed the string as part of our And pre-parsing
4626 loc, resultlist = next(exprs)._parse(
4627 instring, loc, do_actions, callPreParse=False
4628 )
4630 # iterate over remaining expressions
4631 raise_syntax_error_immediately = False
4632 for e in exprs:
4633 # if isinstance(e, And._ErrorStop):
4634 if type(e) is And._ErrorStop:
4635 raise_syntax_error_immediately = True
4636 continue
4638 if raise_syntax_error_immediately:
4639 try:
4640 loc, exprtokens = e._parse(instring, loc, do_actions)
4641 except ParseSyntaxException:
4642 raise
4643 except ParseBaseException as pe:
4644 pe.__traceback__ = None
4645 raise ParseSyntaxException._from_exception(pe)
4646 except IndexError:
4647 raise ParseSyntaxException(
4648 instring, len(instring), self.errmsg, self
4649 )
4650 else:
4651 loc, exprtokens = e._parse(instring, loc, do_actions)
4652 resultlist += exprtokens
4653 return loc, resultlist
4655 def __iadd__(self, other):
4656 if isinstance(other, str_type):
4657 other = self._literalStringClass(other)
4658 if not isinstance(other, ParserElement):
4659 return NotImplemented
4660 return self.append(other) # And([self, other])
4662 def _checkRecursion(self, parseElementList):
4663 subRecCheckList = parseElementList[:] + [self]
4664 for e in self.exprs:
4665 e._checkRecursion(subRecCheckList)
4666 if not e.mayReturnEmpty:
4667 break
4669 def _generateDefaultName(self) -> str:
4670 inner = " ".join(str(e) for e in self.exprs)
4671 # strip off redundant inner {}'s
4672 while len(inner) > 1 and inner[0 :: len(inner) - 1] == "{}":
4673 inner = inner[1:-1]
4674 return f"{{{inner}}}"
4677class Or(ParseExpression):
4678 """Requires that at least one :class:`ParserElement` is found. If
4679 two expressions match, the expression that matches the longest
4680 string will be used. May be constructed using the ``'^'``
4681 operator.
4683 Example:
4685 .. testcode::
4687 # construct Or using '^' operator
4689 number = Word(nums) ^ Combine(Word(nums) + '.' + Word(nums))
4690 print(number.search_string("123 3.1416 789"))
4692 prints:
4694 .. testoutput::
4696 [['123'], ['3.1416'], ['789']]
4697 """
4699 def __init__(
4700 self, exprs: typing.Iterable[ParserElement], savelist: bool = False
4701 ) -> None:
4702 super().__init__(exprs, savelist)
4703 if self.exprs:
4704 self._may_return_empty = any(e.mayReturnEmpty for e in self.exprs)
4705 self.skipWhitespace = all(e.skipWhitespace for e in self.exprs)
4706 else:
4707 self._may_return_empty = True
4709 def streamline(self) -> ParserElement:
4710 super().streamline()
4711 if self.exprs:
4712 self._may_return_empty = any(e.mayReturnEmpty for e in self.exprs)
4713 self.saveAsList = any(e.saveAsList for e in self.exprs)
4714 self.skipWhitespace = all(
4715 e.skipWhitespace and not isinstance(e, White) for e in self.exprs
4716 )
4717 else:
4718 self.saveAsList = False
4719 return self
4721 def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType:
4722 maxExcLoc = -1
4723 maxException = None
4724 matches: list[tuple[int, ParserElement]] = []
4725 fatals: list[ParseFatalException] = []
4726 if all(e.callPreparse for e in self.exprs):
4727 loc = self.preParse(instring, loc)
4728 for e in self.exprs:
4729 try:
4730 loc2 = e.try_parse(instring, loc, raise_fatal=True)
4731 except ParseFatalException as pfe:
4732 pfe.__traceback__ = None
4733 pfe.parser_element = e
4734 fatals.append(pfe)
4735 maxException = None
4736 maxExcLoc = -1
4737 except ParseException as err:
4738 if not fatals:
4739 err.__traceback__ = None
4740 if err.loc > maxExcLoc:
4741 maxException = err
4742 maxExcLoc = err.loc
4743 except IndexError:
4744 if len(instring) > maxExcLoc:
4745 maxException = ParseException(
4746 instring, len(instring), e.errmsg, self
4747 )
4748 maxExcLoc = len(instring)
4749 else:
4750 # save match among all matches, to retry longest to shortest
4751 matches.append((loc2, e))
4753 if matches:
4754 # re-evaluate all matches in descending order of length of match, in case attached actions
4755 # might change whether or how much they match of the input.
4756 matches.sort(key=itemgetter(0), reverse=True)
4758 if not do_actions:
4759 # no further conditions or parse actions to change the selection of
4760 # alternative, so the first match will be the best match
4761 best_expr = matches[0][1]
4762 return best_expr._parse(instring, loc, do_actions)
4764 longest: tuple[int, typing.Optional[ParseResults]] = -1, None
4765 for loc1, expr1 in matches:
4766 if loc1 <= longest[0]:
4767 # already have a longer match than this one will deliver, we are done
4768 return longest
4770 try:
4771 loc2, toks = expr1._parse(instring, loc, do_actions)
4772 except ParseException as err:
4773 err.__traceback__ = None
4774 if err.loc > maxExcLoc:
4775 maxException = err
4776 maxExcLoc = err.loc
4777 else:
4778 if loc2 >= loc1:
4779 return loc2, toks
4780 # didn't match as much as before
4781 elif loc2 > longest[0]:
4782 longest = loc2, toks
4784 if longest != (-1, None):
4785 return longest
4787 if fatals:
4788 if len(fatals) > 1:
4789 fatals.sort(key=lambda e: -e.loc)
4790 if fatals[0].loc == fatals[1].loc:
4791 fatals.sort(key=lambda e: (-e.loc, -len(str(e.parser_element))))
4792 max_fatal = fatals[0]
4793 raise max_fatal
4795 if maxException is not None:
4796 # infer from this check that all alternatives failed at the current position
4797 # so emit this collective error message instead of any single error message
4798 parse_start_loc = self.preParse(instring, loc)
4799 if maxExcLoc == parse_start_loc:
4800 maxException.msg = self.errmsg or ""
4801 raise maxException
4803 raise ParseException(instring, loc, "no defined alternatives to match", self)
4805 def __ixor__(self, other):
4806 if isinstance(other, str_type):
4807 other = self._literalStringClass(other)
4808 if not isinstance(other, ParserElement):
4809 return NotImplemented
4810 return self.append(other) # Or([self, other])
4812 def _generateDefaultName(self) -> str:
4813 return f"{{{' ^ '.join(str(e) for e in self.exprs)}}}"
4815 def _setResultsName(self, name, list_all_matches=False) -> ParserElement:
4816 if (
4817 __diag__.warn_multiple_tokens_in_named_alternation
4818 and Diagnostics.warn_multiple_tokens_in_named_alternation
4819 not in self.suppress_warnings_
4820 ):
4821 if any(
4822 isinstance(e, And)
4823 and Diagnostics.warn_multiple_tokens_in_named_alternation
4824 not in e.suppress_warnings_
4825 for e in self.exprs
4826 ):
4827 warning = (
4828 "warn_multiple_tokens_in_named_alternation:"
4829 f" setting results name {name!r} on {type(self).__name__} expression"
4830 " will return a list of all parsed tokens in an And alternative,"
4831 " in prior versions only the first token was returned; enclose"
4832 " contained argument in Group"
4833 )
4834 warnings.warn(warning, PyparsingDiagnosticWarning, stacklevel=3)
4836 return super()._setResultsName(name, list_all_matches)
4839class MatchFirst(ParseExpression):
4840 """Requires that at least one :class:`ParserElement` is found. If
4841 more than one expression matches, the first one listed is the one that will
4842 match. May be constructed using the ``'|'`` operator.
4844 Example: Construct MatchFirst using '|' operator
4846 .. doctest::
4848 # watch the order of expressions to match
4849 >>> number = Word(nums) | Combine(Word(nums) + '.' + Word(nums))
4850 >>> print(number.search_string("123 3.1416 789")) # Fail!
4851 [['123'], ['3'], ['1416'], ['789']]
4853 # put more selective expression first
4854 >>> number = Combine(Word(nums) + '.' + Word(nums)) | Word(nums)
4855 >>> print(number.search_string("123 3.1416 789")) # Better
4856 [['123'], ['3.1416'], ['789']]
4857 """
4859 def __init__(
4860 self, exprs: typing.Iterable[ParserElement], savelist: bool = False
4861 ) -> None:
4862 super().__init__(exprs, savelist)
4863 if self.exprs:
4864 self._may_return_empty = any(e.mayReturnEmpty for e in self.exprs)
4865 self.skipWhitespace = all(e.skipWhitespace for e in self.exprs)
4866 else:
4867 self._may_return_empty = True
4869 def streamline(self) -> ParserElement:
4870 if self.streamlined:
4871 return self
4873 super().streamline()
4874 if self.exprs:
4875 self.saveAsList = any(e.saveAsList for e in self.exprs)
4876 self._may_return_empty = any(e.mayReturnEmpty for e in self.exprs)
4877 self.skipWhitespace = all(
4878 e.skipWhitespace and not isinstance(e, White) for e in self.exprs
4879 )
4880 else:
4881 self.saveAsList = False
4882 self._may_return_empty = True
4883 return self
4885 def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType:
4886 maxExcLoc = -1
4887 maxException = None
4889 for e in self.exprs:
4890 try:
4891 return e._parse(instring, loc, do_actions)
4892 except ParseFatalException as pfe:
4893 pfe.__traceback__ = None
4894 pfe.parser_element = e
4895 raise
4896 except ParseException as err:
4897 if err.loc > maxExcLoc:
4898 maxException = err
4899 maxExcLoc = err.loc
4900 except IndexError:
4901 if len(instring) > maxExcLoc:
4902 maxException = ParseException(
4903 instring, len(instring), e.errmsg, self
4904 )
4905 maxExcLoc = len(instring)
4907 if maxException is not None:
4908 # infer from this check that all alternatives failed at the current position
4909 # so emit this collective error message instead of any individual error message
4910 parse_start_loc = self.preParse(instring, loc)
4911 if maxExcLoc == parse_start_loc:
4912 maxException.msg = self.errmsg or ""
4913 raise maxException
4915 raise ParseException(instring, loc, "no defined alternatives to match", self)
4917 def __ior__(self, other):
4918 if isinstance(other, str_type):
4919 other = self._literalStringClass(other)
4920 if not isinstance(other, ParserElement):
4921 return NotImplemented
4922 return self.append(other) # MatchFirst([self, other])
4924 def _generateDefaultName(self) -> str:
4925 return f"{{{' | '.join(str(e) for e in self.exprs)}}}"
4927 def _setResultsName(self, name, list_all_matches=False) -> ParserElement:
4928 if (
4929 __diag__.warn_multiple_tokens_in_named_alternation
4930 and Diagnostics.warn_multiple_tokens_in_named_alternation
4931 not in self.suppress_warnings_
4932 ):
4933 if any(
4934 isinstance(e, And)
4935 and Diagnostics.warn_multiple_tokens_in_named_alternation
4936 not in e.suppress_warnings_
4937 for e in self.exprs
4938 ):
4939 warning = (
4940 "warn_multiple_tokens_in_named_alternation:"
4941 f" setting results name {name!r} on {type(self).__name__} expression"
4942 " will return a list of all parsed tokens in an And alternative,"
4943 " in prior versions only the first token was returned; enclose"
4944 " contained argument in Group"
4945 )
4946 warnings.warn(warning, PyparsingDiagnosticWarning, stacklevel=3)
4948 return super()._setResultsName(name, list_all_matches)
4951class Each(ParseExpression):
4952 """Requires all given :class:`ParserElement` s to be found, but in
4953 any order. Expressions may be separated by whitespace.
4955 May be constructed using the ``'&'`` operator.
4957 Example:
4959 .. testcode::
4961 color = one_of("RED ORANGE YELLOW GREEN BLUE PURPLE BLACK WHITE BROWN")
4962 shape_type = one_of("SQUARE CIRCLE TRIANGLE STAR HEXAGON OCTAGON")
4963 integer = Word(nums)
4964 shape_attr = "shape:" + shape_type("shape")
4965 posn_attr = "posn:" + Group(integer("x") + ',' + integer("y"))("posn")
4966 color_attr = "color:" + color("color")
4967 size_attr = "size:" + integer("size")
4969 # use Each (using operator '&') to accept attributes in any order
4970 # (shape and posn are required, color and size are optional)
4971 shape_spec = shape_attr & posn_attr & Opt(color_attr) & Opt(size_attr)
4973 shape_spec.run_tests('''
4974 shape: SQUARE color: BLACK posn: 100, 120
4975 shape: CIRCLE size: 50 color: BLUE posn: 50,80
4976 color:GREEN size:20 shape:TRIANGLE posn:20,40
4977 '''
4978 )
4980 prints:
4982 .. testoutput::
4983 :options: +NORMALIZE_WHITESPACE
4986 shape: SQUARE color: BLACK posn: 100, 120
4987 ['shape:', 'SQUARE', 'color:', 'BLACK', 'posn:', ['100', ',', '120']]
4988 - color: 'BLACK'
4989 - posn: ['100', ',', '120']
4990 - x: '100'
4991 - y: '120'
4992 - shape: 'SQUARE'
4993 ...
4995 shape: CIRCLE size: 50 color: BLUE posn: 50,80
4996 ['shape:', 'CIRCLE', 'size:', '50', 'color:', 'BLUE',
4997 'posn:', ['50', ',', '80']]
4998 - color: 'BLUE'
4999 - posn: ['50', ',', '80']
5000 - x: '50'
5001 - y: '80'
5002 - shape: 'CIRCLE'
5003 - size: '50'
5004 ...
5006 color:GREEN size:20 shape:TRIANGLE posn:20,40
5007 ['color:', 'GREEN', 'size:', '20', 'shape:', 'TRIANGLE',
5008 'posn:', ['20', ',', '40']]
5009 - color: 'GREEN'
5010 - posn: ['20', ',', '40']
5011 - x: '20'
5012 - y: '40'
5013 - shape: 'TRIANGLE'
5014 - size: '20'
5015 ...
5016 """
5018 def __init__(
5019 self, exprs: typing.Iterable[ParserElement], savelist: bool = True
5020 ) -> None:
5021 super().__init__(exprs, savelist)
5022 if self.exprs:
5023 self._may_return_empty = all(e.mayReturnEmpty for e in self.exprs)
5024 else:
5025 self._may_return_empty = True
5026 self.skipWhitespace = True
5027 self.initExprGroups = True
5028 self.saveAsList = True
5030 def __iand__(self, other):
5031 if isinstance(other, str_type):
5032 other = self._literalStringClass(other)
5033 if not isinstance(other, ParserElement):
5034 return NotImplemented
5035 return self.append(other) # Each([self, other])
5037 def streamline(self) -> ParserElement:
5038 super().streamline()
5039 if self.exprs:
5040 self._may_return_empty = all(e.mayReturnEmpty for e in self.exprs)
5041 else:
5042 self._may_return_empty = True
5043 return self
5045 def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType:
5046 if self.initExprGroups:
5047 self.opt1map = dict(
5048 (id(e.expr), e) for e in self.exprs if isinstance(e, Opt)
5049 )
5050 opt1 = [e.expr for e in self.exprs if isinstance(e, Opt)]
5051 opt2 = [
5052 e
5053 for e in self.exprs
5054 if e.mayReturnEmpty and not isinstance(e, (Opt, Regex, ZeroOrMore))
5055 ]
5056 self.optionals = opt1 + opt2
5057 self.multioptionals = [
5058 e.expr.set_results_name(e.resultsName, list_all_matches=True)
5059 for e in self.exprs
5060 if isinstance(e, _MultipleMatch)
5061 ]
5062 self.multirequired = [
5063 e.expr.set_results_name(e.resultsName, list_all_matches=True)
5064 for e in self.exprs
5065 if isinstance(e, OneOrMore)
5066 ]
5067 self.required = [
5068 e for e in self.exprs if not isinstance(e, (Opt, ZeroOrMore, OneOrMore))
5069 ]
5070 self.required += self.multirequired
5071 self.initExprGroups = False
5073 tmpLoc = loc
5074 tmpReqd = self.required[:]
5075 tmpOpt = self.optionals[:]
5076 multis = self.multioptionals[:]
5077 matchOrder: list[ParserElement] = []
5079 keepMatching = True
5080 failed: list[ParserElement] = []
5081 fatals: list[ParseFatalException] = []
5082 while keepMatching:
5083 tmpExprs = tmpReqd + tmpOpt + multis
5084 failed.clear()
5085 fatals.clear()
5086 for e in tmpExprs:
5087 try:
5088 tmpLoc = e.try_parse(instring, tmpLoc, raise_fatal=True)
5089 except ParseFatalException as pfe:
5090 pfe.__traceback__ = None
5091 pfe.parser_element = e
5092 fatals.append(pfe)
5093 failed.append(e)
5094 except ParseException:
5095 failed.append(e)
5096 else:
5097 matchOrder.append(self.opt1map.get(id(e), e))
5098 if e in tmpReqd:
5099 tmpReqd.remove(e)
5100 elif e in tmpOpt:
5101 tmpOpt.remove(e)
5102 if len(failed) == len(tmpExprs):
5103 keepMatching = False
5105 # look for any ParseFatalExceptions
5106 if fatals:
5107 if len(fatals) > 1:
5108 fatals.sort(key=lambda e: -e.loc)
5109 if fatals[0].loc == fatals[1].loc:
5110 fatals.sort(key=lambda e: (-e.loc, -len(str(e.parser_element))))
5111 max_fatal = fatals[0]
5112 raise max_fatal
5114 if tmpReqd:
5115 missing = ", ".join([str(e) for e in tmpReqd])
5116 raise ParseException(
5117 instring,
5118 loc,
5119 f"Missing one or more required elements ({missing})",
5120 )
5122 # add any unmatched Opts, in case they have default values defined
5123 matchOrder += [e for e in self.exprs if isinstance(e, Opt) and e.expr in tmpOpt]
5125 total_results = ParseResults([])
5126 for e in matchOrder:
5127 loc, results = e._parse(instring, loc, do_actions)
5128 total_results += results
5130 return loc, total_results
5132 def _generateDefaultName(self) -> str:
5133 return f"{{{' & '.join(str(e) for e in self.exprs)}}}"
5136class ParseElementEnhance(ParserElement):
5137 """Abstract subclass of :class:`ParserElement`, for combining and
5138 post-processing parsed tokens.
5139 """
5141 def __init__(self, expr: Union[ParserElement, str], savelist: bool = False) -> None:
5142 super().__init__(savelist)
5143 if isinstance(expr, str_type):
5144 expr_str = typing.cast(str, expr)
5145 if issubclass(self._literalStringClass, Token):
5146 expr = self._literalStringClass(expr_str) # type: ignore[call-arg]
5147 elif issubclass(type(self), self._literalStringClass):
5148 expr = Literal(expr_str)
5149 else:
5150 expr = self._literalStringClass(Literal(expr_str)) # type: ignore[assignment, call-arg]
5151 expr = typing.cast(ParserElement, expr)
5152 self.expr = expr
5153 if expr is not None:
5154 self.mayIndexError = expr.mayIndexError
5155 self._may_return_empty = expr.mayReturnEmpty
5156 self.set_whitespace_chars(
5157 expr.whiteChars, copy_defaults=expr.copyDefaultWhiteChars
5158 )
5159 self.skipWhitespace = expr.skipWhitespace
5160 self.saveAsList = expr.saveAsList
5161 self.callPreparse = expr.callPreparse
5162 self.ignoreExprs.extend(expr.ignoreExprs)
5164 def recurse(self) -> list[ParserElement]:
5165 return [self.expr] if self.expr is not None else []
5167 def parseImpl(self, instring, loc, do_actions=True):
5168 if self.expr is None:
5169 raise ParseException(instring, loc, "No expression defined", self)
5171 try:
5172 return self.expr._parse(instring, loc, do_actions, callPreParse=False)
5173 except ParseSyntaxException:
5174 raise
5175 except ParseBaseException as pbe:
5176 pbe.pstr = pbe.pstr or instring
5177 pbe.loc = pbe.loc or loc
5178 pbe.parser_element = pbe.parser_element or self
5179 if not isinstance(self, Forward) and self.customName is not None:
5180 if self.errmsg:
5181 pbe.msg = self.errmsg
5182 raise
5184 def leave_whitespace(self, recursive: bool = True) -> ParserElement:
5185 """
5186 Extends ``leave_whitespace`` defined in base class, and also invokes ``leave_whitespace`` on
5187 the contained expression.
5188 """
5189 super().leave_whitespace(recursive)
5191 if recursive:
5192 if self.expr is not None:
5193 self.expr = self.expr.copy()
5194 self.expr.leave_whitespace(recursive)
5195 return self
5197 def ignore_whitespace(self, recursive: bool = True) -> ParserElement:
5198 """
5199 Extends ``ignore_whitespace`` defined in base class, and also invokes ``ignore_whitespace`` on
5200 the contained expression.
5201 """
5202 super().ignore_whitespace(recursive)
5204 if recursive:
5205 if self.expr is not None:
5206 self.expr = self.expr.copy()
5207 self.expr.ignore_whitespace(recursive)
5208 return self
5210 def ignore(self, other) -> ParserElement:
5211 """
5212 Define expression to be ignored (e.g., comments) while doing pattern
5213 matching; may be called repeatedly, to define multiple comment or other
5214 ignorable patterns.
5215 """
5216 if not isinstance(other, Suppress) or other not in self.ignoreExprs:
5217 super().ignore(other)
5218 if self.expr is not None:
5219 self.expr.ignore(self.ignoreExprs[-1])
5221 return self
5223 def streamline(self) -> ParserElement:
5224 super().streamline()
5225 if self.expr is not None:
5226 self.expr.streamline()
5227 return self
5229 def _checkRecursion(self, parseElementList):
5230 if self in parseElementList:
5231 raise RecursiveGrammarException(parseElementList + [self])
5232 subRecCheckList = parseElementList[:] + [self]
5233 if self.expr is not None:
5234 self.expr._checkRecursion(subRecCheckList)
5236 def validate(self, validateTrace=None) -> None:
5237 warnings.warn(
5238 "ParserElement.validate() is deprecated, and should not be used to check for left recursion",
5239 PyparsingDeprecationWarning,
5240 stacklevel=2,
5241 )
5242 if validateTrace is None:
5243 validateTrace = []
5244 tmp = validateTrace[:] + [self]
5245 if self.expr is not None:
5246 self.expr.validate(tmp)
5247 self._checkRecursion([])
5249 def _generateDefaultName(self) -> str:
5250 return f"{type(self).__name__}:({self.expr})"
5252 # Compatibility synonyms
5253 # fmt: off
5254 leaveWhitespace = replaced_by_pep8("leaveWhitespace", leave_whitespace)
5255 ignoreWhitespace = replaced_by_pep8("ignoreWhitespace", ignore_whitespace)
5256 # fmt: on
5259class IndentedBlock(ParseElementEnhance):
5260 """
5261 Expression to match one or more expressions at a given indentation level.
5262 Useful for parsing text where structure is implied by indentation (like Python source code).
5264 Example:
5266 .. testcode::
5268 '''
5269 BNF:
5270 statement ::= assignment_stmt | if_stmt
5271 assignment_stmt ::= identifier '=' rvalue
5272 rvalue ::= identifier | integer
5273 if_stmt ::= 'if' bool_condition block
5274 block ::= ([indent] statement)...
5275 identifier ::= [A..Za..z]
5276 integer ::= [0..9]...
5277 bool_condition ::= 'TRUE' | 'FALSE'
5278 '''
5280 IF, TRUE, FALSE = Keyword.using_each("IF TRUE FALSE".split())
5282 statement = Forward()
5283 identifier = Char(alphas)
5284 integer = Word(nums).add_parse_action(lambda t: int(t[0]))
5285 rvalue = identifier | integer
5286 assignment_stmt = identifier + "=" + rvalue
5288 if_stmt = IF + (TRUE | FALSE) + IndentedBlock(statement)
5290 statement <<= Group(assignment_stmt | if_stmt)
5292 result = if_stmt.parse_string('''
5293 IF TRUE
5294 a = 1000
5295 b = 2000
5296 IF FALSE
5297 z = 100
5298 ''')
5299 print(result.dump())
5301 .. testoutput::
5303 ['IF', 'TRUE', [['a', '=', 1000], ['b', '=', 2000], ['IF', 'FALSE', [['z', '=', 100]]]]]
5304 [0]:
5305 IF
5306 [1]:
5307 TRUE
5308 [2]:
5309 [['a', '=', 1000], ['b', '=', 2000], ['IF', 'FALSE', [['z', '=', 100]]]]
5310 [0]:
5311 ['a', '=', 1000]
5312 [1]:
5313 ['b', '=', 2000]
5314 [2]:
5315 ['IF', 'FALSE', [['z', '=', 100]]]
5316 [0]:
5317 IF
5318 [1]:
5319 FALSE
5320 [2]:
5321 [['z', '=', 100]]
5322 [0]:
5323 ['z', '=', 100]
5324 """
5326 class _Indent(Empty):
5327 def __init__(self, ref_col: int) -> None:
5328 super().__init__()
5329 self.errmsg = f"expected indent at column {ref_col}"
5330 self.add_condition(lambda s, l, t: col(l, s) == ref_col)
5332 class _IndentGreater(Empty):
5333 def __init__(self, ref_col: int) -> None:
5334 super().__init__()
5335 self.errmsg = f"expected indent at column greater than {ref_col}"
5336 self.add_condition(lambda s, l, t: col(l, s) > ref_col)
5338 def __init__(
5339 self, expr: ParserElement, *, recursive: bool = False, grouped: bool = True
5340 ) -> None:
5341 super().__init__(expr, savelist=True)
5342 # if recursive:
5343 # raise NotImplementedError("IndentedBlock with recursive is not implemented")
5344 self._recursive = recursive
5345 self._grouped = grouped
5346 self.parent_anchor = 1
5348 def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType:
5349 # advance parse position to non-whitespace by using an Empty()
5350 # this should be the column to be used for all subsequent indented lines
5351 anchor_loc = Empty().preParse(instring, loc)
5353 # see if self.expr matches at the current location - if not it will raise an exception
5354 # and no further work is necessary
5355 self.expr.try_parse(instring, anchor_loc, do_actions=do_actions)
5357 indent_col = col(anchor_loc, instring)
5358 peer_detect_expr = self._Indent(indent_col)
5360 inner_expr = Empty() + peer_detect_expr + self.expr
5361 if self._recursive:
5362 sub_indent = self._IndentGreater(indent_col)
5363 nested_block = IndentedBlock(
5364 self.expr, recursive=self._recursive, grouped=self._grouped
5365 )
5366 nested_block.set_debug(self.debug)
5367 nested_block.parent_anchor = indent_col
5368 inner_expr += Opt(sub_indent + nested_block)
5370 inner_expr.set_name(f"inner {hex(id(inner_expr))[-4:].upper()}@{indent_col}")
5371 block = OneOrMore(inner_expr)
5373 trailing_undent = self._Indent(self.parent_anchor) | StringEnd()
5375 if self._grouped:
5376 wrapper = Group
5377 else:
5378 wrapper = lambda expr: expr # type: ignore[misc, assignment]
5379 return (wrapper(block) + Optional(trailing_undent)).parseImpl(
5380 instring, anchor_loc, do_actions
5381 )
5384class AtStringStart(ParseElementEnhance):
5385 """Matches if expression matches at the beginning of the parse
5386 string::
5388 AtStringStart(Word(nums)).parse_string("123")
5389 # prints ["123"]
5391 AtStringStart(Word(nums)).parse_string(" 123")
5392 # raises ParseException
5393 """
5395 def __init__(self, expr: Union[ParserElement, str]) -> None:
5396 super().__init__(expr)
5397 self.callPreparse = False
5399 def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType:
5400 if loc != 0:
5401 raise ParseException(instring, loc, "not found at string start")
5402 return super().parseImpl(instring, loc, do_actions)
5405class AtLineStart(ParseElementEnhance):
5406 r"""Matches if an expression matches at the beginning of a line within
5407 the parse string
5409 Example:
5411 .. testcode::
5413 test = '''\
5414 BBB this line
5415 BBB and this line
5416 BBB but not this one
5417 A BBB and definitely not this one
5418 '''
5420 for t in (AtLineStart('BBB') + rest_of_line).search_string(test):
5421 print(t)
5423 prints:
5425 .. testoutput::
5427 ['BBB', ' this line']
5428 ['BBB', ' and this line']
5429 """
5431 def __init__(self, expr: Union[ParserElement, str]) -> None:
5432 super().__init__(expr)
5433 self.callPreparse = False
5435 def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType:
5436 if col(loc, instring) != 1:
5437 raise ParseException(instring, loc, "not found at line start")
5438 return super().parseImpl(instring, loc, do_actions)
5441class FollowedBy(ParseElementEnhance):
5442 """Lookahead matching of the given parse expression.
5443 ``FollowedBy`` does *not* advance the parsing position within
5444 the input string, it only verifies that the specified parse
5445 expression matches at the current position. ``FollowedBy``
5446 always returns a null token list. If any results names are defined
5447 in the lookahead expression, those *will* be returned for access by
5448 name.
5450 Example:
5452 .. testcode::
5454 # use FollowedBy to match a label only if it is followed by a ':'
5455 data_word = Word(alphas)
5456 label = data_word + FollowedBy(':')
5457 attr_expr = Group(
5458 label + Suppress(':')
5459 + OneOrMore(data_word, stop_on=label).set_parse_action(' '.join)
5460 )
5462 attr_expr[1, ...].parse_string(
5463 "shape: SQUARE color: BLACK posn: upper left").pprint()
5465 prints:
5467 .. testoutput::
5469 [['shape', 'SQUARE'], ['color', 'BLACK'], ['posn', 'upper left']]
5470 """
5472 def __init__(self, expr: Union[ParserElement, str]) -> None:
5473 super().__init__(expr)
5474 self._may_return_empty = True
5476 def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType:
5477 # by using self._expr.parse and deleting the contents of the returned ParseResults list
5478 # we keep any named results that were defined in the FollowedBy expression
5479 _, ret = self.expr._parse(instring, loc, do_actions=do_actions)
5480 del ret[:]
5482 return loc, ret
5485class PrecededBy(ParseElementEnhance):
5486 """Lookbehind matching of the given parse expression.
5487 ``PrecededBy`` does not advance the parsing position within the
5488 input string, it only verifies that the specified parse expression
5489 matches prior to the current position. ``PrecededBy`` always
5490 returns a null token list, but if a results name is defined on the
5491 given expression, it is returned.
5493 Parameters:
5495 - ``expr`` - expression that must match prior to the current parse
5496 location
5497 - ``retreat`` - (default= ``None``) - (int) maximum number of characters
5498 to lookbehind prior to the current parse location
5500 If the lookbehind expression is a string, :class:`Literal`,
5501 :class:`Keyword`, or a :class:`Word` or :class:`CharsNotIn`
5502 with a specified exact or maximum length, then the retreat
5503 parameter is not required. Otherwise, retreat must be specified to
5504 give a maximum number of characters to look back from
5505 the current parse position for a lookbehind match.
5507 Example:
5509 .. testcode::
5511 # VB-style variable names with type prefixes
5512 int_var = PrecededBy("#") + pyparsing_common.identifier
5513 str_var = PrecededBy("$") + pyparsing_common.identifier
5514 """
5516 def __init__(self, expr: Union[ParserElement, str], retreat: int = 0) -> None:
5517 super().__init__(expr)
5518 self.expr = self.expr().leave_whitespace()
5519 self._may_return_empty = True
5520 self.mayIndexError = False
5521 self.exact = False
5522 if isinstance(expr, str_type):
5523 expr = typing.cast(str, expr)
5524 retreat = len(expr)
5525 self.exact = True
5526 elif isinstance(expr, (Literal, Keyword)):
5527 retreat = expr.matchLen
5528 self.exact = True
5529 elif isinstance(expr, (Word, CharsNotIn)) and expr.maxLen != _MAX_INT:
5530 retreat = expr.maxLen
5531 self.exact = True
5532 elif isinstance(expr, PositionToken):
5533 retreat = 0
5534 self.exact = True
5535 self.retreat = retreat
5536 self.errmsg = f"not preceded by {expr}"
5537 self.skipWhitespace = False
5538 self.parseAction.append(lambda s, l, t: t.__delitem__(slice(None, None)))
5540 def parseImpl(self, instring, loc=0, do_actions=True) -> ParseImplReturnType:
5541 if self.exact:
5542 if loc < self.retreat:
5543 raise ParseException(instring, loc, self.errmsg, self)
5544 start = loc - self.retreat
5545 _, ret = self.expr._parse(instring, start)
5546 return loc, ret
5548 # retreat specified a maximum lookbehind window, iterate
5549 test_expr = self.expr + StringEnd()
5550 instring_slice = instring[max(0, loc - self.retreat) : loc]
5551 last_expr: ParseBaseException = ParseException(instring, loc, self.errmsg, self)
5553 for offset in range(1, min(loc, self.retreat + 1) + 1):
5554 try:
5555 # print('trying', offset, instring_slice, repr(instring_slice[loc - offset:]))
5556 _, ret = test_expr._parse(instring_slice, len(instring_slice) - offset)
5557 except ParseBaseException as pbe:
5558 last_expr = pbe
5559 else:
5560 break
5561 else:
5562 raise last_expr
5564 return loc, ret
5567class Located(ParseElementEnhance):
5568 """
5569 Decorates a returned token with its starting and ending
5570 locations in the input string.
5572 This helper adds the following results names:
5574 - ``locn_start`` - location where matched expression begins
5575 - ``locn_end`` - location where matched expression ends
5576 - ``value`` - the actual parsed results
5578 Be careful if the input text contains ``<TAB>`` characters, you
5579 may want to call :class:`ParserElement.parse_with_tabs`
5581 Example:
5583 .. testcode::
5585 wd = Word(alphas)
5586 for match in Located(wd).search_string("ljsdf123lksdjjf123lkkjj1222"):
5587 print(match)
5589 prints:
5591 .. testoutput::
5593 [0, ['ljsdf'], 5]
5594 [8, ['lksdjjf'], 15]
5595 [18, ['lkkjj'], 23]
5596 """
5598 def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType:
5599 # skip leading whitespace before capturing the start location, so
5600 # locn_start marks the start of the match and not preceding whitespace,
5601 # even when the wrapped expression delegates whitespace skipping to its
5602 # sub-expressions (e.g. And/MatchFirst). Issue #621.
5603 start = self.expr.preParse(instring, loc)
5604 loc, tokens = self.expr._parse(instring, start, do_actions, callPreParse=False)
5605 ret_tokens = ParseResults([start, tokens, loc])
5606 ret_tokens["locn_start"] = start
5607 ret_tokens["value"] = tokens
5608 ret_tokens["locn_end"] = loc
5609 if self.resultsName:
5610 # must return as a list, so that the name will be attached to the complete group
5611 return loc, [ret_tokens]
5612 else:
5613 return loc, ret_tokens
5616class NotAny(ParseElementEnhance):
5617 """
5618 Lookahead to disallow matching with the given parse expression.
5619 ``NotAny`` does *not* advance the parsing position within the
5620 input string, it only verifies that the specified parse expression
5621 does *not* match at the current position. Also, ``NotAny`` does
5622 *not* skip over leading whitespace. ``NotAny`` always returns
5623 a null token list. May be constructed using the ``'~'`` operator.
5625 Example:
5627 .. testcode::
5629 AND, OR, NOT = map(CaselessKeyword, "AND OR NOT".split())
5631 # take care not to mistake keywords for identifiers
5632 ident = ~(AND | OR | NOT) + Word(alphas)
5633 boolean_term = Opt(NOT) + ident
5635 # very crude boolean expression - to support parenthesis groups and
5636 # operation hierarchy, use infix_notation
5637 boolean_expr = boolean_term + ((AND | OR) + boolean_term)[...]
5639 # integers that are followed by "." are actually floats
5640 integer = Word(nums) + ~Char(".")
5641 """
5643 def __init__(self, expr: Union[ParserElement, str]) -> None:
5644 super().__init__(expr)
5645 # do NOT use self.leave_whitespace(), don't want to propagate to exprs
5646 # self.leave_whitespace()
5647 self.skipWhitespace = False
5649 self._may_return_empty = True
5650 self.errmsg = f"Found unwanted token, {self.expr}"
5652 def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType:
5653 if self.expr.can_parse_next(instring, loc, do_actions=do_actions):
5654 raise ParseException(instring, loc, self.errmsg, self)
5655 return loc, []
5657 def _generateDefaultName(self) -> str:
5658 return f"~{{{self.expr}}}"
5661class _MultipleMatch(ParseElementEnhance):
5662 def __init__(
5663 self,
5664 expr: Union[str, ParserElement],
5665 stop_on: typing.Optional[Union[ParserElement, str]] = None,
5666 max: typing.Optional[int] = None,
5667 **kwargs,
5668 ) -> None:
5669 stopOn: typing.Optional[Union[ParserElement, str]] = deprecate_argument(
5670 kwargs, "stopOn", None
5671 )
5673 super().__init__(expr)
5674 if max is not None and max <= 0:
5675 raise ValueError("max must be greater than 0")
5676 stopOn = stopOn or stop_on
5677 self.saveAsList = True
5678 self.max_count = max
5679 ender = stopOn
5680 if isinstance(ender, str_type):
5681 ender = self._literalStringClass(ender)
5682 self.stopOn(ender)
5684 def stop_on(self, ender) -> ParserElement:
5685 if isinstance(ender, str_type):
5686 ender = self._literalStringClass(ender)
5687 self.not_ender = ~ender if ender is not None else None
5688 return self
5690 stopOn = stop_on
5692 def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType:
5693 self_expr_parse = self.expr._parse
5694 self_skip_ignorables = self._skipIgnorables
5695 check_ender = False
5696 if self.not_ender is not None:
5697 try_not_ender = self.not_ender.try_parse
5698 check_ender = True
5700 # must be at least one (but first see if we are the stopOn sentinel;
5701 # if so, fail)
5702 if check_ender:
5703 try_not_ender(instring, loc)
5704 loc, tokens = self_expr_parse(instring, loc, do_actions)
5705 match_count = 1
5706 try:
5707 hasIgnoreExprs = not not self.ignoreExprs
5708 while self.max_count is None or match_count < self.max_count:
5709 if check_ender:
5710 try_not_ender(instring, loc)
5711 if hasIgnoreExprs:
5712 preloc = self_skip_ignorables(instring, loc)
5713 else:
5714 preloc = loc
5715 loc, tmptokens = self_expr_parse(instring, preloc, do_actions)
5716 tokens += tmptokens
5717 match_count += 1
5718 except (ParseException, IndexError):
5719 pass
5721 return loc, tokens
5723 def _setResultsName(self, name, list_all_matches=False) -> ParserElement:
5724 if (
5725 __diag__.warn_ungrouped_named_tokens_in_collection
5726 and Diagnostics.warn_ungrouped_named_tokens_in_collection
5727 not in self.suppress_warnings_
5728 ):
5729 for e in [self.expr] + self.expr.recurse():
5730 if (
5731 isinstance(e, ParserElement)
5732 and e.resultsName
5733 and (
5734 Diagnostics.warn_ungrouped_named_tokens_in_collection
5735 not in e.suppress_warnings_
5736 )
5737 ):
5738 warning = (
5739 "warn_ungrouped_named_tokens_in_collection:"
5740 f" setting results name {name!r} on {type(self).__name__} expression"
5741 f" collides with {e.resultsName!r} on contained expression"
5742 )
5743 warnings.warn(warning, PyparsingDiagnosticWarning, stacklevel=3)
5744 break
5746 return super()._setResultsName(name, list_all_matches)
5749class OneOrMore(_MultipleMatch):
5750 """
5751 Repetition of one or more of the given expression.
5753 Parameters:
5755 - ``expr`` - expression that must match one or more times
5756 - ``stop_on`` - (default= ``None``) - expression for a terminating sentinel
5757 (only required if the sentinel would ordinarily match the repetition
5758 expression)
5760 Example:
5762 .. doctest::
5764 >>> data_word = Word(alphas)
5765 >>> label = data_word + FollowedBy(':')
5766 >>> attr_expr = Group(
5767 ... label + Suppress(':')
5768 ... + OneOrMore(data_word).set_parse_action(' '.join))
5770 >>> text = "shape: SQUARE posn: upper left color: BLACK"
5772 # Fail! read 'posn' as data instead of next label
5773 >>> attr_expr[1, ...].parse_string(text).pprint()
5774 [['shape', 'SQUARE posn']]
5776 # use stop_on attribute for OneOrMore
5777 # to avoid reading label string as part of the data
5778 >>> attr_expr = Group(
5779 ... label + Suppress(':')
5780 ... + OneOrMore(
5781 ... data_word, stop_on=label).set_parse_action(' '.join))
5782 >>> OneOrMore(attr_expr).parse_string(text).pprint() # Better
5783 [['shape', 'SQUARE'], ['posn', 'upper left'], ['color', 'BLACK']]
5785 # could also be written as
5786 >>> (attr_expr * (1,)).parse_string(text).pprint()
5787 [['shape', 'SQUARE'], ['posn', 'upper left'], ['color', 'BLACK']]
5788 """
5790 def _generateDefaultName(self) -> str:
5791 return f"{{{self.expr}}}..."
5794class ZeroOrMore(_MultipleMatch):
5795 """
5796 Optional repetition of zero or more of the given expression.
5798 Parameters:
5800 - ``expr`` - expression that must match zero or more times
5801 - ``stop_on`` - expression for a terminating sentinel
5802 (only required if the sentinel would ordinarily match the repetition
5803 expression) - (default= ``None``)
5805 Example: similar to :class:`OneOrMore`
5806 """
5808 def __init__(
5809 self,
5810 expr: Union[str, ParserElement],
5811 stop_on: typing.Optional[Union[ParserElement, str]] = None,
5812 max: typing.Optional[int] = None,
5813 **kwargs,
5814 ) -> None:
5815 stopOn: Union[ParserElement, str] = deprecate_argument(kwargs, "stopOn", None)
5817 super().__init__(expr, stop_on=stopOn or stop_on, max=max)
5818 self._may_return_empty = True
5820 def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType:
5821 try:
5822 return super().parseImpl(instring, loc, do_actions)
5823 except (ParseException, IndexError):
5824 return loc, ParseResults([], name=self.resultsName)
5826 def _generateDefaultName(self) -> str:
5827 return f"[{self.expr}]..."
5830class DelimitedList(ParseElementEnhance):
5831 """Helper to define a delimited list of expressions - the delimiter
5832 defaults to ','. By default, the list elements and delimiters can
5833 have intervening whitespace, and comments, but this can be
5834 overridden by passing ``combine=True`` in the constructor. If
5835 ``combine`` is set to ``True``, the matching tokens are
5836 returned as a single token string, with the delimiters included;
5837 otherwise, the matching tokens are returned as a list of tokens,
5838 with the delimiters suppressed.
5840 If ``allow_trailing_delim`` is set to True, then the list may end with
5841 a delimiter.
5843 Example:
5845 .. doctest::
5847 >>> DelimitedList(Word(alphas)).parse_string("aa,bb,cc")
5848 ParseResults(['aa', 'bb', 'cc'], {})
5849 >>> DelimitedList(Word(hexnums), delim=':', combine=True
5850 ... ).parse_string("AA:BB:CC:DD:EE")
5851 ParseResults(['AA:BB:CC:DD:EE'], {})
5853 .. versionadded:: 3.1.0
5854 """
5856 def __init__(
5857 self,
5858 expr: Union[str, ParserElement],
5859 delim: Union[str, ParserElement] = ",",
5860 combine: bool = False,
5861 min: typing.Optional[int] = None,
5862 max: typing.Optional[int] = None,
5863 *,
5864 allow_trailing_delim: bool = False,
5865 ) -> None:
5866 if isinstance(expr, str_type):
5867 expr = ParserElement._literalStringClass(expr)
5868 expr = typing.cast(ParserElement, expr)
5870 if min is not None and min < 1:
5871 raise ValueError("min must be greater than 0")
5873 if max is not None and min is not None and max < min:
5874 raise ValueError("max must be greater than, or equal to min")
5876 self.content = expr
5877 self.raw_delim = str(delim)
5878 self.delim = delim
5879 self.combine = combine
5880 if not combine:
5881 self.delim = Suppress(delim) if not isinstance(delim, Suppress) else delim
5882 self.min = min or 1
5883 self.max = max
5884 self.allow_trailing_delim = allow_trailing_delim
5886 delim_list_expr = self.content + (self.delim + self.content) * (
5887 self.min - 1,
5888 None if self.max is None else self.max - 1,
5889 )
5890 if self.allow_trailing_delim:
5891 delim_list_expr += Opt(self.delim)
5893 if self.combine:
5894 delim_list_expr = Combine(delim_list_expr)
5896 super().__init__(delim_list_expr, savelist=True)
5898 def _generateDefaultName(self) -> str:
5899 content_expr = self.content.streamline()
5900 return f"{content_expr} [{self.raw_delim} {content_expr}]..."
5903class _NullToken:
5904 def __bool__(self):
5905 return False
5907 def __str__(self):
5908 return ""
5911class Opt(ParseElementEnhance):
5912 """
5913 Optional matching of the given expression.
5915 :param expr: expression that must match zero or more times
5916 :param default: (optional) - value to be returned
5917 if the optional expression is not found.
5919 Example:
5921 .. testcode::
5923 # US postal code can be a 5-digit zip, plus optional 4-digit qualifier
5924 zip = Combine(Word(nums, exact=5) + Opt('-' + Word(nums, exact=4)))
5925 zip.run_tests('''
5926 # traditional ZIP code
5927 12345
5929 # ZIP+4 form
5930 12101-0001
5932 # invalid ZIP
5933 98765-
5934 ''')
5936 prints:
5938 .. testoutput::
5939 :options: +NORMALIZE_WHITESPACE
5942 # traditional ZIP code
5943 12345
5944 ['12345']
5946 # ZIP+4 form
5947 12101-0001
5948 ['12101-0001']
5950 # invalid ZIP
5951 98765-
5952 98765-
5953 ^
5954 ParseException: Expected end of text, found '-' (at char 5), (line:1, col:6)
5955 FAIL: Expected end of text, found '-' (at char 5), (line:1, col:6)
5956 """
5958 __optionalNotMatched = _NullToken()
5960 def __init__(
5961 self, expr: Union[ParserElement, str], default: Any = __optionalNotMatched
5962 ) -> None:
5963 super().__init__(expr, savelist=False)
5964 self.saveAsList = self.expr.saveAsList
5965 self.defaultValue = default
5966 self._may_return_empty = True
5968 def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType:
5969 self_expr = self.expr
5970 try:
5971 loc, tokens = self_expr._parse(
5972 instring, loc, do_actions, callPreParse=False
5973 )
5974 except (ParseException, IndexError):
5975 default_value = self.defaultValue
5976 if default_value is not self.__optionalNotMatched:
5977 if self_expr.resultsName:
5978 tokens = ParseResults([default_value])
5979 tokens[self_expr.resultsName] = default_value
5980 else:
5981 tokens = [default_value] # type: ignore[assignment]
5982 else:
5983 tokens = [] # type: ignore[assignment]
5984 return loc, tokens
5986 def _generateDefaultName(self) -> str:
5987 inner = str(self.expr)
5988 # strip off redundant inner {}'s
5989 while len(inner) > 1 and inner[0 :: len(inner) - 1] == "{}":
5990 inner = inner[1:-1]
5991 return f"[{inner}]"
5994Optional = Opt
5997class SkipTo(ParseElementEnhance):
5998 """
5999 Token for skipping over all undefined text until the matched
6000 expression is found.
6002 :param expr: target expression marking the end of the data to be skipped
6003 :param include: if ``True``, the target expression is also parsed
6004 (the skipped text and target expression are returned
6005 as a 2-element list) (default= ``False``).
6007 :param ignore: (default= ``None``) used to define grammars
6008 (typically quoted strings and comments)
6009 that might contain false matches to the target expression
6011 :param fail_on: (default= ``None``) define expressions that
6012 are not allowed to be included in the skipped test;
6013 if found before the target expression is found,
6014 the :class:`SkipTo` is not a match
6016 Example:
6018 .. testcode::
6020 report = '''
6021 Outstanding Issues Report - 1 Jan 2000
6023 # | Severity | Description | Days Open
6024 -----+----------+-------------------------------------------+-----------
6025 101 | Critical | Intermittent system crash | 6
6026 94 | Cosmetic | Spelling error on Login ('log|n') | 14
6027 79 | Minor | System slow when running too many reports | 47
6028 '''
6029 integer = Word(nums)
6030 SEP = Suppress('|')
6031 # use SkipTo to simply match everything up until the next SEP
6032 # - ignore quoted strings, so that a '|' character inside a quoted string does not match
6033 # - parse action will call token.strip() for each matched token, i.e., the description body
6034 string_data = SkipTo(SEP, ignore=quoted_string)
6035 string_data.set_parse_action(token_map(str.strip))
6036 ticket_expr = (integer("issue_num") + SEP
6037 + string_data("sev") + SEP
6038 + string_data("desc") + SEP
6039 + integer("days_open"))
6041 for tkt in ticket_expr.search_string(report):
6042 print(tkt.dump())
6044 prints:
6046 .. testoutput::
6048 ['101', 'Critical', 'Intermittent system crash', '6']
6049 - days_open: '6'
6050 - desc: 'Intermittent system crash'
6051 - issue_num: '101'
6052 - sev: 'Critical'
6053 ['94', 'Cosmetic', "Spelling error on Login ('log|n')", '14']
6054 - days_open: '14'
6055 - desc: "Spelling error on Login ('log|n')"
6056 - issue_num: '94'
6057 - sev: 'Cosmetic'
6058 ['79', 'Minor', 'System slow when running too many reports', '47']
6059 - days_open: '47'
6060 - desc: 'System slow when running too many reports'
6061 - issue_num: '79'
6062 - sev: 'Minor'
6063 """
6065 def __init__(
6066 self,
6067 other: Union[ParserElement, str],
6068 include: bool = False,
6069 ignore: typing.Optional[Union[ParserElement, str]] = None,
6070 fail_on: typing.Optional[Union[ParserElement, str]] = None,
6071 **kwargs,
6072 ) -> None:
6073 failOn: typing.Optional[Union[ParserElement, str]] = deprecate_argument(
6074 kwargs, "failOn", None
6075 )
6077 super().__init__(other)
6078 failOn = failOn or fail_on
6079 self.ignoreExpr = ignore
6080 self._may_return_empty = True
6081 self.mayIndexError = False
6082 self.includeMatch = include
6083 self.saveAsList = False
6084 if isinstance(failOn, str_type):
6085 self.failOn = self._literalStringClass(failOn)
6086 else:
6087 self.failOn = failOn
6088 self.errmsg = f"No match found for {self.expr}"
6089 self.ignorer = Empty().leave_whitespace()
6090 self._update_ignorer()
6092 def _update_ignorer(self):
6093 # rebuild internal ignore expr from current ignore exprs and assigned ignoreExpr
6094 self.ignorer.ignoreExprs.clear()
6095 for e in self.expr.ignoreExprs:
6096 self.ignorer.ignore(e)
6097 if self.ignoreExpr:
6098 self.ignorer.ignore(self.ignoreExpr)
6100 def ignore(self, expr):
6101 """
6102 Define expression to be ignored (e.g., comments) while doing pattern
6103 matching; may be called repeatedly, to define multiple comment or other
6104 ignorable patterns.
6105 """
6106 super().ignore(expr)
6107 self._update_ignorer()
6109 def parseImpl(self, instring, loc, do_actions=True):
6110 startloc = loc
6111 instrlen = len(instring)
6112 self_expr_parse = self.expr._parse
6113 self_failOn_canParseNext = (
6114 self.failOn.can_parse_next if self.failOn is not None else None
6115 )
6116 ignorer_try_parse = self.ignorer.try_parse if self.ignorer.ignoreExprs else None
6118 tmploc = loc
6119 while tmploc <= instrlen:
6120 if self_failOn_canParseNext is not None:
6121 # break if failOn expression matches
6122 if self_failOn_canParseNext(instring, tmploc):
6123 break
6125 if ignorer_try_parse is not None:
6126 # advance past ignore expressions
6127 prev_tmploc = tmploc
6128 while 1:
6129 try:
6130 tmploc = ignorer_try_parse(instring, tmploc)
6131 except ParseBaseException:
6132 break
6133 # see if all ignorers matched, but didn't actually ignore anything
6134 if tmploc == prev_tmploc:
6135 break
6136 prev_tmploc = tmploc
6138 try:
6139 self_expr_parse(instring, tmploc, do_actions=False, callPreParse=False)
6140 except (ParseException, IndexError):
6141 # no match, advance loc in string
6142 tmploc += 1
6143 else:
6144 # matched skipto expr, done
6145 break
6147 else:
6148 # ran off the end of the input string without matching skipto expr, fail
6149 raise ParseException(instring, loc, self.errmsg, self)
6151 # build up return values
6152 loc = tmploc
6153 skiptext = instring[startloc:loc]
6154 skipresult = ParseResults(skiptext)
6156 if self.includeMatch:
6157 loc, mat = self_expr_parse(instring, loc, do_actions, callPreParse=False)
6158 skipresult += mat
6160 return loc, skipresult
6163class Forward(ParseElementEnhance):
6164 """
6165 Forward declaration of an expression to be defined later -
6166 used for recursive grammars, such as algebraic infix notation.
6167 When the expression is known, it is assigned to the ``Forward``
6168 instance using the ``'<<'`` operator.
6170 .. Note::
6172 Take care when assigning to ``Forward`` not to overlook
6173 precedence of operators.
6175 Specifically, ``'|'`` has a lower precedence than ``'<<'``, so that::
6177 fwd_expr << a | b | c
6179 will actually be evaluated as::
6181 (fwd_expr << a) | b | c
6183 thereby leaving b and c out as parseable alternatives.
6184 It is recommended that you explicitly group the values
6185 inserted into the :class:`Forward`::
6187 fwd_expr << (a | b | c)
6189 Converting to use the ``'<<='`` operator instead will avoid this problem.
6191 See :meth:`ParseResults.pprint` for an example of a recursive
6192 parser created using :class:`Forward`.
6193 """
6195 def __init__(
6196 self, other: typing.Optional[Union[ParserElement, str]] = None
6197 ) -> None:
6198 self.caller_frame = traceback.extract_stack(limit=2)[0]
6199 super().__init__(other, savelist=False) # type: ignore[arg-type]
6200 self.lshift_line = None
6202 def __lshift__(self, other) -> Forward:
6203 if hasattr(self, "caller_frame"):
6204 del self.caller_frame
6205 if isinstance(other, str_type):
6206 other = self._literalStringClass(other)
6208 if not isinstance(other, ParserElement):
6209 return NotImplemented
6211 self.expr = other
6212 self.streamlined = other.streamlined
6213 self.mayIndexError = self.expr.mayIndexError
6214 self._may_return_empty = self.expr.mayReturnEmpty
6215 self.set_whitespace_chars(
6216 self.expr.whiteChars, copy_defaults=self.expr.copyDefaultWhiteChars
6217 )
6218 self.skipWhitespace = self.expr.skipWhitespace
6219 self.saveAsList = self.expr.saveAsList
6220 self.ignoreExprs.extend(self.expr.ignoreExprs)
6221 self.lshift_line = traceback.extract_stack(limit=2)[-2] # type: ignore[assignment]
6222 return self
6224 def __ilshift__(self, other) -> Forward:
6225 if not isinstance(other, ParserElement):
6226 return NotImplemented
6228 return self << other
6230 def __or__(self, other) -> ParserElement:
6231 caller_line = traceback.extract_stack(limit=2)[-2]
6232 if (
6233 __diag__.warn_on_match_first_with_lshift_operator
6234 and caller_line == self.lshift_line
6235 and Diagnostics.warn_on_match_first_with_lshift_operator
6236 not in self.suppress_warnings_
6237 ):
6238 warnings.warn(
6239 "warn_on_match_first_with_lshift_operator:"
6240 " using '<<' operator with '|' is probably an error, use '<<='",
6241 PyparsingDiagnosticWarning,
6242 stacklevel=2,
6243 )
6244 ret = super().__or__(other)
6245 return ret
6247 def __del__(self):
6248 # see if we are getting dropped because of '=' reassignment of var instead of '<<=' or '<<'
6249 if (
6250 self.expr is None
6251 and __diag__.warn_on_assignment_to_Forward
6252 and Diagnostics.warn_on_assignment_to_Forward not in self.suppress_warnings_
6253 ):
6254 warnings.warn_explicit(
6255 "warn_on_assignment_to_Forward:"
6256 " Forward defined here but no expression attached later using '<<=' or '<<'",
6257 UserWarning,
6258 filename=self.caller_frame.filename,
6259 lineno=self.caller_frame.lineno,
6260 )
6262 def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType:
6263 if (
6264 self.expr is None
6265 and __diag__.warn_on_parse_using_empty_Forward
6266 and Diagnostics.warn_on_parse_using_empty_Forward
6267 not in self.suppress_warnings_
6268 ):
6269 # walk stack until parse_string, scan_string, search_string, or transform_string is found
6270 parse_fns = (
6271 "parse_string",
6272 "scan_string",
6273 "search_string",
6274 "transform_string",
6275 )
6276 tb = traceback.extract_stack(limit=200)
6277 for i, frm in enumerate(reversed(tb), start=1):
6278 if frm.name in parse_fns:
6279 stacklevel = i + 1
6280 break
6281 else:
6282 stacklevel = 2
6283 warnings.warn(
6284 "warn_on_parse_using_empty_Forward:"
6285 " Forward expression was never assigned a value, will not parse any input",
6286 PyparsingDiagnosticWarning,
6287 stacklevel=stacklevel,
6288 )
6289 if not ParserElement._left_recursion_enabled:
6290 return super().parseImpl(instring, loc, do_actions)
6291 # ## Bounded Recursion algorithm ##
6292 # Recursion only needs to be processed at ``Forward`` elements, since they are
6293 # the only ones that can actually refer to themselves. The general idea is
6294 # to handle recursion stepwise: We start at no recursion, then recurse once,
6295 # recurse twice, ..., until more recursion offers no benefit (we hit the bound).
6296 #
6297 # The "trick" here is that each ``Forward`` gets evaluated in two contexts
6298 # - to *match* a specific recursion level, and
6299 # - to *search* the bounded recursion level
6300 # and the two run concurrently. The *search* must *match* each recursion level
6301 # to find the best possible match. This is handled by a memo table, which
6302 # provides the previous match to the next level match attempt.
6303 #
6304 # See also "Left Recursion in Parsing Expression Grammars", Medeiros et al.
6305 #
6306 # There is a complication since we not only *parse* but also *transform* via
6307 # actions: We do not want to run the actions too often while expanding. Thus,
6308 # we expand using `do_actions=False` and only run `do_actions=True` if the next
6309 # recursion level is acceptable.
6310 with ParserElement.recursion_lock:
6311 memo = ParserElement.recursion_memos
6312 try:
6313 # we are parsing at a specific recursion expansion - use it as-is
6314 prev_loc, prev_result = memo[loc, self, do_actions]
6315 if isinstance(prev_result, Exception):
6316 raise prev_result
6317 return prev_loc, prev_result.copy()
6318 except KeyError:
6319 act_key = (loc, self, True)
6320 peek_key = (loc, self, False)
6321 # we are searching for the best recursion expansion - keep on improving
6322 # both `do_actions` cases must be tracked separately here!
6323 prev_loc, prev_peek = memo[peek_key] = (
6324 loc - 1,
6325 ParseException(
6326 instring, loc, "Forward recursion without base case", self
6327 ),
6328 )
6329 if do_actions:
6330 memo[act_key] = memo[peek_key]
6331 while True:
6332 try:
6333 new_loc, new_peek = super().parseImpl(instring, loc, False)
6334 except ParseException:
6335 # we failed before getting any match - do not hide the error
6336 if isinstance(prev_peek, Exception):
6337 raise
6338 new_loc, new_peek = prev_loc, prev_peek
6339 # the match did not get better: we are done
6340 if new_loc <= prev_loc:
6341 if do_actions:
6342 # replace the match for do_actions=False as well,
6343 # in case the action did backtrack
6344 prev_loc, prev_result = memo[peek_key] = memo[act_key]
6345 del memo[peek_key], memo[act_key]
6346 return prev_loc, copy.copy(prev_result)
6347 del memo[peek_key]
6348 return prev_loc, copy.copy(prev_peek)
6349 # the match did get better: see if we can improve further
6350 if do_actions:
6351 try:
6352 memo[act_key] = super().parseImpl(instring, loc, True)
6353 except ParseException as e:
6354 memo[peek_key] = memo[act_key] = (new_loc, e)
6355 raise
6356 prev_loc, prev_peek = memo[peek_key] = new_loc, new_peek
6358 def leave_whitespace(self, recursive: bool = True) -> ParserElement:
6359 """
6360 Extends ``leave_whitespace`` defined in base class.
6361 """
6362 self.skipWhitespace = False
6363 return self
6365 def ignore_whitespace(self, recursive: bool = True) -> ParserElement:
6366 """
6367 Extends ``ignore_whitespace`` defined in base class.
6368 """
6369 self.skipWhitespace = True
6370 return self
6372 def streamline(self) -> ParserElement:
6373 if not self.streamlined:
6374 self.streamlined = True
6375 if self.expr is not None:
6376 self.expr.streamline()
6377 return self
6379 def validate(self, validateTrace=None) -> None:
6380 warnings.warn(
6381 "ParserElement.validate() is deprecated, and should not be used to check for left recursion",
6382 PyparsingDeprecationWarning,
6383 stacklevel=2,
6384 )
6385 if validateTrace is None:
6386 validateTrace = []
6388 if self not in validateTrace:
6389 tmp = validateTrace[:] + [self]
6390 if self.expr is not None:
6391 self.expr.validate(tmp)
6392 self._checkRecursion([])
6394 def _generateDefaultName(self) -> str:
6395 # Avoid infinite recursion by setting a temporary _defaultName
6396 save_default_name = self._defaultName
6397 self._defaultName = ": ..."
6399 # Use the string representation of main expression.
6400 try:
6401 if self.expr is not None:
6402 ret_string = str(self.expr)[:1000]
6403 else:
6404 ret_string = "None"
6405 except Exception:
6406 ret_string = "..."
6408 self._defaultName = save_default_name
6409 return f"{type(self).__name__}: {ret_string}"
6411 def copy(self) -> ParserElement:
6412 """
6413 Returns a copy of this expression.
6415 Generally only used internally by pyparsing.
6416 """
6417 if self.expr is not None:
6418 return super().copy()
6419 else:
6420 ret = Forward()
6421 ret <<= self
6422 return ret
6424 def _setResultsName(self, name, list_all_matches=False) -> ParserElement:
6425 # fmt: off
6426 if (
6427 __diag__.warn_name_set_on_empty_Forward
6428 and Diagnostics.warn_name_set_on_empty_Forward not in self.suppress_warnings_
6429 and self.expr is None
6430 ):
6431 warning = (
6432 "warn_name_set_on_empty_Forward:"
6433 f" setting results name {name!r} on {type(self).__name__} expression"
6434 " that has no contained expression"
6435 )
6436 warnings.warn(warning, PyparsingDiagnosticWarning, stacklevel=3)
6437 # fmt: on
6439 return super()._setResultsName(name, list_all_matches)
6441 # Compatibility synonyms
6442 # fmt: off
6443 leaveWhitespace = replaced_by_pep8("leaveWhitespace", leave_whitespace)
6444 ignoreWhitespace = replaced_by_pep8("ignoreWhitespace", ignore_whitespace)
6445 # fmt: on
6448class TokenConverter(ParseElementEnhance):
6449 """
6450 Abstract subclass of :class:`ParseElementEnhance`, for converting parsed results.
6451 """
6453 def __init__(self, expr: Union[ParserElement, str], savelist=False) -> None:
6454 super().__init__(expr) # , savelist)
6455 self.saveAsList = False
6458class Combine(TokenConverter):
6459 """Converter to concatenate all matching tokens to a single string.
6460 By default, the matching patterns must also be contiguous in the
6461 input string; this can be disabled by specifying
6462 ``'adjacent=False'`` in the constructor.
6464 Example:
6466 .. doctest::
6468 >>> real = Word(nums) + '.' + Word(nums)
6469 >>> print(real.parse_string('3.1416'))
6470 ['3', '.', '1416']
6472 >>> # will also erroneously match the following
6473 >>> print(real.parse_string('3. 1416'))
6474 ['3', '.', '1416']
6476 >>> real = Combine(Word(nums) + '.' + Word(nums))
6477 >>> print(real.parse_string('3.1416'))
6478 ['3.1416']
6480 >>> # no match when there are internal spaces
6481 >>> print(real.parse_string('3. 1416'))
6482 Traceback (most recent call last):
6483 ParseException: Expected W:(0123...)
6484 """
6486 def __init__(
6487 self,
6488 expr: ParserElement,
6489 join_string: str = "",
6490 adjacent: bool = True,
6491 *,
6492 joinString: typing.Optional[str] = None,
6493 ) -> None:
6494 super().__init__(expr)
6495 joinString = joinString if joinString is not None else join_string
6496 # suppress whitespace-stripping in contained parse expressions, but re-enable it on the Combine itself
6497 if adjacent:
6498 self.leave_whitespace()
6499 self.adjacent = adjacent
6500 self.skipWhitespace = True
6501 self.joinString = joinString
6502 self.callPreparse = True
6504 def ignore(self, other) -> ParserElement:
6505 """
6506 Define expression to be ignored (e.g., comments) while doing pattern
6507 matching; may be called repeatedly, to define multiple comment or other
6508 ignorable patterns.
6509 """
6510 if self.adjacent:
6511 ParserElement.ignore(self, other)
6512 else:
6513 super().ignore(other)
6514 return self
6516 def postParse(self, instring, loc, tokenlist):
6517 retToks = tokenlist.copy()
6518 del retToks[:]
6519 retToks += ParseResults(
6520 ["".join(tokenlist._asStringList(self.joinString))], modal=self.modalResults
6521 )
6523 if self.resultsName and retToks.haskeys():
6524 return [retToks]
6525 else:
6526 return retToks
6529class Group(TokenConverter):
6530 """Converter to return the matched tokens as a list - useful for
6531 returning tokens of :class:`ZeroOrMore` and :class:`OneOrMore` expressions.
6533 The optional ``aslist`` argument when set to True will return the
6534 parsed tokens as a Python list instead of a pyparsing ParseResults.
6536 Example:
6538 .. doctest::
6540 >>> ident = Word(alphas)
6541 >>> num = Word(nums)
6542 >>> term = ident | num
6543 >>> func = ident + Opt(DelimitedList(term))
6544 >>> print(func.parse_string("fn a, b, 100"))
6545 ['fn', 'a', 'b', '100']
6547 >>> func = ident + Group(Opt(DelimitedList(term)))
6548 >>> print(func.parse_string("fn a, b, 100"))
6549 ['fn', ['a', 'b', '100']]
6550 """
6552 def __init__(self, expr: ParserElement, aslist: bool = False) -> None:
6553 super().__init__(expr)
6554 self.saveAsList = True
6555 self._asPythonList = aslist
6557 def postParse(self, instring, loc, tokenlist):
6558 if self._asPythonList:
6559 return ParseResults.List(
6560 tokenlist.as_list()
6561 if isinstance(tokenlist, ParseResults)
6562 else list(tokenlist)
6563 )
6565 return [tokenlist]
6568class Dict(TokenConverter):
6569 """Converter to return a repetitive expression as a list, but also
6570 as a dictionary. Each element can also be referenced using the first
6571 token in the expression as its key. Useful for tabular report
6572 scraping when the first column can be used as a item key.
6574 The optional ``asdict`` argument when set to True will return the
6575 parsed tokens as a Python dict instead of a pyparsing ParseResults.
6577 Example:
6579 .. doctest::
6581 >>> data_word = Word(alphas)
6582 >>> label = data_word + FollowedBy(':')
6584 >>> attr_expr = (
6585 ... label + Suppress(':')
6586 ... + OneOrMore(data_word, stop_on=label)
6587 ... .set_parse_action(' '.join)
6588 ... )
6590 >>> text = "shape: SQUARE posn: upper left color: light blue texture: burlap"
6592 >>> # print attributes as plain groups
6593 >>> print(attr_expr[1, ...].parse_string(text).dump())
6594 ['shape', 'SQUARE', 'posn', 'upper left', 'color', 'light blue', 'texture', 'burlap']
6596 # instead of OneOrMore(expr), parse using Dict(Group(expr)[1, ...])
6597 # Dict will auto-assign names.
6598 >>> result = Dict(Group(attr_expr)[1, ...]).parse_string(text)
6599 >>> print(result.dump())
6600 [['shape', 'SQUARE'], ['posn', 'upper left'], ['color', 'light blue'], ['texture', 'burlap']]
6601 - color: 'light blue'
6602 - posn: 'upper left'
6603 - shape: 'SQUARE'
6604 - texture: 'burlap'
6605 [0]:
6606 ['shape', 'SQUARE']
6607 [1]:
6608 ['posn', 'upper left']
6609 [2]:
6610 ['color', 'light blue']
6611 [3]:
6612 ['texture', 'burlap']
6614 # access named fields as dict entries, or output as dict
6615 >>> print(result['shape'])
6616 SQUARE
6617 >>> print(result.as_dict())
6618 {'shape': 'SQUARE', 'posn': 'upper left', 'color': 'light blue', 'texture': 'burlap'}
6620 See more examples at :class:`ParseResults` of accessing fields by results name.
6621 """
6623 def __init__(self, expr: ParserElement, asdict: bool = False) -> None:
6624 super().__init__(expr)
6625 self.saveAsList = True
6626 self._asPythonDict = asdict
6628 def postParse(self, instring, loc, tokenlist):
6629 for i, tok in enumerate(tokenlist):
6630 if len(tok) == 0:
6631 continue
6633 ikey = tok[0]
6634 if isinstance(ikey, int):
6635 ikey = str(ikey).strip()
6637 # ParseResults are not hashable, cannot be used as a dict key
6638 if isinstance(ikey, ParseResults):
6639 continue
6641 if len(tok) == 1:
6642 tokenlist[ikey] = _ParseResultsWithOffset("", i)
6644 elif len(tok) == 2 and not isinstance(tok[1], ParseResults):
6645 tokenlist[ikey] = _ParseResultsWithOffset(tok[1], i)
6647 else:
6648 try:
6649 dictvalue = tok.copy() # ParseResults(i)
6650 except Exception:
6651 exc = TypeError(
6652 "could not extract dict values from parsed results"
6653 " - Dict expression must contain Grouped expressions"
6654 )
6655 raise exc from None
6657 del dictvalue[0]
6659 if len(dictvalue) != 1 or (
6660 isinstance(dictvalue, ParseResults) and dictvalue.haskeys()
6661 ):
6662 tokenlist[ikey] = _ParseResultsWithOffset(dictvalue, i)
6663 else:
6664 tokenlist[ikey] = _ParseResultsWithOffset(dictvalue[0], i)
6666 # Mark the tokenlist so that as_dict() knows to serialize an empty
6667 # result as {} rather than [], preserving dict semantics even when
6668 # there are no matched entries.
6669 tokenlist._is_dict_context = True
6671 if self._asPythonDict:
6672 return [tokenlist.as_dict()] if self.resultsName else tokenlist.as_dict()
6674 return [tokenlist] if self.resultsName else tokenlist
6677class Suppress(TokenConverter):
6678 """Converter for ignoring the results of a parsed expression.
6680 Example:
6682 .. doctest::
6684 >>> source = "a, b, c,d"
6685 >>> wd = Word(alphas)
6686 >>> wd_list1 = wd + (',' + wd)[...]
6687 >>> print(wd_list1.parse_string(source))
6688 ['a', ',', 'b', ',', 'c', ',', 'd']
6690 # often, delimiters that are useful during parsing are just in the
6691 # way afterward - use Suppress to keep them out of the parsed output
6692 >>> wd_list2 = wd + (Suppress(',') + wd)[...]
6693 >>> print(wd_list2.parse_string(source))
6694 ['a', 'b', 'c', 'd']
6696 # Skipped text (using '...') can be suppressed as well
6697 >>> source = "lead in START relevant text END trailing text"
6698 >>> start_marker = Keyword("START")
6699 >>> end_marker = Keyword("END")
6700 >>> find_body = Suppress(...) + start_marker + ... + end_marker
6701 >>> print(find_body.parse_string(source))
6702 ['START', 'relevant text ', 'END']
6704 (See also :class:`DelimitedList`.)
6705 """
6707 def __init__(self, expr: Union[ParserElement, str], savelist: bool = False) -> None:
6708 if expr is ...:
6709 expr = _PendingSkip(NoMatch())
6710 super().__init__(expr)
6712 def __add__(self, other) -> ParserElement:
6713 if isinstance(self.expr, _PendingSkip):
6714 return Suppress(SkipTo(other)) + other
6716 return super().__add__(other)
6718 def __sub__(self, other) -> ParserElement:
6719 if isinstance(self.expr, _PendingSkip):
6720 return Suppress(SkipTo(other)) - other
6722 return super().__sub__(other)
6724 def postParse(self, instring, loc, tokenlist):
6725 return []
6727 def suppress(self) -> ParserElement:
6728 return self
6731# XXX: Example needs to be re-done for updated output
6732def trace_parse_action(f: ParseAction) -> ParseAction:
6733 """Decorator for debugging parse actions.
6735 When the parse action is called, this decorator will print
6736 ``">> entering method-name(line:<current_source_line>, <parse_location>, <matched_tokens>)"``.
6737 When the parse action completes, the decorator will print
6738 ``"<<"`` followed by the returned value, or any exception that the parse action raised.
6740 Example:
6742 .. testsetup:: stderr
6744 import sys
6745 sys.stderr = sys.stdout
6747 .. testcleanup:: stderr
6749 sys.stderr = sys.__stderr__
6751 .. testcode:: stderr
6753 wd = Word(alphas)
6755 @trace_parse_action
6756 def remove_duplicate_chars(tokens):
6757 return ''.join(sorted(set(''.join(tokens))))
6759 wds = wd[1, ...].set_parse_action(remove_duplicate_chars)
6760 print(wds.parse_string("slkdjs sld sldd sdlf sdljf"))
6762 prints:
6764 .. testoutput:: stderr
6765 :options: +NORMALIZE_WHITESPACE
6767 >>entering remove_duplicate_chars(line: 'slkdjs sld sldd sdlf sdljf',
6768 0, ParseResults(['slkdjs', 'sld', 'sldd', 'sdlf', 'sdljf'], {}))
6769 <<leaving remove_duplicate_chars (ret: 'dfjkls')
6770 ['dfjkls']
6772 .. versionchanged:: 3.1.0
6773 Exception type added to output
6774 """
6775 f = _trim_arity(f)
6777 def z(*paArgs):
6778 thisFunc = f.__name__
6779 s, l, t = paArgs[-3:]
6780 if len(paArgs) > 3:
6781 thisFunc = f"{type(paArgs[0]).__name__}.{thisFunc}"
6782 sys.stderr.write(f">>entering {thisFunc}(line: {line(l, s)!r}, {l}, {t!r})\n")
6783 try:
6784 ret = f(*paArgs)
6785 except Exception as exc:
6786 sys.stderr.write(
6787 f"<<leaving {thisFunc} (exception: {type(exc).__name__}: {exc})\n"
6788 )
6789 raise
6790 sys.stderr.write(f"<<leaving {thisFunc} (ret: {ret!r})\n")
6791 return ret
6793 z.__name__ = f.__name__
6794 return z
6797# convenience constants for positional expressions
6798empty = Empty().set_name("empty")
6799line_start = LineStart().set_name("line_start")
6800line_end = LineEnd().set_name("line_end")
6801string_start = StringStart().set_name("string_start")
6802string_end = StringEnd().set_name("string_end")
6804_escapedPunc = Regex(r"\\[\\[\]\/\-\*\.\$\+\^\?()~ ]").set_parse_action(
6805 lambda s, l, t: t[0][1]
6806)
6807_escapedHexChar = Regex(r"\\0?[xX][0-9a-fA-F]+").set_parse_action(
6808 lambda s, l, t: chr(int(t[0].lstrip(r"\0x"), 16))
6809)
6810_escapedOctChar = Regex(r"\\0[0-7]+").set_parse_action(
6811 lambda s, l, t: chr(int(t[0][1:], 8))
6812)
6813_singleChar = (
6814 _escapedPunc | _escapedHexChar | _escapedOctChar | CharsNotIn(r"\]", exact=1)
6815)
6816_charRange = Group(_singleChar + Suppress("-") + _singleChar)
6817_reBracketExpr = (
6818 Literal("[")
6819 + Opt("^").set_results_name("negate")
6820 + Group(OneOrMore(_charRange | _singleChar)).set_results_name("body")
6821 + Literal("]")
6822)
6825def srange(s: str) -> str:
6826 r"""Helper to easily define string ranges for use in :class:`Word`
6827 construction. Borrows syntax from regexp ``'[]'`` string range
6828 definitions::
6830 srange("[0-9]") -> "0123456789"
6831 srange("[a-z]") -> "abcdefghijklmnopqrstuvwxyz"
6832 srange("[a-z$_]") -> "abcdefghijklmnopqrstuvwxyz$_"
6834 The input string must be enclosed in []'s, and the returned string
6835 is the expanded character set joined into a single string. The
6836 values enclosed in the []'s may be:
6838 - a single character
6839 - an escaped character with a leading backslash (such as ``\-``
6840 or ``\]``)
6841 - an escaped hex character with a leading ``'\x'``
6842 (``\x21``, which is a ``'!'`` character) (``\0x##``
6843 is also supported for backwards compatibility)
6844 - an escaped octal character with a leading ``'\0'``
6845 (``\041``, which is a ``'!'`` character)
6846 - a range of any of the above, separated by a dash (``'a-z'``,
6847 etc.)
6848 - any combination of the above (``'aeiouy'``,
6849 ``'a-zA-Z0-9_$'``, etc.)
6850 """
6852 def _expanded(p):
6853 if isinstance(p, ParseResults):
6854 yield from (chr(c) for c in range(ord(p[0]), ord(p[1]) + 1))
6855 else:
6856 yield p
6858 try:
6859 return "".join(
6860 [c for part in _reBracketExpr.parse_string(s).body for c in _expanded(part)]
6861 )
6862 except Exception as e:
6863 return ""
6866def token_map(func, *args) -> ParseAction:
6867 """Helper to define a parse action by mapping a function to all
6868 elements of a :class:`ParseResults` list. If any additional args are passed,
6869 they are forwarded to the given function as additional arguments
6870 after the token, as in
6871 ``hex_integer = Word(hexnums).set_parse_action(token_map(int, 16))``,
6872 which will convert the parsed data to an integer using base 16.
6874 Example (compare the last to example in :class:`ParserElement.transform_string`::
6876 hex_ints = Word(hexnums)[1, ...].set_parse_action(token_map(int, 16))
6877 hex_ints.run_tests('''
6878 00 11 22 aa FF 0a 0d 1a
6879 ''')
6881 upperword = Word(alphas).set_parse_action(token_map(str.upper))
6882 upperword[1, ...].run_tests('''
6883 my kingdom for a horse
6884 ''')
6886 wd = Word(alphas).set_parse_action(token_map(str.title))
6887 wd[1, ...].set_parse_action(' '.join).run_tests('''
6888 now is the winter of our discontent made glorious summer by this sun of york
6889 ''')
6891 prints::
6893 00 11 22 aa FF 0a 0d 1a
6894 [0, 17, 34, 170, 255, 10, 13, 26]
6896 my kingdom for a horse
6897 ['MY', 'KINGDOM', 'FOR', 'A', 'HORSE']
6899 now is the winter of our discontent made glorious summer by this sun of york
6900 ['Now Is The Winter Of Our Discontent Made Glorious Summer By This Sun Of York']
6901 """
6903 def pa(s, l, t):
6904 return [func(tokn, *args) for tokn in t]
6906 func_name = getattr(func, "__name__", getattr(func, "__class__").__name__)
6907 pa.__name__ = func_name
6909 return pa
6912def autoname_elements() -> None:
6913 """
6914 Utility to simplify mass-naming of parser elements, for
6915 generating railroad diagram with named subdiagrams.
6916 """
6918 # guard against _getframe not being implemented in the current Python
6919 getframe_fn = getattr(sys, "_getframe", lambda _: None)
6920 calling_frame = getframe_fn(1)
6921 if calling_frame is None:
6922 return
6924 # find all locals in the calling frame that are ParserElements
6925 calling_frame = typing.cast(types.FrameType, calling_frame)
6926 for name, var in calling_frame.f_locals.items():
6927 # if no custom name defined, set the name to the var name
6928 if isinstance(var, ParserElement) and not var.customName:
6929 var.set_name(name)
6932dbl_quoted_string = (
6933 Regex(r'"(?:[^"\n\r\\]|(?:"")|(?:\\(?:[^x]|x[0-9a-fA-F]+)))*"')
6934).set_name("string enclosed in double quotes")
6936sgl_quoted_string = Regex(
6937 r"'(?:[^'\n\r\\]|(?:'')|(?:\\(?:[^x]|x[0-9a-fA-F]+)))*'"
6938).set_name("string enclosed in single quotes")
6940quoted_string = Combine(
6941 Regex(r'"(?:[^"\n\r\\]|(?:"")|(?:\\(?:[^x]|x[0-9a-fA-F]+)))*"').set_name(
6942 "double quoted string"
6943 )
6944 | Regex(r"'(?:[^'\n\r\\]|(?:'')|(?:\\(?:[^x]|x[0-9a-fA-F]+)))*'").set_name(
6945 "single quoted string"
6946 )
6947).set_name("quoted string using single or double quotes")
6949# XXX: Is there some way to make this show up in API docs?
6950# .. versionadded:: 3.1.0
6951python_quoted_string = Combine(
6952 Regex(r'"""(?:[^"\\]|""(?!")|"(?!"")|\\.)*"""', flags=re.MULTILINE).set_name(
6953 "multiline double quoted string"
6954 )
6955 ^ Regex(r"'''(?:[^'\\]|''(?!')|'(?!'')|\\.)*'''", flags=re.MULTILINE).set_name(
6956 "multiline single quoted string"
6957 )
6958 ^ Regex(r'"(?:[^"\n\r\\]|(?:\\")|(?:\\(?:[^x]|x[0-9a-fA-F]+)))*"').set_name(
6959 "double quoted string"
6960 )
6961 ^ Regex(r"'(?:[^'\n\r\\]|(?:\\')|(?:\\(?:[^x]|x[0-9a-fA-F]+)))*'").set_name(
6962 "single quoted string"
6963 )
6964).set_name("Python quoted string")
6966unicode_string = Combine("u" + quoted_string.copy()).set_name("unicode string literal")
6969alphas8bit = srange(r"[\0xc0-\0xd6\0xd8-\0xf6\0xf8-\0xff]")
6970punc8bit = srange(r"[\0xa1-\0xbf\0xd7\0xf7]")
6972# build list of built-in expressions, for future reference if a global default value
6973# gets updated
6974_builtin_exprs: list[ParserElement] = [
6975 v for v in vars().values() if isinstance(v, ParserElement)
6976]
6978# Compatibility synonyms
6979# fmt: off
6980sglQuotedString = sgl_quoted_string
6981dblQuotedString = dbl_quoted_string
6982quotedString = quoted_string
6983unicodeString = unicode_string
6984lineStart = line_start
6985lineEnd = line_end
6986stringStart = string_start
6987stringEnd = string_end
6988nullDebugAction = replaced_by_pep8("nullDebugAction", null_debug_action)
6989traceParseAction = replaced_by_pep8("traceParseAction", trace_parse_action)
6990conditionAsParseAction = replaced_by_pep8("conditionAsParseAction", condition_as_parse_action)
6991tokenMap = replaced_by_pep8("tokenMap", token_map)
6992# fmt: on