Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/glom/core.py: 58%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""*glom gets results.*
3The ``glom`` package has one central entrypoint,
4:func:`glom.glom`. Everything else in the package revolves around that
5one function. Sometimes, big things come in small packages.
7A couple of conventional terms you'll see repeated many times below:
9* **target** - glom is built to work on any data, so we simply
10 refer to the object being accessed as the *"target"*
11* **spec** - *(aka "glomspec", short for specification)* The
12 accompanying template used to specify the structure of the return
13 value.
15Now that you know the terms, let's take a look around glom's powerful
16semantics.
18"""
21import os
22import sys
23import pdb
24import copy
25import warnings
26import weakref
27import operator
28from abc import ABCMeta
29from pprint import pprint
30import string
31from collections import OrderedDict
32import traceback
34from face.helpers import get_wrap_width
35from boltons.typeutils import make_sentinel
36from boltons.iterutils import is_iterable
37#from boltons.funcutils import format_invocation
39basestring = str
40_AbstractIterableBase = ABCMeta('_AbstractIterableBase', (object,), {})
41from collections import ChainMap
42from reprlib import Repr, recursive_repr
44GLOM_DEBUG = os.getenv('GLOM_DEBUG', '').strip().lower()
45GLOM_DEBUG = False if (GLOM_DEBUG in ('', '0', 'false')) else True
47TRACE_WIDTH = max(get_wrap_width(max_width=110), 50) # min width
49PATH_STAR = True
50# should * and ** be interpreted as parallel traversal in Path.from_text()?
51# Changed to True in 23.1, this option to disable will go away soon
53_type_type = type
55_MISSING = make_sentinel('_MISSING')
56SKIP = make_sentinel('SKIP')
57SKIP.__doc__ = """
58The ``SKIP`` singleton can be returned from a function or included
59via a :class:`~glom.Val` to cancel assignment into the output
60object.
62>>> target = {'a': 'b'}
63>>> spec = {'a': lambda t: t['a'] if t['a'] == 'a' else SKIP}
64>>> glom(target, spec)
65{}
66>>> target = {'a': 'a'}
67>>> glom(target, spec)
68{'a': 'a'}
70Mostly used to drop keys from dicts (as above) or filter objects from
71lists.
73.. note::
75 SKIP was known as OMIT in versions 18.3.1 and prior. Versions 19+
76 will remove the OMIT alias entirely.
77"""
78OMIT = SKIP # backwards compat, remove in 19+
80STOP = make_sentinel('STOP')
81STOP.__doc__ = """
82The ``STOP`` singleton can be used to halt iteration of a list or
83execution of a tuple of subspecs.
85>>> target = range(10)
86>>> spec = [lambda x: x if x < 5 else STOP]
87>>> glom(target, spec)
88[0, 1, 2, 3, 4]
89"""
91LAST_CHILD_SCOPE = make_sentinel('LAST_CHILD_SCOPE')
92LAST_CHILD_SCOPE.__doc__ = """
93Marker that can be used by parents to keep track of the last child
94scope executed. Useful for "lifting" results out of child scopes
95for scopes that want to chain the scopes of their children together
96similar to tuple.
97"""
99NO_PYFRAME = make_sentinel('NO_PYFRAME')
100NO_PYFRAME.__doc__ = """
101Used internally to mark scopes which are no longer wrapped
102in a recursive glom() call, so that they can be cleaned up correctly
103in case of exceptions
104"""
106MODE = make_sentinel('MODE')
108MIN_MODE = make_sentinel('MIN_MODE')
110CHILD_ERRORS = make_sentinel('CHILD_ERRORS')
111CHILD_ERRORS.__doc__ = """
112``CHILD_ERRORS`` is used by glom internals to keep track of
113failed child branches of the current scope.
114"""
116CUR_ERROR = make_sentinel('CUR_ERROR')
117CUR_ERROR.__doc__ = """
118``CUR_ERROR`` is used by glom internals to keep track of
119thrown exceptions.
120"""
122_PKG_DIR_PATH = os.path.dirname(os.path.abspath(__file__))
124class GlomError(Exception):
125 """The base exception for all the errors that might be raised from
126 :func:`glom` processing logic.
128 By default, exceptions raised from within functions passed to glom
129 (e.g., ``len``, ``sum``, any ``lambda``) will not be wrapped in a
130 GlomError.
131 """
132 @classmethod
133 def wrap(cls, exc):
134 # TODO: need to test this against a wide array of exception types
135 # this approach to wrapping errors works for exceptions
136 # defined in pure-python as well as C
137 exc_type = type(exc)
138 bases = (GlomError,) if issubclass(GlomError, exc_type) else (exc_type, GlomError)
139 exc_wrapper_type = type(f"GlomError.wrap({exc_type.__name__})", bases, {})
140 try:
141 wrapper = exc_wrapper_type(*exc.args)
142 wrapper.__wrapped = exc
143 return wrapper
144 except Exception: # maybe exception can't be re-created
145 return exc
147 def _set_wrapped(self, exc):
148 self.__wrapped = exc
150 def _finalize(self, scope):
151 # careful when changing how this functionality works; pytest seems to mess with
152 # the traceback module or sys.exc_info(). we saw different stacks when originally
153 # developing this in June 2020.
154 etype, evalue, _ = sys.exc_info()
155 tb_lines = traceback.format_exc().strip().splitlines()
156 limit = 0
157 for line in reversed(tb_lines):
158 if _PKG_DIR_PATH in line:
159 limit -= 1
160 break
161 limit += 1
162 self._tb_lines = tb_lines[-limit:]
163 # if the first line is trying to put a caret at a byte-code location on a line that
164 # isn't being displayed, skip it
165 if set(self._tb_lines[0]) <= {' ', '^', '~'}:
166 self._tb_lines = self._tb_lines[1:]
167 self._scope = scope
169 def __str__(self):
170 if getattr(self, '_finalized_str', None):
171 return self._finalized_str
172 elif getattr(self, '_scope', None) is not None:
173 self._target_spec_trace = format_target_spec_trace(self._scope, self.__wrapped)
174 parts = ["error raised while processing, details below.",
175 " Target-spec trace (most recent last):",
176 self._target_spec_trace]
177 parts.extend(self._tb_lines)
178 self._finalized_str = "\n".join(parts)
179 return self._finalized_str
181 # else, not finalized
182 try:
183 exc_get_message = self.get_message
184 except AttributeError:
185 exc_get_message = super().__str__
186 return exc_get_message()
189def _unpack_stack(scope, only_errors=True):
190 """
191 convert scope to [[scope, spec, target, error, [children]]]
193 this is a convenience method for printing stacks
195 only_errors=True means ignore branches which may still be hanging around
196 which were not involved in the stack trace of the error
198 only_errors=False could be useful for debugger / introspection (similar
199 to traceback.print_stack())
200 """
201 stack = []
202 scope = scope.maps[0]
203 while LAST_CHILD_SCOPE in scope:
204 child = scope[LAST_CHILD_SCOPE]
205 branches = scope[CHILD_ERRORS]
206 if branches == [child]:
207 branches = [] # if there's only one branch, count it as linear
208 stack.append([scope, scope[Spec], scope[T], scope.get(CUR_ERROR), branches])
210 # NB: this id() business is necessary to avoid a
211 # nondeterministic bug in abc's __eq__ see #189 for details
212 if id(child) in [id(b) for b in branches]:
213 break # if child already covered by branches, stop the linear descent
215 scope = child.maps[0]
216 else: # if break executed above, cur scope was already added
217 stack.append([scope, scope[Spec], scope[T], scope.get(CUR_ERROR), []])
218 # push errors "down" to where they were first raised / first observed
219 for i in range(len(stack) - 1):
220 cur, nxt = stack[i], stack[i + 1]
221 if cur[3] == nxt[3]:
222 cur[3] = None
223 if only_errors: # trim the stack to the last error
224 # leave at least 1 to not break formatting func below
225 # TODO: make format_target_spec_trace() tolerate an "empty" stack cleanly
226 while len(stack) > 1 and stack[-1][3] is None:
227 stack.pop()
228 return stack
231def _format_trace_value(value, maxlen):
232 s = bbrepr(value).replace("\\'", "'")
233 if len(s) > maxlen:
234 try:
235 suffix = '... (len=%s)' % len(value)
236 except Exception:
237 suffix = '...'
238 s = s[:maxlen - len(suffix)] + suffix
239 return s
242def format_target_spec_trace(scope, root_error, width=TRACE_WIDTH, depth=0, prev_target=_MISSING, last_branch=True):
243 """
244 unpack a scope into a multi-line but short summary
245 """
246 segments = []
247 indent = " " + "|" * depth
248 tick = "| " if depth else "- "
249 def mk_fmt(label, t=None):
250 pre = indent + (t or tick) + label + ": "
251 fmt_width = width - len(pre)
252 return lambda v: pre + _format_trace_value(v, fmt_width)
253 fmt_t = mk_fmt("Target")
254 fmt_s = mk_fmt("Spec")
255 fmt_b = mk_fmt("Spec", "+ ")
256 recurse = lambda s, last=False: format_target_spec_trace(s, root_error, width, depth + 1, prev_target, last)
257 tb_exc_line = lambda e: "".join(traceback.format_exception_only(type(e), e))[:-1]
258 fmt_e = lambda e: indent + tick + tb_exc_line(e)
259 for scope, spec, target, error, branches in _unpack_stack(scope):
260 if target is not prev_target:
261 segments.append(fmt_t(target))
262 prev_target = target
263 if branches:
264 segments.append(fmt_b(spec))
265 segments.extend([recurse(s) for s in branches[:-1]])
266 segments.append(recurse(branches[-1], last_branch))
267 else:
268 segments.append(fmt_s(spec))
269 if error is not None and error is not root_error:
270 last_line_error = True
271 segments.append(fmt_e(error))
272 else:
273 last_line_error = False
274 if depth: # \ on first line, X on last line
275 remark = lambda s, m: s[:depth + 1] + m + s[depth + 2:]
276 segments[0] = remark(segments[0], "\\")
277 if not last_branch or last_line_error:
278 segments[-1] = remark(segments[-1], "X")
279 return "\n".join(segments)
282# TODO: not used (yet)
283def format_oneline_trace(scope):
284 """
285 unpack a scope into a single line summary
286 (shortest summary possible)
287 """
288 # the goal here is to do a kind of delta-compression --
289 # if the target is the same, don't repeat it
290 segments = []
291 prev_target = _MISSING
292 for scope, spec, target, error, branches in _unpack_stack(scope, only_errors=False):
293 segments.append('/')
294 if type(spec) in (TType, Path):
295 segments.append(bbrepr(spec))
296 else:
297 segments.append(type(spec).__name__)
298 if target != prev_target:
299 segments.append('!')
300 segments.append(type(target).__name__)
301 if Path in scope:
302 segments.append('<')
303 segments.append('->'.join([str(p) for p in scope[Path]]))
304 segments.append('>')
305 prev_target = target
307 return "".join(segments)
310class PathAccessError(GlomError, AttributeError, KeyError, IndexError):
311 """This :exc:`GlomError` subtype represents a failure to access an
312 attribute as dictated by the spec. The most commonly-seen error
313 when using glom, it maintains a copy of the original exception and
314 produces a readable error message for easy debugging.
316 If you see this error, you may want to:
318 * Check the target data is accurate using :class:`~glom.Inspect`
319 * Catch the exception and return a semantically meaningful error message
320 * Use :class:`glom.Coalesce` to specify a default
321 * Use the top-level ``default`` kwarg on :func:`~glom.glom()`
323 In any case, be glad you got this error and not the one it was
324 wrapping!
326 Args:
327 exc (Exception): The error that arose when we tried to access
328 *path*. Typically an instance of KeyError, AttributeError,
329 IndexError, or TypeError, and sometimes others.
330 path (Path): The full Path glom was in the middle of accessing
331 when the error occurred.
332 part_idx (int): The index of the part of the *path* that caused
333 the error.
335 >>> target = {'a': {'b': None}}
336 >>> glom(target, 'a.b.c')
337 Traceback (most recent call last):
338 ...
339 PathAccessError: could not access 'c', part 2 of Path('a', 'b', 'c'), got error: ...
341 """
342 def __init__(self, exc, path, part_idx):
343 self.exc = exc
344 self.path = path
345 self.part_idx = part_idx
347 def get_message(self):
348 try:
349 path_part = self.path.values()[self.part_idx]
350 except (AttributeError, IndexError):
351 path_part = self.path
352 return ('could not access %r, part %r of %r, got error: %r'
353 % (path_part, self.part_idx, self.path, self.exc))
355 def __repr__(self):
356 cn = self.__class__.__name__
357 return f'{cn}({self.exc!r}, {self.path!r}, {self.part_idx!r})'
360class PathAssignError(GlomError):
361 """This :exc:`GlomError` subtype is raised when an assignment fails,
362 stemming from an :func:`~glom.assign` call or other
363 :class:`~glom.Assign` usage.
365 One example would be assigning to an out-of-range position in a list::
367 >>> assign(["short", "list"], Path(5), 'too far') # doctest: +SKIP
368 Traceback (most recent call last):
369 ...
370 PathAssignError: could not assign 5 on object at Path(), got error: IndexError(...
372 Other assignment failures could be due to assigning to an
373 ``@property`` or exception being raised inside a ``__setattr__()``.
375 """
376 def __init__(self, exc, path, dest_name):
377 self.exc = exc
378 self.path = path
379 self.dest_name = dest_name
381 def get_message(self):
382 return ('could not assign %r on object at %r, got error: %r'
383 % (self.dest_name, self.path, self.exc))
385 def __repr__(self):
386 cn = self.__class__.__name__
387 return f'{cn}({self.exc!r}, {self.path!r}, {self.dest_name!r})'
390class CoalesceError(GlomError):
391 """This :exc:`GlomError` subtype is raised from within a
392 :class:`Coalesce` spec's processing, when none of the subspecs
393 match and no default is provided.
395 The exception object itself keeps track of several values which
396 may be useful for processing:
398 Args:
399 coal_obj (Coalesce): The original failing spec, see
400 :class:`Coalesce`'s docs for details.
401 skipped (list): A list of ignored values and exceptions, in the
402 order that their respective subspecs appear in the original
403 *coal_obj*.
404 path: Like many GlomErrors, this exception knows the path at
405 which it occurred.
407 >>> target = {}
408 >>> glom(target, Coalesce('a', 'b'))
409 Traceback (most recent call last):
410 ...
411 CoalesceError: no valid values found. Tried ('a', 'b') and got (PathAccessError, PathAccessError) ...
413 .. note::
415 Coalesce is a *branching* specifier type, so as of v20.7.0, its
416 exception messages feature an error tree. See
417 :ref:`branched-exceptions` for details on how to interpret these
418 exceptions.
420 """
421 def __init__(self, coal_obj, skipped, path):
422 self.coal_obj = coal_obj
423 self.skipped = skipped
424 self.path = path
426 def __repr__(self):
427 cn = self.__class__.__name__
428 return f'{cn}({self.coal_obj!r}, {self.skipped!r}, {self.path!r})'
430 def get_message(self):
431 missed_specs = tuple(self.coal_obj.subspecs)
432 skipped_vals = [v.__class__.__name__
433 if isinstance(v, self.coal_obj.skip_exc)
434 else '<skipped %s>' % v.__class__.__name__
435 for v in self.skipped]
436 msg = ('no valid values found. Tried %r and got (%s)'
437 % (missed_specs, ', '.join(skipped_vals)))
438 if self.coal_obj.skip is not _MISSING:
439 msg += f', skip set to {self.coal_obj.skip!r}'
440 if self.coal_obj.skip_exc is not GlomError:
441 msg += f', skip_exc set to {self.coal_obj.skip_exc!r}'
442 if self.path is not None:
443 msg += f' (at path {self.path!r})'
444 return msg
447class BadSpec(GlomError, TypeError):
448 """Raised when a spec structure is malformed, e.g., when a specifier
449 type is invalid for the current mode."""
452class UnregisteredTarget(GlomError):
453 """This :class:`GlomError` subtype is raised when a spec calls for an
454 unsupported action on a target type. For instance, trying to
455 iterate on an non-iterable target:
457 >>> glom(object(), ['a.b.c'])
458 Traceback (most recent call last):
459 ...
460 UnregisteredTarget: target type 'object' not registered for 'iterate', expected one of registered types: (...)
462 It should be noted that this is a pretty uncommon occurrence in
463 production glom usage. See the :ref:`setup-and-registration`
464 section for details on how to avoid this error.
466 An UnregisteredTarget takes and tracks a few values:
468 Args:
469 op (str): The name of the operation being performed ('get' or 'iterate')
470 target_type (type): The type of the target being processed.
471 type_map (dict): A mapping of target types that do support this operation
472 path: The path at which the error occurred.
474 """
475 def __init__(self, op, target_type, type_map, path):
476 self.op = op
477 self.target_type = target_type
478 self.type_map = type_map
479 self.path = path
480 super().__init__(op, target_type, type_map, path)
482 def __repr__(self):
483 cn = self.__class__.__name__
484 # <type %r> is because Python 3 inexplicably changed the type
485 # repr from <type *> to <class *>
486 return ('%s(%r, <type %r>, %r, %r)'
487 % (cn, self.op, self.target_type.__name__, self.type_map, self.path))
489 def get_message(self):
490 if not self.type_map:
491 return ("glom() called without registering any types for operation '%s'. see"
492 " glom.register() or Glommer's constructor for details." % (self.op,))
493 reg_types = sorted([t.__name__ for t, h in self.type_map.items() if h])
494 reg_types_str = '()' if not reg_types else ('(%s)' % ', '.join(reg_types))
495 msg = ("target type %r not registered for '%s', expected one of"
496 " registered types: %s" % (self.target_type.__name__, self.op, reg_types_str))
497 if self.path:
498 msg += f' (at {self.path!r})'
499 return msg
502if getattr(__builtins__, '__dict__', None) is not None:
503 # pypy's __builtins__ is a module, as is CPython's REPL, but at
504 # normal execution time it's a dict?
505 __builtins__ = __builtins__.__dict__
508_BUILTIN_ID_NAME_MAP = {id(v): k
509 for k, v in __builtins__.items()}
512class _BBRepr(Repr):
513 """A better repr for builtins, when the built-in repr isn't
514 roundtrippable.
515 """
516 def __init__(self):
517 super().__init__()
518 # turn up all the length limits very high
519 for name in self.__dict__:
520 if not isinstance(getattr(self, name), int):
521 continue
522 setattr(self, name, 1024)
524 def repr1(self, x, level):
525 ret = Repr.repr1(self, x, level)
526 if not ret.startswith('<'):
527 return ret
528 return _BUILTIN_ID_NAME_MAP.get(id(x), ret)
531bbrepr = recursive_repr()(_BBRepr().repr)
534class _BBReprFormatter(string.Formatter):
535 """
536 allow format strings to be evaluated where {!r} will use bbrepr
537 instead of repr
538 """
539 def convert_field(self, value, conversion):
540 if conversion == 'r':
541 return bbrepr(value).replace("\\'", "'")
542 return super().convert_field(value, conversion)
545bbformat = _BBReprFormatter().format
548# TODO: push this back up to boltons with repr kwarg
549def format_invocation(name='', args=(), kwargs=None, **kw):
550 """Given a name, positional arguments, and keyword arguments, format
551 a basic Python-style function call.
553 >>> print(format_invocation('func', args=(1, 2), kwargs={'c': 3}))
554 func(1, 2, c=3)
555 >>> print(format_invocation('a_func', args=(1,)))
556 a_func(1)
557 >>> print(format_invocation('kw_func', kwargs=[('a', 1), ('b', 2)]))
558 kw_func(a=1, b=2)
560 """
561 _repr = kw.pop('repr', bbrepr)
562 if kw:
563 raise TypeError('unexpected keyword args: %r' % ', '.join(kw.keys()))
564 kwargs = kwargs or {}
565 a_text = ', '.join([_repr(a) for a in args])
566 if isinstance(kwargs, dict):
567 kwarg_items = [(k, kwargs[k]) for k in sorted(kwargs)]
568 else:
569 kwarg_items = kwargs
570 kw_text = ', '.join([f'{k}={_repr(v)}' for k, v in kwarg_items])
572 all_args_text = a_text
573 if all_args_text and kw_text:
574 all_args_text += ', '
575 all_args_text += kw_text
577 return f'{name}({all_args_text})'
580class Path:
581 """Path objects specify explicit paths when the default
582 ``'a.b.c'``-style general access syntax won't work or isn't
583 desirable. Use this to wrap ints, datetimes, and other valid
584 keys, as well as strings with dots that shouldn't be expanded.
586 >>> target = {'a': {'b': 'c', 'd.e': 'f', 2: 3}}
587 >>> glom(target, Path('a', 2))
588 3
589 >>> glom(target, Path('a', 'd.e'))
590 'f'
592 Paths can be used to join together other Path objects, as
593 well as :data:`~glom.T` objects:
595 >>> Path(T['a'], T['b'])
596 T['a']['b']
597 >>> Path(Path('a', 'b'), Path('c', 'd'))
598 Path('a', 'b', 'c', 'd')
600 Paths also support indexing and slicing, with each access
601 returning a new Path object:
603 >>> path = Path('a', 'b', 1, 2)
604 >>> path[0]
605 Path('a')
606 >>> path[-2:]
607 Path(1, 2)
609 To build a Path object from a string, use :meth:`Path.from_text()`.
610 This is the default behavior when the top-level :func:`~glom.glom`
611 function gets a string spec.
612 """
613 def __init__(self, *path_parts):
614 if not path_parts:
615 self.path_t = T
616 return
617 if isinstance(path_parts[0], TType):
618 path_t = path_parts[0]
619 offset = 1
620 else:
621 path_t = T
622 offset = 0
623 for part in path_parts[offset:]:
624 if isinstance(part, Path):
625 part = part.path_t
626 if isinstance(part, TType):
627 sub_parts = part.__ops__
628 if sub_parts[0] is not T:
629 raise ValueError('path segment must be path from T, not %r'
630 % sub_parts[0])
631 i = 1
632 while i < len(sub_parts):
633 path_t = _t_child(path_t, sub_parts[i], sub_parts[i + 1])
634 i += 2
635 else:
636 path_t = _t_child(path_t, 'P', part)
637 self.path_t = path_t
639 _CACHE = {True: {}, False: {}}
640 _MAX_CACHE = 10000
641 _STAR_WARNED = False
643 @classmethod
644 def from_text(cls, text):
645 """Make a Path from .-delimited text:
647 >>> Path.from_text('a.b.c')
648 Path('a', 'b', 'c')
650 This is the default behavior when :func:`~glom.glom` gets a string spec.
651 """
652 def create():
653 segs = text.split('.')
654 if PATH_STAR:
655 segs = [
656 _T_STAR if seg == '*' else
657 _T_STARSTAR if seg == '**' else seg
658 for seg in segs]
659 elif not cls._STAR_WARNED:
660 if '*' in segs or '**' in segs:
661 warnings.warn(
662 "'*' and '**' have changed behavior in glom version 23.1."
663 " Recommend switch to T['*'] or T['**'].")
664 cls._STAR_WARNED = True
665 return cls(*segs)
667 cache = cls._CACHE[PATH_STAR] # remove this when PATH_STAR is default
668 if text not in cache:
669 if len(cache) > cls._MAX_CACHE:
670 return create()
671 cache[text] = create()
672 return cache[text]
674 def glomit(self, target, scope):
675 # The entrypoint for the Path extension
676 return _t_eval(target, self.path_t, scope)
678 def __len__(self):
679 return (len(self.path_t.__ops__) - 1) // 2
681 def __eq__(self, other):
682 if type(other) is Path:
683 return self.path_t.__ops__ == other.path_t.__ops__
684 elif type(other) is TType:
685 return self.path_t.__ops__ == other.__ops__
686 return False
688 def __ne__(self, other):
689 return not self == other
691 def values(self):
692 """
693 Returns a tuple of values referenced in this path.
695 >>> Path(T.a.b, 'c', T['d']).values()
696 ('a', 'b', 'c', 'd')
697 """
698 cur_t_path = self.path_t.__ops__
699 return cur_t_path[2::2]
701 def items(self):
702 """
703 Returns a tuple of (operation, value) pairs.
705 >>> Path(T.a.b, 'c', T['d']).items()
706 (('.', 'a'), ('.', 'b'), ('P', 'c'), ('[', 'd'))
708 """
709 cur_t_path = self.path_t.__ops__
710 return tuple(zip(cur_t_path[1::2], cur_t_path[2::2]))
712 def startswith(self, other):
713 if isinstance(other, basestring):
714 other = Path(other)
715 if isinstance(other, Path):
716 other = other.path_t
717 if not isinstance(other, TType):
718 raise TypeError('can only check if Path starts with string, Path or T')
719 o_path = other.__ops__
720 return self.path_t.__ops__[:len(o_path)] == o_path
722 def from_t(self):
723 '''return the same path but starting from T'''
724 t_path = self.path_t.__ops__
725 if t_path[0] is S:
726 new_t = TType()
727 new_t.__ops__ = (T,) + t_path[1:]
728 return Path(new_t)
729 return self
731 def __getitem__(self, i):
732 cur_t_path = self.path_t.__ops__
733 try:
734 step = i.step
735 start = i.start if i.start is not None else 0
736 stop = i.stop
738 start = (start * 2) + 1 if start >= 0 else (start * 2) + len(cur_t_path)
739 if stop is not None:
740 stop = (stop * 2) + 1 if stop >= 0 else (stop * 2) + len(cur_t_path)
741 except AttributeError:
742 step = 1
743 start = (i * 2) + 1 if i >= 0 else (i * 2) + len(cur_t_path)
744 if start < 0 or start >= len(cur_t_path):
745 raise IndexError('Path index %d out of range for Path of length %d' % (i, (len(cur_t_path) - 1) // 2))
746 stop = ((i + 1) * 2) + 1 if i >= 0 else ((i + 1) * 2) + len(cur_t_path)
748 new_t = TType()
749 new_path = cur_t_path[start:stop]
750 if step is not None and step != 1:
751 new_path = tuple(zip(new_path[::2], new_path[1::2]))[::step]
752 new_path = sum(new_path, ())
753 new_t.__ops__ = (cur_t_path[0],) + new_path
754 return Path(new_t)
756 def __repr__(self):
757 return _format_path(self.path_t.__ops__[1:])
760def _format_path(t_path):
761 path_parts, cur_t_path = [], []
762 i = 0
763 while i < len(t_path):
764 op, arg = t_path[i], t_path[i + 1]
765 i += 2
766 if op == 'P':
767 if cur_t_path:
768 path_parts.append(cur_t_path)
769 cur_t_path = []
770 path_parts.append(arg)
771 else:
772 cur_t_path.append(op)
773 cur_t_path.append(arg)
774 if path_parts and cur_t_path:
775 path_parts.append(cur_t_path)
777 if path_parts or not cur_t_path:
778 return 'Path(%s)' % ', '.join([_format_t(part)
779 if type(part) is list else repr(part)
780 for part in path_parts])
781 return _format_t(cur_t_path)
784class Spec:
785 """Spec objects serve three purposes, here they are, roughly ordered
786 by utility:
788 1. As a form of compiled or "curried" glom call, similar to
789 Python's built-in :func:`re.compile`.
790 2. A marker as an object as representing a spec rather than a
791 literal value in certain cases where that might be ambiguous.
792 3. A way to update the scope within another Spec.
794 In the second usage, Spec objects are the complement to
795 :class:`~glom.Val`, wrapping a value and marking that it
796 should be interpreted as a glom spec, rather than a literal value.
797 This is useful in places where it would be interpreted as a value
798 by default. (Such as T[key], Call(func) where key and func are
799 assumed to be literal values and not specs.)
801 Args:
802 spec: The glom spec.
803 scope (dict): additional values to add to the scope when
804 evaluating this Spec
806 """
807 def __init__(self, spec, scope=None):
808 self.spec = spec
809 self.scope = scope or {}
811 def glom(self, target, **kw):
812 scope = dict(self.scope)
813 scope.update(kw.get('scope', {}))
814 kw['scope'] = ChainMap(scope)
815 glom_ = scope.get(glom, glom)
816 return glom_(target, self.spec, **kw)
818 def glomit(self, target, scope):
819 scope.update(self.scope)
820 return scope[glom](target, self.spec, scope)
822 def __repr__(self):
823 cn = self.__class__.__name__
824 if self.scope:
825 return f'{cn}({bbrepr(self.spec)}, scope={self.scope!r})'
826 return f'{cn}({bbrepr(self.spec)})'
829class Coalesce:
830 """Coalesce objects specify fallback behavior for a list of
831 subspecs.
833 Subspecs are passed as positional arguments, and keyword arguments
834 control defaults. Each subspec is evaluated in turn, and if none
835 match, a :exc:`CoalesceError` is raised, or a default is returned,
836 depending on the options used.
838 .. note::
840 This operation may seem very familar if you have experience with
841 `SQL`_ or even `C# and others`_.
844 In practice, this fallback behavior's simplicity is only surpassed
845 by its utility:
847 >>> target = {'c': 'd'}
848 >>> glom(target, Coalesce('a', 'b', 'c'))
849 'd'
851 glom tries to get ``'a'`` from ``target``, but gets a
852 KeyError. Rather than raise a :exc:`~glom.PathAccessError` as usual,
853 glom *coalesces* into the next subspec, ``'b'``. The process
854 repeats until it gets to ``'c'``, which returns our value,
855 ``'d'``. If our value weren't present, we'd see:
857 >>> target = {}
858 >>> glom(target, Coalesce('a', 'b'))
859 Traceback (most recent call last):
860 ...
861 CoalesceError: no valid values found. Tried ('a', 'b') and got (PathAccessError, PathAccessError) ...
863 Same process, but because ``target`` is empty, we get a
864 :exc:`CoalesceError`.
866 .. note::
868 Coalesce is a *branching* specifier type, so as of v20.7.0, its
869 exception messages feature an error tree. See
870 :ref:`branched-exceptions` for details on how to interpret these
871 exceptions.
874 If we want to avoid an exception, and we know which value we want
875 by default, we can set *default*:
877 >>> target = {}
878 >>> glom(target, Coalesce('a', 'b', 'c'), default='d-fault')
879 'd-fault'
881 ``'a'``, ``'b'``, and ``'c'`` weren't present so we got ``'d-fault'``.
883 Args:
885 subspecs: One or more glommable subspecs
886 default: A value to return if no subspec results in a valid value
887 default_factory: A callable whose result will be returned as a default
888 skip: A value, tuple of values, or predicate function
889 representing values to ignore
890 skip_exc: An exception or tuple of exception types to catch and
891 move on to the next subspec. Defaults to :exc:`GlomError`, the
892 parent type of all glom runtime exceptions.
894 If all subspecs produce skipped values or exceptions, a
895 :exc:`CoalesceError` will be raised. For more examples, check out
896 the :doc:`tutorial`, which makes extensive use of Coalesce.
898 .. _SQL: https://en.wikipedia.org/w/index.php?title=Null_(SQL)&oldid=833093792#COALESCE
899 .. _C# and others: https://en.wikipedia.org/w/index.php?title=Null_coalescing_operator&oldid=839493322#C#
901 """
902 def __init__(self, *subspecs, **kwargs):
903 self.subspecs = subspecs
904 self._orig_kwargs = dict(kwargs)
905 self.default = kwargs.pop('default', _MISSING)
906 self.default_factory = kwargs.pop('default_factory', _MISSING)
907 if self.default and self.default_factory:
908 raise ValueError('expected one of "default" or "default_factory", not both')
909 self.skip = kwargs.pop('skip', _MISSING)
910 if self.skip is _MISSING:
911 self.skip_func = lambda v: False
912 elif callable(self.skip):
913 self.skip_func = self.skip
914 elif isinstance(self.skip, tuple):
915 self.skip_func = lambda v: v in self.skip
916 else:
917 self.skip_func = lambda v: v == self.skip
918 self.skip_exc = kwargs.pop('skip_exc', GlomError)
919 if kwargs:
920 raise TypeError(f'unexpected keyword args: {sorted(kwargs.keys())!r}')
922 def glomit(self, target, scope):
923 skipped = []
924 for subspec in self.subspecs:
925 try:
926 ret = scope[glom](target, subspec, scope)
927 if not self.skip_func(ret):
928 break
929 skipped.append(ret)
930 except self.skip_exc as e:
931 skipped.append(e)
932 continue
933 else:
934 if self.default is not _MISSING:
935 ret = arg_val(target, self.default, scope)
936 elif self.default_factory is not _MISSING:
937 ret = self.default_factory()
938 else:
939 raise CoalesceError(self, skipped, scope[Path])
940 return ret
942 def __repr__(self):
943 cn = self.__class__.__name__
944 return format_invocation(cn, self.subspecs, self._orig_kwargs, repr=bbrepr)
947class Inspect:
948 """The :class:`~glom.Inspect` specifier type provides a way to get
949 visibility into glom's evaluation of a specification, enabling
950 debugging of those tricky problems that may arise with unexpected
951 data.
953 :class:`~glom.Inspect` can be inserted into an existing spec in one of two
954 ways. First, as a wrapper around the spec in question, or second,
955 as an argument-less placeholder wherever a spec could be.
957 :class:`~glom.Inspect` supports several modes, controlled by
958 keyword arguments. Its default, no-argument mode, simply echos the
959 state of the glom at the point where it appears:
961 >>> target = {'a': {'b': {}}}
962 >>> val = glom(target, Inspect('a.b')) # wrapping a spec
963 ---
964 path: ['a.b']
965 target: {'a': {'b': {}}}
966 output: {}
967 ---
969 Debugging behavior aside, :class:`~glom.Inspect` has no effect on
970 values in the target, spec, or result.
972 Args:
973 echo (bool): Whether to print the path, target, and output of
974 each inspected glom. Defaults to True.
975 recursive (bool): Whether or not the Inspect should be applied
976 at every level, at or below the spec that it wraps. Defaults
977 to False.
978 breakpoint (bool): This flag controls whether a debugging prompt
979 should appear before evaluating each inspected spec. Can also
980 take a callable. Defaults to False.
981 post_mortem (bool): This flag controls whether exceptions
982 should be caught and interactively debugged with :mod:`pdb` on
983 inspected specs.
985 All arguments above are keyword-only to avoid overlap with a
986 wrapped spec.
988 .. note::
990 Just like ``pdb.set_trace()``, be careful about leaving stray
991 ``Inspect()`` instances in production glom specs.
993 """
994 def __init__(self, *a, **kw):
995 self.wrapped = a[0] if a else Path()
996 self.recursive = kw.pop('recursive', False)
997 self.echo = kw.pop('echo', True)
998 breakpoint = kw.pop('breakpoint', False)
999 if breakpoint is True:
1000 breakpoint = pdb.set_trace
1001 if breakpoint and not callable(breakpoint):
1002 raise TypeError('breakpoint expected bool or callable, not: %r' % breakpoint)
1003 self.breakpoint = breakpoint
1004 post_mortem = kw.pop('post_mortem', False)
1005 if post_mortem is True:
1006 post_mortem = pdb.post_mortem
1007 if post_mortem and not callable(post_mortem):
1008 raise TypeError('post_mortem expected bool or callable, not: %r' % post_mortem)
1009 self.post_mortem = post_mortem
1011 def __repr__(self):
1012 return '<INSPECT>'
1014 def glomit(self, target, scope):
1015 # stash the real handler under Inspect,
1016 # and replace the child handler with a trace callback
1017 scope[Inspect] = scope[glom]
1018 scope[glom] = self._trace
1019 return scope[glom](target, self.wrapped, scope)
1021 def _trace(self, target, spec, scope):
1022 if not self.recursive:
1023 scope[glom] = scope[Inspect]
1024 if self.echo:
1025 print('---')
1026 # TODO: switch from scope[Path] to the Target-Spec format trace above
1027 # ... but maybe be smart about only printing deltas instead of the whole
1028 # thing
1029 print('path: ', scope[Path] + [spec])
1030 print('target:', target)
1031 if self.breakpoint:
1032 # TODO: real debugger here?
1033 self.breakpoint()
1034 try:
1035 ret = scope[Inspect](target, spec, scope)
1036 except Exception:
1037 if self.post_mortem:
1038 self.post_mortem()
1039 raise
1040 if self.echo:
1041 print('output:', ret)
1042 print('---')
1043 return ret
1046class Call:
1047 """:class:`Call` specifies when a target should be passed to a function,
1048 *func*.
1050 :class:`Call` is similar to :func:`~functools.partial` in that
1051 it is no more powerful than ``lambda`` or other functions, but
1052 it is designed to be more readable, with a better ``repr``.
1054 Args:
1055 func (callable): a function or other callable to be called with
1056 the target
1058 :class:`Call` combines well with :attr:`~glom.T` to construct objects. For
1059 instance, to generate a dict and then pass it to a constructor:
1061 >>> class ExampleClass(object):
1062 ... def __init__(self, attr):
1063 ... self.attr = attr
1064 ...
1065 >>> target = {'attr': 3.14}
1066 >>> glom(target, Call(ExampleClass, kwargs=T)).attr
1067 3.14
1069 This does the same as ``glom(target, lambda target:
1070 ExampleClass(**target))``, but it's easy to see which one reads
1071 better.
1073 .. note::
1075 ``Call`` is mostly for functions. Use a :attr:`~glom.T` object
1076 if you need to call a method.
1078 .. warning::
1080 :class:`Call` has a successor with a fuller-featured API, new
1081 in 19.10.0: the :class:`Invoke` specifier type.
1082 """
1083 def __init__(self, func=None, args=None, kwargs=None):
1084 if func is None:
1085 func = T
1086 if not (callable(func) or isinstance(func, (Spec, TType))):
1087 raise TypeError('expected func to be a callable or T'
1088 ' expression, not: %r' % (func,))
1089 if args is None:
1090 args = ()
1091 if kwargs is None:
1092 kwargs = {}
1093 self.func, self.args, self.kwargs = func, args, kwargs
1095 def glomit(self, target, scope):
1096 'run against the current target'
1097 r = lambda spec: arg_val(target, spec, scope)
1098 return r(self.func)(*r(self.args), **r(self.kwargs))
1100 def __repr__(self):
1101 cn = self.__class__.__name__
1102 return f'{cn}({bbrepr(self.func)}, args={self.args!r}, kwargs={self.kwargs!r})'
1105def _is_spec(obj, strict=False):
1106 # a little util for codifying the spec type checking in glom
1107 if isinstance(obj, TType):
1108 return True
1109 if strict:
1110 return type(obj) is Spec
1112 return _has_callable_glomit(obj) # pragma: no cover
1115class Invoke:
1116 """Specifier type designed for easy invocation of callables from glom.
1118 Args:
1119 func (callable): A function or other callable object.
1121 ``Invoke`` is similar to :func:`functools.partial`, but with the
1122 ability to set up a "templated" call which interleaves constants and
1123 glom specs.
1125 For example, the following creates a spec which can be used to
1126 check if targets are integers:
1128 >>> is_int = Invoke(isinstance).specs(T).constants(int)
1129 >>> glom(5, is_int)
1130 True
1132 And this composes like any other glom spec:
1134 >>> target = [7, object(), 9]
1135 >>> glom(target, [is_int])
1136 [True, False, True]
1138 Another example, mixing positional and keyword arguments:
1140 >>> spec = Invoke(sorted).specs(T).constants(key=int, reverse=True)
1141 >>> target = ['10', '5', '20', '1']
1142 >>> glom(target, spec)
1143 ['20', '10', '5', '1']
1145 Invoke also helps with evaluating zero-argument functions:
1147 >>> glom(target={}, spec=Invoke(int))
1148 0
1150 (A trivial example, but from timestamps to UUIDs, zero-arg calls do come up!)
1152 .. note::
1154 ``Invoke`` is mostly for functions, object construction, and callable
1155 objects. For calling methods, consider the :attr:`~glom.T` object.
1157 """
1158 def __init__(self, func):
1159 if not callable(func) and not _is_spec(func, strict=True):
1160 raise TypeError('expected func to be a callable or Spec instance,'
1161 ' not: %r' % (func,))
1162 self.func = func
1163 self._args = ()
1164 # a registry of every known kwarg to its freshest value as set
1165 # by the methods below. the **kw dict is used as a unique marker.
1166 self._cur_kwargs = {}
1168 @classmethod
1169 def specfunc(cls, spec):
1170 """Creates an :class:`Invoke` instance where the function is
1171 indicated by a spec.
1173 >>> spec = Invoke.specfunc('func').constants(5)
1174 >>> glom({'func': range}, (spec, list))
1175 [0, 1, 2, 3, 4]
1177 """
1178 return cls(Spec(spec))
1180 def constants(self, *a, **kw):
1181 """Returns a new :class:`Invoke` spec, with the provided positional
1182 and keyword argument values stored for passing to the
1183 underlying function.
1185 >>> spec = Invoke(T).constants(5)
1186 >>> glom(range, (spec, list))
1187 [0, 1, 2, 3, 4]
1189 Subsequent positional arguments are appended:
1191 >>> spec = Invoke(T).constants(2).constants(10, 2)
1192 >>> glom(range, (spec, list))
1193 [2, 4, 6, 8]
1195 Keyword arguments also work as one might expect:
1197 >>> round_2 = Invoke(round).constants(ndigits=2).specs(T)
1198 >>> glom(3.14159, round_2)
1199 3.14
1201 :meth:`~Invoke.constants()` and other :class:`Invoke`
1202 methods may be called multiple times, just remember that every
1203 call returns a new spec.
1204 """
1205 ret = self.__class__(self.func)
1206 ret._args = self._args + ('C', a, kw)
1207 ret._cur_kwargs = dict(self._cur_kwargs)
1208 ret._cur_kwargs.update({k: kw for k, _ in kw.items()})
1209 return ret
1211 def specs(self, *a, **kw):
1212 """Returns a new :class:`Invoke` spec, with the provided positional
1213 and keyword arguments stored to be interpreted as specs, with
1214 the results passed to the underlying function.
1216 >>> spec = Invoke(range).specs('value')
1217 >>> glom({'value': 5}, (spec, list))
1218 [0, 1, 2, 3, 4]
1220 Subsequent positional arguments are appended:
1222 >>> spec = Invoke(range).specs('start').specs('end', 'step')
1223 >>> target = {'start': 2, 'end': 10, 'step': 2}
1224 >>> glom(target, (spec, list))
1225 [2, 4, 6, 8]
1227 Keyword arguments also work as one might expect:
1229 >>> multiply = lambda x, y: x * y
1230 >>> times_3 = Invoke(multiply).constants(y=3).specs(x='value')
1231 >>> glom({'value': 5}, times_3)
1232 15
1234 :meth:`~Invoke.specs()` and other :class:`Invoke`
1235 methods may be called multiple times, just remember that every
1236 call returns a new spec.
1238 """
1239 ret = self.__class__(self.func)
1240 ret._args = self._args + ('S', a, kw)
1241 ret._cur_kwargs = dict(self._cur_kwargs)
1242 ret._cur_kwargs.update({k: kw for k, _ in kw.items()})
1243 return ret
1245 def star(self, args=None, kwargs=None):
1246 """Returns a new :class:`Invoke` spec, with *args* and/or *kwargs*
1247 specs set to be "starred" or "star-starred" (respectively)
1249 >>> spec = Invoke(zip).star(args='lists')
1250 >>> target = {'lists': [[1, 2], [3, 4], [5, 6]]}
1251 >>> list(glom(target, spec))
1252 [(1, 3, 5), (2, 4, 6)]
1254 Args:
1255 args (spec): A spec to be evaluated and "starred" into the
1256 underlying function.
1257 kwargs (spec): A spec to be evaluated and "star-starred" into
1258 the underlying function.
1260 One or both of the above arguments should be set.
1262 The :meth:`~Invoke.star()`, like other :class:`Invoke`
1263 methods, may be called multiple times. The *args* and *kwargs*
1264 will be stacked in the order in which they are provided.
1265 """
1266 if args is None and kwargs is None:
1267 raise TypeError('expected one or both of args/kwargs to be passed')
1268 ret = self.__class__(self.func)
1269 ret._args = self._args + ('*', args, kwargs)
1270 ret._cur_kwargs = dict(self._cur_kwargs)
1271 return ret
1273 def __repr__(self):
1274 base_fname = self.__class__.__name__
1275 fname_map = {'C': 'constants', 'S': 'specs', '*': 'star'}
1276 if type(self.func) is Spec:
1277 base_fname += '.specfunc'
1278 args = (self.func.spec,)
1279 else:
1280 args = (self.func,)
1281 chunks = [format_invocation(base_fname, args, repr=bbrepr)]
1283 for i in range(len(self._args) // 3):
1284 op, args, _kwargs = self._args[i * 3: i * 3 + 3]
1285 fname = fname_map[op]
1286 if op in ('C', 'S'):
1287 kwargs = [(k, v) for k, v in _kwargs.items()
1288 if self._cur_kwargs[k] is _kwargs]
1289 else:
1290 kwargs = {}
1291 if args:
1292 kwargs['args'] = args
1293 if _kwargs:
1294 kwargs['kwargs'] = _kwargs
1295 args = ()
1297 chunks.append('.' + format_invocation(fname, args, kwargs, repr=bbrepr))
1299 return ''.join(chunks)
1301 def glomit(self, target, scope):
1302 all_args = []
1303 all_kwargs = {}
1305 recurse = lambda spec: scope[glom](target, spec, scope)
1306 func = recurse(self.func) if _is_spec(self.func, strict=True) else self.func
1308 for i in range(len(self._args) // 3):
1309 op, args, kwargs = self._args[i * 3: i * 3 + 3]
1310 if op == 'C':
1311 all_args.extend(args)
1312 all_kwargs.update({k: v for k, v in kwargs.items()
1313 if self._cur_kwargs[k] is kwargs})
1314 elif op == 'S':
1315 all_args.extend([recurse(arg) for arg in args])
1316 all_kwargs.update({k: recurse(v) for k, v in kwargs.items()
1317 if self._cur_kwargs[k] is kwargs})
1318 elif op == '*':
1319 if args is not None:
1320 all_args.extend(recurse(args))
1321 if kwargs is not None:
1322 all_kwargs.update(recurse(kwargs))
1324 return func(*all_args, **all_kwargs)
1327class Ref:
1328 """Name a part of a spec and refer to it elsewhere in the same spec,
1329 useful for trees and other self-similar data structures.
1331 Args:
1332 name (str): The name of the spec to reference.
1333 subspec: Pass a spec to name it *name*, or leave unset to refer
1334 to an already-named spec.
1335 """
1336 def __init__(self, name, subspec=_MISSING):
1337 self.name, self.subspec = name, subspec
1339 def glomit(self, target, scope):
1340 subspec = self.subspec
1341 scope_key = (Ref, self.name)
1342 if subspec is _MISSING:
1343 subspec = scope[scope_key]
1344 else:
1345 scope[scope_key] = subspec
1346 return scope[glom](target, subspec, scope)
1348 def __repr__(self):
1349 if self.subspec is _MISSING:
1350 args = bbrepr(self.name)
1351 else:
1352 args = bbrepr((self.name, self.subspec))[1:-1]
1353 return "Ref(" + args + ")"
1356class TType:
1357 """``T``, short for "target". A singleton object that enables
1358 object-oriented expression of a glom specification.
1360 .. note::
1362 ``T`` is a singleton, and does not need to be constructed.
1364 Basically, think of ``T`` as your data's stunt double. Everything
1365 that you do to ``T`` will be recorded and executed during the
1366 :func:`glom` call. Take this example:
1368 >>> spec = T['a']['b']['c']
1369 >>> target = {'a': {'b': {'c': 'd'}}}
1370 >>> glom(target, spec)
1371 'd'
1373 So far, we've relied on the ``'a.b.c'``-style shorthand for
1374 access, or used the :class:`~glom.Path` objects, but if you want
1375 to explicitly do attribute and key lookups, look no further than
1376 ``T``.
1378 But T doesn't stop with unambiguous access. You can also call
1379 methods and perform almost any action you would with a normal
1380 object:
1382 >>> spec = ('a', (T['b'].items(), list)) # reviewed below
1383 >>> glom(target, spec)
1384 [('c', 'd')]
1386 A ``T`` object can go anywhere in the spec. As seen in the example
1387 above, we access ``'a'``, use a ``T`` to get ``'b'`` and iterate
1388 over its ``items``, turning them into a ``list``.
1390 You can even use ``T`` with :class:`~glom.Call` to construct objects:
1392 >>> class ExampleClass(object):
1393 ... def __init__(self, attr):
1394 ... self.attr = attr
1395 ...
1396 >>> target = {'attr': 3.14}
1397 >>> glom(target, Call(ExampleClass, kwargs=T)).attr
1398 3.14
1400 On a further note, while ``lambda`` works great in glom specs, and
1401 can be very handy at times, ``T`` and :class:`~glom.Call`
1402 eliminate the need for the vast majority of ``lambda`` usage with
1403 glom.
1405 Unlike ``lambda`` and other functions, ``T`` roundtrips
1406 beautifully and transparently:
1408 >>> T['a'].b['c']('success')
1409 T['a'].b['c']('success')
1411 ``T``-related access errors raise a :exc:`~glom.PathAccessError`
1412 during the :func:`~glom.glom` call.
1414 .. note::
1416 While ``T`` is clearly useful, powerful, and here to stay, its
1417 semantics are still being refined. Currently, operations beyond
1418 method calls and attribute/item access are considered
1419 experimental and should not be relied upon.
1421 .. note::
1423 ``T`` attributes starting with __ are reserved to avoid
1424 colliding with many built-in Python behaviors, current and
1425 future. The ``T.__()`` method is available for cases where
1426 they are needed. For example, ``T.__('class__')`` is
1427 equivalent to accessing the ``__class__`` attribute.
1429 """
1430 __slots__ = ('__ops__',)
1432 def __getattr__(self, name):
1433 if name.startswith('__'):
1434 raise AttributeError('T instances reserve dunder attributes.'
1435 ' To access the "{name}" attribute, use'
1436 ' T.__("{d_name}")'.format(name=name, d_name=name[2:]))
1437 return _t_child(self, '.', name)
1439 def __getitem__(self, item):
1440 return _t_child(self, '[', item)
1442 def __call__(self, *args, **kwargs):
1443 if self is S:
1444 if args:
1445 raise TypeError(f'S() takes no positional arguments, got: {args!r}')
1446 if not kwargs:
1447 raise TypeError('S() expected at least one kwarg, got none')
1448 # TODO: typecheck kwarg vals?
1449 return _t_child(self, '(', (args, kwargs))
1451 def __star__(self):
1452 return _t_child(self, 'x', None)
1454 def __starstar__(self):
1455 return _t_child(self, 'X', None)
1457 def __stars__(self):
1458 """how many times the result will be wrapped in extra lists"""
1459 t_ops = self.__ops__[1::2]
1460 return t_ops.count('x') + t_ops.count('X')
1462 def __add__(self, arg):
1463 return _t_child(self, '+', arg)
1465 def __sub__(self, arg):
1466 return _t_child(self, '-', arg)
1468 def __mul__(self, arg):
1469 return _t_child(self, '*', arg)
1471 def __floordiv__(self, arg):
1472 return _t_child(self, '#', arg)
1474 def __truediv__(self, arg):
1475 return _t_child(self, '/', arg)
1477 __div__ = __truediv__
1479 def __mod__(self, arg):
1480 return _t_child(self, '%', arg)
1482 def __pow__(self, arg):
1483 return _t_child(self, ':', arg)
1485 def __and__(self, arg):
1486 return _t_child(self, '&', arg)
1488 def __or__(self, arg):
1489 return _t_child(self, '|', arg)
1491 def __xor__(self, arg):
1492 return _t_child(self, '^', arg)
1494 def __invert__(self):
1495 return _t_child(self, '~', None)
1497 def __neg__(self):
1498 return _t_child(self, '_', None)
1500 def __(self, name):
1501 return _t_child(self, '.', '__' + name)
1503 def __repr__(self):
1504 t_path = self.__ops__
1505 return _format_t(t_path[1:], t_path[0])
1507 def __getstate__(self):
1508 t_path = self.__ops__
1509 return tuple(({T: 'T', S: 'S', A: 'A'}[t_path[0]],) + t_path[1:])
1511 def __setstate__(self, state):
1512 self.__ops__ = ({'T': T, 'S': S, 'A': A}[state[0]],) + state[1:]
1515def _t_child(parent, operation, arg):
1516 base = parent.__ops__
1517 if base[0] is A and operation not in ('.', '[', 'P'):
1518 # whitelist rather than blacklist assignment friendly operations
1519 # TODO: error type?
1520 raise BadSpec("operation not allowed on A assignment path")
1521 t = TType()
1522 t.__ops__ = base + (operation, arg)
1523 return t
1526def _s_first_magic(scope, key, _t):
1527 """
1528 enable S.a to do S['a'] or S['a'].val as a special
1529 case for accessing user defined string variables
1530 """
1531 err = None
1532 try:
1533 cur = scope[key]
1534 except KeyError as e:
1535 err = PathAccessError(e, Path(_t), 0) # always only one level depth, hence 0
1536 if err:
1537 raise err
1538 return cur
1541def _t_eval(target, _t, scope):
1542 t_path = _t.__ops__
1543 i = 1
1544 fetch_till = len(t_path)
1545 root = t_path[0]
1546 if root is T:
1547 cur = target
1548 elif root is S or root is A:
1549 # A is basically the same as S, but last step is assign
1550 if root is A:
1551 fetch_till -= 2
1552 if fetch_till < 1:
1553 raise BadSpec('cannot assign without destination')
1554 cur = scope
1555 if fetch_till > 1 and t_path[1] in ('.', 'P'):
1556 cur = _s_first_magic(cur, t_path[2], _t)
1557 i += 2
1558 elif root is S and fetch_till > 1 and t_path[1] == '(':
1559 # S(var='spec') style assignment
1560 _, kwargs = t_path[2]
1561 scope.update({
1562 k: arg_val(target, v, scope) for k, v in kwargs.items()})
1563 return target
1565 else:
1566 raise ValueError('TType instance with invalid root') # pragma: no cover
1567 pae = None
1568 while i < fetch_till:
1569 op, arg = t_path[i], t_path[i + 1]
1570 arg = arg_val(target, arg, scope)
1571 if op == '.':
1572 try:
1573 cur = getattr(cur, arg)
1574 except AttributeError as e:
1575 pae = PathAccessError(e, Path(_t), i // 2)
1576 elif op == '[':
1577 try:
1578 cur = cur[arg]
1579 except (KeyError, IndexError, TypeError) as e:
1580 pae = PathAccessError(e, Path(_t), i // 2)
1581 elif op == 'P':
1582 # Path type stuff (fuzzy match)
1583 get = scope[TargetRegistry].get_handler('get', cur, path=t_path[2:i+2:2])
1584 try:
1585 cur = get(cur, arg)
1586 except Exception as e:
1587 pae = PathAccessError(e, Path(_t), i // 2)
1588 elif op in 'xX':
1589 nxt = []
1590 get_handler = scope[TargetRegistry].get_handler
1591 if op == 'x': # increases arity of cur each time through
1592 # TODO: so many try/except -- could scope[TargetRegistry] stuff be cached on type?
1593 _extend_children(nxt, cur, get_handler)
1594 elif op == 'X':
1595 sofar = set()
1596 _extend_children(nxt, cur, get_handler)
1597 for item in nxt:
1598 if id(item) not in sofar:
1599 sofar.add(id(item))
1600 _extend_children(nxt, item, get_handler)
1601 nxt.insert(0, cur)
1602 # handle the rest of the t_path in recursive calls
1603 cur = []
1604 todo = TType()
1605 todo.__ops__ = (root,) + t_path[i+2:]
1606 for child in nxt:
1607 try:
1608 cur.append(_t_eval(child, todo, scope))
1609 except PathAccessError:
1610 pass
1611 break # we handled the rest in recursive call, break loop
1612 elif op == '(':
1613 args, kwargs = arg
1614 scope[Path] += t_path[2:i+2:2]
1615 cur = scope[glom](
1616 target, Call(cur, args, kwargs), scope)
1617 # call with target rather than cur,
1618 # because it is probably more intuitive
1619 # if args to the call "reset" their path
1620 # e.g. "T.a" should mean the same thing
1621 # in both of these specs: T.a and T.b(T.a)
1622 else: # arithmetic operators
1623 try:
1624 if op == '+':
1625 cur = cur + arg
1626 elif op == '-':
1627 cur = cur - arg
1628 elif op == '*':
1629 cur = cur * arg
1630 #elif op == '#':
1631 # cur = cur // arg # TODO: python 2 friendly approach?
1632 elif op == '/':
1633 cur = cur / arg
1634 elif op == '%':
1635 cur = cur % arg
1636 elif op == ':':
1637 cur = cur ** arg
1638 elif op == '&':
1639 cur = cur & arg
1640 elif op == '|':
1641 cur = cur | arg
1642 elif op == '^':
1643 cur = cur ^ arg
1644 elif op == '~':
1645 cur = ~cur
1646 elif op == '_':
1647 cur = -cur
1648 except (TypeError, ZeroDivisionError) as e:
1649 pae = PathAccessError(e, Path(_t), i // 2)
1650 if pae:
1651 raise pae
1652 i += 2
1653 if root is A:
1654 op, arg = t_path[-2:]
1655 if cur is scope:
1656 op = '[' # all assignment on scope is setitem
1657 _assign_op(dest=cur, op=op, arg=arg, val=target, path=_t, scope=scope)
1658 return target # A should not change the target
1659 return cur
1662def _assign_op(dest, op, arg, val, path, scope):
1663 """helper method for doing the assignment on a T operation"""
1664 if op == '[':
1665 dest[arg] = val
1666 elif op == '.':
1667 setattr(dest, arg, val)
1668 elif op == 'P':
1669 _assign = scope[TargetRegistry].get_handler('assign', dest)
1670 try:
1671 _assign(dest, arg, val)
1672 except Exception as e:
1673 raise PathAssignError(e, path, arg)
1674 else: # pragma: no cover
1675 raise ValueError('unsupported T operation for assignment')
1678def _extend_children(children, item, get_handler):
1679 try: # dict or obj-like
1680 keys = get_handler('keys', item)
1681 get = get_handler('get', item)
1682 except UnregisteredTarget:
1683 try:
1684 iterate = get_handler('iterate', item)
1685 except UnregisteredTarget:
1686 pass
1687 else:
1688 try: # list-like
1689 children.extend(iterate(item))
1690 except Exception:
1691 pass
1692 else:
1693 try:
1694 for key in keys(item):
1695 try:
1696 children.append(get(item, key))
1697 except Exception:
1698 pass
1699 except Exception:
1700 pass
1703T = TType() # target aka Mr. T aka "this"
1704S = TType() # like T, but means grab stuff from Scope, not Target
1705A = TType() # like S, but shorthand to assign target to scope
1707T.__ops__ = (T,)
1708S.__ops__ = (S,)
1709A.__ops__ = (A,)
1711_T_STAR = T.__star__() # helper constant for Path.from_text
1712_T_STARSTAR = T.__starstar__() # helper constant for Path.from_text
1714UP = make_sentinel('UP')
1715ROOT = make_sentinel('ROOT')
1718def _format_slice(x):
1719 if type(x) is not slice:
1720 return bbrepr(x)
1721 fmt = lambda v: "" if v is None else bbrepr(v)
1722 if x.step is None:
1723 return fmt(x.start) + ":" + fmt(x.stop)
1724 return fmt(x.start) + ":" + fmt(x.stop) + ":" + fmt(x.step)
1727def _format_t(path, root=T):
1728 prepr = [{T: 'T', S: 'S', A: 'A'}[root]]
1729 i = 0
1730 while i < len(path):
1731 op, arg = path[i], path[i + 1]
1732 if op == '.':
1733 prepr.append('.' + arg)
1734 elif op == '[':
1735 if type(arg) is tuple:
1736 index = ", ".join([_format_slice(x) for x in arg])
1737 else:
1738 index = _format_slice(arg)
1739 prepr.append(f"[{index}]")
1740 elif op == '(':
1741 args, kwargs = arg
1742 prepr.append(format_invocation(args=args, kwargs=kwargs, repr=bbrepr))
1743 elif op == 'P':
1744 return _format_path(path)
1745 elif op == 'x':
1746 prepr.append(".__star__()")
1747 elif op == 'X':
1748 prepr.append(".__starstar__()")
1749 elif op in ('_', '~'): # unary arithmetic operators
1750 if any([o in path[:i] for o in '+-/%:&|^~_']):
1751 prepr = ['('] + prepr + [')']
1752 prepr = ['-' if op == '_' else op] + prepr
1753 else: # binary arithmetic operators
1754 formatted_arg = bbrepr(arg)
1755 if type(arg) is TType:
1756 arg_path = arg.__ops__
1757 if any([o in arg_path for o in '+-/%:&|^~_']):
1758 formatted_arg = '(' + formatted_arg + ')'
1759 prepr.append(' ' + ('**' if op == ':' else op) + ' ')
1760 prepr.append(formatted_arg)
1761 i += 2
1762 return "".join(prepr)
1765class Val:
1766 """Val objects are specs which evaluate to the wrapped *value*.
1768 >>> target = {'a': {'b': 'c'}}
1769 >>> spec = {'a': 'a.b', 'readability': Val('counts')}
1770 >>> pprint(glom(target, spec))
1771 {'a': 'c', 'readability': 'counts'}
1773 Instead of accessing ``'counts'`` as a key like it did with
1774 ``'a.b'``, :func:`~glom.glom` just unwrapped the Val and
1775 included the value.
1777 :class:`~glom.Val` takes one argument, the value to be returned.
1779 .. note::
1781 :class:`Val` was named ``Literal`` in versions of glom before
1782 20.7.0. An alias has been preserved for backwards
1783 compatibility, but reprs have changed.
1785 """
1786 def __init__(self, value):
1787 self.value = value
1789 def glomit(self, target, scope):
1790 return self.value
1792 def __repr__(self):
1793 cn = self.__class__.__name__
1794 return f'{cn}({bbrepr(self.value)})'
1797Literal = Val # backwards compat for pre-20.7.0
1800class ScopeVars:
1801 """This is the runtime partner of :class:`Vars` -- this is what
1802 actually lives in the scope and stores runtime values.
1804 While not part of the importable API of glom, it's half expected
1805 that some folks may write sepcs to populate and export scopes, at
1806 which point this type makes it easy to access values by attribute
1807 access or by converting to a dict.
1809 """
1810 def __init__(self, base, defaults):
1811 self.__dict__ = dict(base)
1812 self.__dict__.update(defaults)
1814 def __iter__(self):
1815 return iter(self.__dict__.items())
1817 def __repr__(self):
1818 return f"{self.__class__.__name__}({bbrepr(self.__dict__)})"
1821class Vars:
1822 """
1823 :class:`Vars` is a helper that can be used with **S** in order to
1824 store shared mutable state.
1826 Takes the same arguments as :class:`dict()`.
1828 Arguments here should be thought of the same way as default arguments
1829 to a function. Each time the spec is evaluated, the same arguments
1830 will be referenced; so, think carefully about mutable data structures.
1831 """
1832 def __init__(self, base=(), **kw):
1833 dict(base) # ensure it is a dict-compatible first arg
1834 self.base = base
1835 self.defaults = kw
1837 def glomit(self, target, spec):
1838 return ScopeVars(self.base, self.defaults)
1840 def __repr__(self):
1841 ret = format_invocation(self.__class__.__name__,
1842 args=(self.base,) if self.base else (),
1843 kwargs=self.defaults,
1844 repr=bbrepr)
1845 return ret
1848class Let:
1849 """
1850 Deprecated, kept for backwards compat. Use S(x='y') instead.
1852 >>> target = {'data': {'val': 9}}
1853 >>> spec = (Let(value=T['data']['val']), {'val': S['value']})
1854 >>> glom(target, spec)
1855 {'val': 9}
1857 """
1858 def __init__(self, **kw):
1859 if not kw:
1860 raise TypeError('expected at least one keyword argument')
1861 self._binding = kw
1863 def glomit(self, target, scope):
1864 scope.update({
1865 k: scope[glom](target, v, scope) for k, v in self._binding.items()})
1866 return target
1868 def __repr__(self):
1869 cn = self.__class__.__name__
1870 return format_invocation(cn, kwargs=self._binding, repr=bbrepr)
1873class Auto:
1874 """
1875 Switch to Auto mode (the default)
1877 TODO: this seems like it should be a sub-class of class Spec() --
1878 if Spec() could help define the interface for new "modes" or dialects
1879 that would also help make match mode feel less duct-taped on
1880 """
1881 def __init__(self, spec=None):
1882 self.spec = spec
1884 def glomit(self, target, scope):
1885 scope[MODE] = AUTO
1886 return scope[glom](target, self.spec, scope)
1888 def __repr__(self):
1889 cn = self.__class__.__name__
1890 rpr = '' if self.spec is None else bbrepr(self.spec)
1891 return f'{cn}({rpr})'
1894class _AbstractIterable(_AbstractIterableBase):
1895 __metaclass__ = ABCMeta
1896 @classmethod
1897 def __subclasshook__(cls, C):
1898 if C in (str, bytes):
1899 return False
1900 return callable(getattr(C, "__iter__", None))
1903class _ObjStyleKeysMeta(type):
1904 def __instancecheck__(cls, C):
1905 return hasattr(C, "__dict__") and hasattr(C.__dict__, "keys")
1908class _ObjStyleKeys(_ObjStyleKeysMeta('_AbstractKeys', (object,), {})):
1909 __metaclass__ = _ObjStyleKeysMeta
1911 @staticmethod
1912 def get_keys(obj):
1913 ret = obj.__dict__.keys()
1914 return ret
1917def _get_sequence_item(target, index):
1918 return target[int(index)]
1921# handlers are 3-arg callables, with args (spec, target, scope)
1922# spec is the first argument for convenience in the case
1923# that the handler is a method of the spec type
1924def _handle_dict(target, spec, scope):
1925 ret = type(spec)() # TODO: works for dict + ordereddict, but sufficient for all?
1926 for field, subspec in spec.items():
1927 val = scope[glom](target, subspec, scope)
1928 if val is SKIP:
1929 continue
1930 if type(field) in (Spec, TType):
1931 field = scope[glom](target, field, scope)
1932 ret[field] = val
1933 return ret
1936def _handle_list(target, spec, scope):
1937 subspec = spec[0]
1938 iterate = scope[TargetRegistry].get_handler('iterate', target, path=scope[Path])
1939 try:
1940 iterator = iterate(target)
1941 except Exception as e:
1942 raise TypeError('failed to iterate on instance of type %r at %r (got %r)'
1943 % (target.__class__.__name__, Path(*scope[Path]), e))
1944 ret = []
1945 base_path = scope[Path]
1946 for i, t in enumerate(iterator):
1947 scope[Path] = base_path + [i]
1948 val = scope[glom](t, subspec, scope)
1949 if val is SKIP:
1950 continue
1951 if val is STOP:
1952 break
1953 ret.append(val)
1954 return ret
1957def _handle_tuple(target, spec, scope):
1958 res = target
1959 for subspec in spec:
1960 scope = chain_child(scope)
1961 nxt = scope[glom](res, subspec, scope)
1962 if nxt is SKIP:
1963 continue
1964 if nxt is STOP:
1965 break
1966 res = nxt
1967 if not isinstance(subspec, list):
1968 scope[Path] += [getattr(subspec, '__name__', subspec)]
1969 return res
1972class Pipe:
1973 """Evaluate specs one after the other, passing the result of
1974 the previous evaluation in as the target of the next spec:
1976 >>> glom({'a': {'b': -5}}, Pipe('a', 'b', abs))
1977 5
1979 Same behavior as ``Auto(tuple(steps))``, but useful for explicit
1980 usage in other modes.
1981 """
1982 def __init__(self, *steps):
1983 self.steps = steps
1985 def glomit(self, target, scope):
1986 return _handle_tuple(target, self.steps, scope)
1988 def __repr__(self):
1989 return self.__class__.__name__ + bbrepr(self.steps)
1992class TargetRegistry:
1993 '''
1994 responsible for registration of target types for iteration
1995 and attribute walking
1996 '''
1997 def __init__(self, register_default_types=True):
1998 self._op_type_map = {}
1999 self._op_type_tree = {} # see _register_fuzzy_type for details
2000 self._type_cache = {}
2002 self._op_auto_map = OrderedDict() # op name to function that returns handler function
2004 self._register_builtin_ops()
2006 if register_default_types:
2007 self._register_default_types()
2008 return
2010 def get_handler(self, op, obj, path=None, raise_exc=True):
2011 """for an operation and object **instance**, obj, return the
2012 closest-matching handler function, raising UnregisteredTarget
2013 if no handler can be found for *obj* (or False if
2014 raise_exc=False)
2016 """
2017 ret = False
2018 obj_type = type(obj)
2019 cache_key = (obj_type, op)
2020 if cache_key not in self._type_cache:
2021 type_map = self.get_type_map(op)
2022 if type_map:
2023 try:
2024 ret = type_map[obj_type]
2025 except KeyError:
2026 type_tree = self._op_type_tree.get(op, {})
2027 closest = self._get_closest_type(obj, type_tree=type_tree)
2028 if closest is None:
2029 ret = False
2030 else:
2031 ret = type_map[closest]
2033 if ret is False and raise_exc:
2034 raise UnregisteredTarget(op, obj_type, type_map=type_map, path=path)
2036 self._type_cache[cache_key] = ret
2037 return self._type_cache[cache_key]
2039 def get_type_map(self, op):
2040 try:
2041 return self._op_type_map[op]
2042 except KeyError:
2043 return OrderedDict()
2045 def _get_closest_type(self, obj, type_tree):
2046 default = None
2047 for cur_type, sub_tree in type_tree.items():
2048 if isinstance(obj, cur_type):
2049 sub_type = self._get_closest_type(obj, type_tree=sub_tree)
2050 ret = cur_type if sub_type is None else sub_type
2051 return ret
2052 return default
2054 def _register_default_types(self):
2055 self.register(object)
2056 self.register(dict, get=operator.getitem)
2057 self.register(dict, keys=dict.keys)
2058 self.register(list, get=_get_sequence_item)
2059 self.register(tuple, get=_get_sequence_item)
2060 self.register(OrderedDict, get=operator.getitem)
2061 self.register(OrderedDict, keys=OrderedDict.keys)
2062 self.register(_AbstractIterable, iterate=iter)
2063 self.register(_ObjStyleKeys, keys=_ObjStyleKeys.get_keys)
2065 def _register_fuzzy_type(self, op, new_type, _type_tree=None):
2066 """Build a "type tree", an OrderedDict mapping registered types to
2067 their subtypes
2069 The type tree's invariant is that a key in the mapping is a
2070 valid parent type of all its children.
2072 Order is preserved such that non-overlapping parts of the
2073 subtree take precedence by which was most recently added.
2074 """
2075 if _type_tree is None:
2076 try:
2077 _type_tree = self._op_type_tree[op]
2078 except KeyError:
2079 _type_tree = self._op_type_tree[op] = OrderedDict()
2081 registered = False
2082 for cur_type, sub_tree in list(_type_tree.items()):
2083 if issubclass(cur_type, new_type):
2084 sub_tree = _type_tree.pop(cur_type) # mutation for recursion brevity
2085 try:
2086 _type_tree[new_type][cur_type] = sub_tree
2087 except KeyError:
2088 _type_tree[new_type] = OrderedDict({cur_type: sub_tree})
2089 registered = True
2090 elif issubclass(new_type, cur_type):
2091 _type_tree[cur_type] = self._register_fuzzy_type(op, new_type, _type_tree=sub_tree)
2092 registered = True
2093 if not registered:
2094 _type_tree[new_type] = OrderedDict()
2095 return _type_tree
2097 def register(self, target_type, **kwargs):
2098 if not isinstance(target_type, type):
2099 raise TypeError(f'register expected a type, not an instance: {target_type!r}')
2100 exact = kwargs.pop('exact', None)
2101 new_op_map = dict(kwargs)
2103 for op_name in sorted(set(self._op_auto_map.keys()) | set(new_op_map.keys())):
2104 cur_type_map = self._op_type_map.setdefault(op_name, OrderedDict())
2106 if op_name in new_op_map:
2107 handler = new_op_map[op_name]
2108 elif target_type in cur_type_map:
2109 handler = cur_type_map[target_type]
2110 else:
2111 try:
2112 handler = self._op_auto_map[op_name](target_type)
2113 except Exception as e:
2114 raise TypeError('error while determining support for operation'
2115 ' "%s" on target type: %s (got %r)'
2116 % (op_name, target_type.__name__, e))
2117 if handler is not False and not callable(handler):
2118 raise TypeError('expected handler for op "%s" to be'
2119 ' callable or False, not: %r' % (op_name, handler))
2120 new_op_map[op_name] = handler
2122 for op_name, handler in new_op_map.items():
2123 self._op_type_map[op_name][target_type] = handler
2125 if not exact:
2126 for op_name in new_op_map:
2127 self._register_fuzzy_type(op_name, target_type)
2129 self._type_cache = {} # reset type cache
2131 return
2133 def register_op(self, op_name, auto_func=None, exact=False):
2134 """add operations beyond the builtins ('get' and 'iterate' at the time
2135 of writing).
2137 auto_func is a function that when passed a type, returns a
2138 handler associated with op_name if it's supported, or False if
2139 it's not.
2141 See glom.core.register_op() for the global version used by
2142 extensions.
2143 """
2144 if not isinstance(op_name, basestring):
2145 raise TypeError(f'expected op_name to be a text name, not: {op_name!r}')
2146 if auto_func is None:
2147 auto_func = lambda t: False
2148 elif not callable(auto_func):
2149 raise TypeError(f'expected auto_func to be callable, not: {auto_func!r}')
2151 # determine support for any previously known types
2152 known_types = set(sum([list(m.keys()) for m
2153 in self._op_type_map.values()], []))
2154 type_map = self._op_type_map.get(op_name, OrderedDict())
2155 type_tree = self._op_type_tree.get(op_name, OrderedDict())
2156 for t in sorted(known_types, key=lambda t: t.__name__):
2157 if t in type_map:
2158 continue
2159 try:
2160 handler = auto_func(t)
2161 except Exception as e:
2162 raise TypeError('error while determining support for operation'
2163 ' "%s" on target type: %s (got %r)'
2164 % (op_name, t.__name__, e))
2165 if handler is not False and not callable(handler):
2166 raise TypeError('expected handler for op "%s" to be'
2167 ' callable or False, not: %r' % (op_name, handler))
2168 type_map[t] = handler
2170 if not exact:
2171 for t in known_types:
2172 self._register_fuzzy_type(op_name, t, _type_tree=type_tree)
2174 self._op_type_map[op_name] = type_map
2175 self._op_type_tree[op_name] = type_tree
2176 self._op_auto_map[op_name] = auto_func
2178 def _register_builtin_ops(self):
2179 def _get_iterable_handler(type_obj):
2180 return iter if callable(getattr(type_obj, '__iter__', None)) else False
2182 self.register_op('iterate', _get_iterable_handler)
2183 self.register_op('get', lambda _: getattr)
2186_DEFAULT_SCOPE = ChainMap({})
2189def glom(target, spec, **kwargs):
2190 """Access or construct a value from a given *target* based on the
2191 specification declared by *spec*.
2193 Accessing nested data, aka deep-get:
2195 >>> target = {'a': {'b': 'c'}}
2196 >>> glom(target, 'a.b')
2197 'c'
2199 Here the *spec* was just a string denoting a path,
2200 ``'a.b'``. As simple as it should be. You can also use
2201 :mod:`glob`-like wildcard selectors:
2203 >>> target = {'a': [{'k': 'v1'}, {'k': 'v2'}]}
2204 >>> glom(target, 'a.*.k')
2205 ['v1', 'v2']
2207 In addition to ``*``, you can also use ``**`` for recursive access:
2209 >>> target = {'a': [{'k': 'v3'}, {'k': 'v4'}], 'k': 'v0'}
2210 >>> glom(target, '**.k')
2211 ['v0', 'v3', 'v4']
2213 The next example shows how to use nested data to
2214 access many fields at once, and make a new nested structure.
2216 Constructing, or restructuring more-complicated nested data:
2218 >>> target = {'a': {'b': 'c', 'd': 'e'}, 'f': 'g', 'h': [0, 1, 2]}
2219 >>> spec = {'a': 'a.b', 'd': 'a.d', 'h': ('h', [lambda x: x * 2])}
2220 >>> output = glom(target, spec)
2221 >>> pprint(output)
2222 {'a': 'c', 'd': 'e', 'h': [0, 2, 4]}
2224 ``glom`` also takes a keyword-argument, *default*. When set,
2225 if a ``glom`` operation fails with a :exc:`GlomError`, the
2226 *default* will be returned, very much like
2227 :meth:`dict.get()`:
2229 >>> glom(target, 'a.xx', default='nada')
2230 'nada'
2232 The *skip_exc* keyword argument controls which errors should
2233 be ignored.
2235 >>> glom({}, lambda x: 100.0 / len(x), default=0.0, skip_exc=ZeroDivisionError)
2236 0.0
2238 Args:
2239 target (object): the object on which the glom will operate.
2240 spec (object): Specification of the output object in the form
2241 of a dict, list, tuple, string, other glom construct, or
2242 any composition of these.
2243 default (object): An optional default to return in the case
2244 an exception, specified by *skip_exc*, is raised.
2245 skip_exc (Exception): An optional exception or tuple of
2246 exceptions to ignore and return *default* (None if
2247 omitted). If *skip_exc* and *default* are both not set,
2248 glom raises errors through.
2249 scope (dict): Additional data that can be accessed
2250 via S inside the glom-spec. Read more: :ref:`scope`.
2252 It's a small API with big functionality, and glom's power is
2253 only surpassed by its intuitiveness. Give it a whirl!
2255 """
2256 # TODO: check spec up front
2257 default = kwargs.pop('default', None if 'skip_exc' in kwargs else _MISSING)
2258 skip_exc = kwargs.pop('skip_exc', () if default is _MISSING else GlomError)
2259 glom_debug = kwargs.pop('glom_debug', GLOM_DEBUG)
2260 scope = _DEFAULT_SCOPE.new_child({
2261 Path: kwargs.pop('path', []),
2262 Inspect: kwargs.pop('inspector', None),
2263 MODE: AUTO,
2264 MIN_MODE: None,
2265 CHILD_ERRORS: [],
2266 'globals': ScopeVars({}, {}),
2267 })
2268 scope[UP] = scope
2269 scope[ROOT] = scope
2270 scope[T] = target
2271 scope.update(kwargs.pop('scope', {}))
2272 err = None
2273 if kwargs:
2274 raise TypeError('unexpected keyword args: %r' % sorted(kwargs.keys()))
2275 try:
2276 try:
2277 ret = _glom(target, spec, scope)
2278 except skip_exc:
2279 if default is _MISSING:
2280 raise
2281 ret = default # should this also be arg_val'd?
2282 except Exception as e:
2283 if glom_debug:
2284 raise
2285 if isinstance(e, GlomError):
2286 # need to change id or else py3 seems to not let us truncate the
2287 # stack trace with the explicit "raise err" below
2288 err = copy.copy(e)
2289 err._set_wrapped(e)
2290 else:
2291 err = GlomError.wrap(e)
2292 if isinstance(err, GlomError):
2293 err._finalize(scope[LAST_CHILD_SCOPE])
2294 else: # wrapping failed, fall back to default behavior
2295 raise
2297 if err:
2298 raise err
2299 return ret
2302def chain_child(scope):
2303 """
2304 used for specs like Auto(tuple), Switch(), etc
2305 that want to chain their child scopes together
2307 returns a new scope that can be passed to
2308 the next recursive glom call, e.g.
2310 scope[glom](target, spec, chain_child(scope))
2311 """
2312 if LAST_CHILD_SCOPE not in scope.maps[0]:
2313 return scope # no children yet, nothing to do
2314 # NOTE: an option here is to drill down on LAST_CHILD_SCOPE;
2315 # this would have some interesting consequences for scoping
2316 # of tuples
2317 nxt_in_chain = scope[LAST_CHILD_SCOPE]
2318 nxt_in_chain.maps[0][NO_PYFRAME] = True
2319 # previous failed branches are forgiven as the
2320 # scope is re-wired into a new stack
2321 del nxt_in_chain.maps[0][CHILD_ERRORS][:]
2322 return nxt_in_chain
2325unbound_methods = {type(str.__len__)} #, type(Ref.glomit)])
2328def _has_callable_glomit(obj):
2329 glomit = getattr(obj, 'glomit', None)
2330 return callable(glomit) and not isinstance(obj, type)
2333def _glom(target, spec, scope):
2334 parent = scope
2335 pmap = parent.maps[0]
2336 scope = scope.new_child({
2337 T: target,
2338 Spec: spec,
2339 UP: parent,
2340 CHILD_ERRORS: [],
2341 MODE: pmap[MODE],
2342 MIN_MODE: pmap[MIN_MODE],
2343 })
2344 pmap[LAST_CHILD_SCOPE] = scope
2346 try:
2347 if type(spec) is TType: # must go first, due to callability
2348 scope[MIN_MODE] = None # None is tombstone
2349 return _t_eval(target, spec, scope)
2350 elif _has_callable_glomit(spec):
2351 scope[MIN_MODE] = None
2352 return spec.glomit(target, scope)
2354 return (scope.maps[0][MIN_MODE] or scope.maps[0][MODE])(target, spec, scope)
2355 except Exception as e:
2356 scope.maps[1][CHILD_ERRORS].append(scope)
2357 scope.maps[0][CUR_ERROR] = e
2358 if NO_PYFRAME in scope.maps[1]:
2359 cur_scope = scope[UP]
2360 while NO_PYFRAME in cur_scope.maps[0]:
2361 cur_scope.maps[1][CHILD_ERRORS].append(cur_scope)
2362 cur_scope.maps[0][CUR_ERROR] = e
2363 cur_scope = cur_scope[UP]
2364 raise
2367def AUTO(target, spec, scope):
2368 if type(spec) is str: # shortcut to make deep-get use case faster
2369 return _t_eval(target, Path.from_text(spec).path_t, scope)
2370 if isinstance(spec, dict):
2371 return _handle_dict(target, spec, scope)
2372 elif isinstance(spec, list):
2373 return _handle_list(target, spec, scope)
2374 elif isinstance(spec, tuple):
2375 return _handle_tuple(target, spec, scope)
2376 elif isinstance(spec, basestring):
2377 return Path.from_text(spec).glomit(target, scope)
2378 elif callable(spec):
2379 return spec(target)
2381 raise TypeError('expected spec to be dict, list, tuple, callable, string,'
2382 ' or other Spec-like type, not: %r' % (spec,))
2385_DEFAULT_SCOPE.update({
2386 glom: _glom,
2387 TargetRegistry: TargetRegistry(register_default_types=True),
2388})
2391def register(target_type, **kwargs):
2392 """Register *target_type* so :meth:`~Glommer.glom()` will
2393 know how to handle instances of that type as targets.
2395 Here's an example of adding basic iterabile support for Django's ORM:
2397 .. code-block:: python
2399 import glom
2400 import django.db.models
2402 glom.register(django.db.models.Manager, iterate=lambda m: m.all())
2403 glom.register(django.db.models.QuerySet, iterate=lambda qs: qs.all())
2407 Args:
2408 target_type (type): A type expected to appear in a glom()
2409 call target
2410 get (callable): A function which takes a target object and
2411 a name, acting as a default accessor. Defaults to
2412 :func:`getattr`.
2413 iterate (callable): A function which takes a target object
2414 and returns an iterator. Defaults to :func:`iter` if
2415 *target_type* appears to be iterable.
2416 exact (bool): Whether or not to match instances of subtypes
2417 of *target_type*.
2419 .. note::
2421 The module-level :func:`register()` function affects the
2422 module-level :func:`glom()` function's behavior. If this
2423 global effect is undesirable for your application, or
2424 you're implementing a library, consider instantiating a
2425 :class:`Glommer` instance, and using the
2426 :meth:`~Glommer.register()` and :meth:`Glommer.glom()`
2427 methods instead.
2429 """
2430 _DEFAULT_SCOPE[TargetRegistry].register(target_type, **kwargs)
2431 return
2434def register_op(op_name, **kwargs):
2435 """For extension authors needing to add operations beyond the builtin
2436 'get', 'iterate', 'keys', 'assign', and 'delete' to the default scope.
2437 See TargetRegistry for more details.
2438 """
2439 _DEFAULT_SCOPE[TargetRegistry].register_op(op_name, **kwargs)
2440 return
2443class Glommer:
2444 """The :class:`Glommer` type mostly serves to encapsulate type
2445 registration context so that advanced uses of glom don't need to
2446 worry about stepping on each other.
2448 Glommer objects are lightweight and, once instantiated, provide
2449 a :func:`glom()` method:
2451 >>> glommer = Glommer()
2452 >>> glommer.glom({}, 'a.b.c', default='d')
2453 'd'
2454 >>> Glommer().glom({'vals': list(range(3))}, ('vals', len))
2455 3
2457 Instances also provide :meth:`~Glommer.register()` method for
2458 localized control over type handling.
2460 Args:
2461 register_default_types (bool): Whether or not to enable the
2462 handling behaviors of the default :func:`glom()`. These
2463 default actions include dict access, list and iterable
2464 iteration, and generic object attribute access. Defaults to
2465 True.
2467 """
2468 def __init__(self, **kwargs):
2469 register_default_types = kwargs.pop('register_default_types', True)
2470 scope = kwargs.pop('scope', _DEFAULT_SCOPE)
2472 # this "freezes" the scope in at the time of construction
2473 self.scope = ChainMap(dict(scope))
2474 self.scope[TargetRegistry] = TargetRegistry(register_default_types=register_default_types)
2476 def register(self, target_type, **kwargs):
2477 """Register *target_type* so :meth:`~Glommer.glom()` will
2478 know how to handle instances of that type as targets.
2480 Args:
2481 target_type (type): A type expected to appear in a glom()
2482 call target
2483 get (callable): A function which takes a target object and
2484 a name, acting as a default accessor. Defaults to
2485 :func:`getattr`.
2486 iterate (callable): A function which takes a target object
2487 and returns an iterator. Defaults to :func:`iter` if
2488 *target_type* appears to be iterable.
2489 exact (bool): Whether or not to match instances of subtypes
2490 of *target_type*.
2492 .. note::
2494 The module-level :func:`register()` function affects the
2495 module-level :func:`glom()` function's behavior. If this
2496 global effect is undesirable for your application, or
2497 you're implementing a library, consider instantiating a
2498 :class:`Glommer` instance, and using the
2499 :meth:`~Glommer.register()` and :meth:`Glommer.glom()`
2500 methods instead.
2502 """
2503 exact = kwargs.pop('exact', False)
2504 self.scope[TargetRegistry].register(target_type, exact=exact, **kwargs)
2505 return
2507 def glom(self, target, spec, **kwargs):
2508 return glom(target, spec, scope=self.scope, **kwargs)
2511class Fill:
2512 """A specifier type which switches to glom into "fill-mode". For the
2513 spec contained within the Fill, glom will only interpret explicit
2514 specifier types (including T objects). Whereas the default mode
2515 has special interpretations for each of these builtins, fill-mode
2516 takes a lighter touch, making Fill great for "filling out" Python
2517 literals, like tuples, dicts, sets, and lists.
2519 >>> target = {'data': [0, 2, 4]}
2520 >>> spec = Fill((T['data'][2], T['data'][0]))
2521 >>> glom(target, spec)
2522 (4, 0)
2524 As you can see, glom's usual built-in tuple item chaining behavior
2525 has switched into a simple tuple constructor.
2527 (Sidenote for Lisp fans: Fill is like glom's quasi-quoting.)
2529 """
2530 def __init__(self, spec=None):
2531 self.spec = spec
2533 def glomit(self, target, scope):
2534 scope[MODE] = FILL
2535 return scope[glom](target, self.spec, scope)
2537 def fill(self, target):
2538 return glom(target, self)
2540 def __repr__(self):
2541 cn = self.__class__.__name__
2542 rpr = '' if self.spec is None else bbrepr(self.spec)
2543 return f'{cn}({rpr})'
2546def FILL(target, spec, scope):
2547 # TODO: register an operator or two for the following to allow
2548 # extension. This operator can probably be shared with the
2549 # upcoming traversal/remap feature.
2550 recurse = lambda val: scope[glom](target, val, scope)
2551 if type(spec) is dict:
2552 return {recurse(key): recurse(val) for key, val in spec.items()}
2553 if type(spec) in (list, tuple, set, frozenset):
2554 result = [recurse(val) for val in spec]
2555 if type(spec) is list:
2556 return result
2557 return type(spec)(result)
2558 if callable(spec):
2559 return spec(target)
2560 return spec
2562class _ArgValuator:
2563 def __init__(self):
2564 self.cache = {}
2566 def mode(self, target, spec, scope):
2567 """
2568 similar to FILL, but without function calling;
2569 useful for default, scope assignment, call/invoke, etc
2570 """
2571 recur = lambda val: scope[glom](target, val, scope)
2572 result = spec
2573 if type(spec) in (list, dict): # can contain themselves
2574 if id(spec) in self.cache:
2575 return self.cache[id(spec)]
2576 result = self.cache[id(spec)] = type(spec)()
2577 if type(spec) is dict:
2578 result.update({recur(key): recur(val) for key, val in spec.items()})
2579 else:
2580 result.extend([recur(val) for val in spec])
2581 if type(spec) in (tuple, set, frozenset): # cannot contain themselves
2582 result = type(spec)([recur(val) for val in spec])
2583 return result
2586def arg_val(target, arg, scope):
2587 """
2588 evaluate an argument to find its value
2589 (arg_val phonetically similar to "eval" -- evaluate as an arg)
2590 """
2591 mode = scope[MIN_MODE]
2592 scope[MIN_MODE] = _ArgValuator().mode
2593 result = scope[glom](target, arg, scope)
2594 scope[MIN_MODE] = mode
2595 return result