Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/invoke/config.py: 20%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1import copy
2import json
3import os
4import types
5from importlib.util import spec_from_loader
6from os import PathLike
7from os.path import join, splitext, expanduser
8from types import ModuleType
9from typing import Any, Dict, Iterator, Optional, Tuple, Type, Union
11from .env import Environment
12from .exceptions import UnknownFileType, UnpicklableConfigMember
13from .runners import Local
14from .terminals import WINDOWS
15from .util import debug, yaml
18try:
19 from importlib.machinery import SourceFileLoader
20except ImportError: # PyPy3
21 from importlib._bootstrap import ( # type: ignore[no-redef]
22 _SourceFileLoader as SourceFileLoader,
23 )
26def load_source(name: str, path: str) -> Dict[str, Any]:
27 if not os.path.exists(path):
28 return {}
29 loader = SourceFileLoader("mod", path)
30 mod = ModuleType("mod")
31 mod.__spec__ = spec_from_loader("mod", loader)
32 loader.exec_module(mod)
33 return vars(mod)
36class DataProxy:
37 """
38 Helper class implementing nested dict+attr access for `.Config`.
40 Specifically, is used both for `.Config` itself, and to wrap any other
41 dicts assigned as config values (recursively).
43 .. warning::
44 All methods (of this object or in subclasses) must take care to
45 initialize new attributes via ``self._set(name='value')``, or they'll
46 run into recursion errors!
48 .. versionadded:: 1.0
49 """
51 # Attributes which get proxied through to inner merged-dict config obj.
52 _proxies = (
53 tuple(
54 """
55 get
56 has_key
57 items
58 iteritems
59 iterkeys
60 itervalues
61 keys
62 values
63 """.split()
64 )
65 + tuple(
66 "__{}__".format(x)
67 for x in """
68 cmp
69 contains
70 iter
71 sizeof
72 """.split()
73 )
74 )
76 @classmethod
77 def from_data(
78 cls,
79 data: Dict[str, Any],
80 root: Optional["DataProxy"] = None,
81 keypath: Tuple[str, ...] = tuple(),
82 ) -> "DataProxy":
83 """
84 Alternate constructor for 'baby' DataProxies used as sub-dict values.
86 Allows creating standalone DataProxy objects while also letting
87 subclasses like `.Config` define their own ``__init__`` without
88 muddling the two.
90 :param dict data:
91 This particular DataProxy's personal data. Required, it's the Data
92 being Proxied.
94 :param root:
95 Optional handle on a root DataProxy/Config which needs notification
96 on data updates.
98 :param tuple keypath:
99 Optional tuple describing the path of keys leading to this
100 DataProxy's location inside the ``root`` structure. Required if
101 ``root`` was given (and vice versa.)
103 .. versionadded:: 1.0
104 """
105 obj = cls()
106 obj._set(_config=data)
107 obj._set(_root=root)
108 obj._set(_keypath=keypath)
109 return obj
111 def __getattr__(self, key: str) -> Any:
112 # NOTE: due to default Python attribute-lookup semantics, "real"
113 # attributes will always be yielded on attribute access and this method
114 # is skipped. That behavior is good for us (it's more intuitive than
115 # having a config key accidentally shadow a real attribute or method).
116 try:
117 return self._get(key)
118 except KeyError:
119 # Proxy most special vars to config for dict procotol.
120 if key in self._proxies:
121 return getattr(self._config, key)
122 # Otherwise, raise useful AttributeError to follow getattr proto.
123 err = "No attribute or config key found for {!r}".format(key)
124 attrs = [x for x in dir(self.__class__) if not x.startswith("_")]
125 err += "\n\nValid keys: {!r}".format(
126 sorted(list(self._config.keys()))
127 )
128 err += "\n\nValid real attributes: {!r}".format(attrs)
129 raise AttributeError(err)
131 def __setattr__(self, key: str, value: Any) -> None:
132 # Turn attribute-sets into config updates anytime we don't have a real
133 # attribute with the given name/key.
134 has_real_attr = key in dir(self)
135 if not has_real_attr:
136 # Make sure to trigger our own __setitem__ instead of going direct
137 # to our internal dict/cache
138 self[key] = value
139 else:
140 super().__setattr__(key, value)
142 def __iter__(self) -> Iterator[Dict[str, Any]]:
143 # For some reason Python is ignoring our __hasattr__ when determining
144 # whether we support __iter__. BOO
145 return iter(self._config)
147 def __eq__(self, other: object) -> bool:
148 # NOTE: Can't proxy __eq__ because the RHS will always be an obj of the
149 # current class, not the proxied-to class, and that causes
150 # NotImplemented.
151 # Try comparing to other objects like ourselves, falling back to a not
152 # very comparable value (None) so comparison fails.
153 other_val = getattr(other, "_config", None)
154 # But we can compare to vanilla dicts just fine, since our _config is
155 # itself just a dict.
156 if isinstance(other, dict):
157 other_val = other
158 return bool(self._config == other_val)
160 def __len__(self) -> int:
161 return len(self._config)
163 def __setitem__(self, key: str, value: str) -> None:
164 self._config[key] = value
165 self._track_modification_of(key, value)
167 def __getitem__(self, key: str) -> Any:
168 return self._get(key)
170 def _get(self, key: str) -> Any:
171 # Short-circuit if pickling/copying mechanisms are asking if we've got
172 # __setstate__ etc; they'll ask this w/o calling our __init__ first, so
173 # we'd be in a RecursionError-causing catch-22 otherwise.
174 if key in ("__setstate__",):
175 raise AttributeError(key)
176 # At this point we should be able to assume a self._config...
177 value = self._config[key]
178 if isinstance(value, dict):
179 # New object's keypath is simply the key, prepended with our own
180 # keypath if we've got one.
181 keypath = (key,)
182 if hasattr(self, "_keypath"):
183 keypath = self._keypath + keypath
184 # If we have no _root, we must be the root, so it's us. Otherwise,
185 # pass along our handle on the root.
186 root = getattr(self, "_root", self)
187 value = DataProxy.from_data(data=value, root=root, keypath=keypath)
188 return value
190 def _set(self, *args: Any, **kwargs: Any) -> None:
191 """
192 Convenience workaround of default 'attrs are config keys' behavior.
194 Uses `object.__setattr__` to work around the class' normal proxying
195 behavior, but is less verbose than using that directly.
197 Has two modes (which may be combined if you really want):
199 - ``self._set('attrname', value)``, just like ``__setattr__``
200 - ``self._set(attname=value)`` (i.e. kwargs), even less typing.
201 """
202 if args:
203 object.__setattr__(self, *args)
204 for key, value in kwargs.items():
205 object.__setattr__(self, key, value)
207 def __repr__(self) -> str:
208 return "<{}: {}>".format(self.__class__.__name__, self._config)
210 def __contains__(self, key: str) -> bool:
211 return key in self._config
213 @property
214 def _is_leaf(self) -> bool:
215 return hasattr(self, "_root")
217 @property
218 def _is_root(self) -> bool:
219 return hasattr(self, "_modify")
221 def _track_removal_of(self, key: str) -> None:
222 # Grab the root object responsible for tracking removals; either the
223 # referenced root (if we're a leaf) or ourselves (if we're not).
224 # (Intermediate nodes never have anything but __getitem__ called on
225 # them, otherwise they're by definition being treated as a leaf.)
226 target = None
227 if self._is_leaf:
228 target = self._root
229 elif self._is_root:
230 target = self
231 if target is not None:
232 target._remove(getattr(self, "_keypath", tuple()), key)
234 def _track_modification_of(self, key: str, value: str) -> None:
235 target = None
236 if self._is_leaf:
237 target = self._root
238 elif self._is_root:
239 target = self
240 if target is not None:
241 target._modify(getattr(self, "_keypath", tuple()), key, value)
243 def __delitem__(self, key: str) -> None:
244 del self._config[key]
245 self._track_removal_of(key)
247 def __delattr__(self, name: str) -> None:
248 # Make sure we don't screw up true attribute deletion for the
249 # situations that actually want it. (Uncommon, but not rare.)
250 if name in self:
251 del self[name]
252 else:
253 object.__delattr__(self, name)
255 def clear(self) -> None:
256 keys = list(self.keys())
257 for key in keys:
258 del self[key]
260 def pop(self, *args: Any) -> Any:
261 # Must test this up front before (possibly) mutating self._config
262 key_existed = args and args[0] in self._config
263 # We always have a _config (whether it's a real dict or a cache of
264 # merged levels) so we can fall back to it for all the corner case
265 # handling re: args (arity, handling a default, raising KeyError, etc)
266 ret = self._config.pop(*args)
267 # If it looks like no popping occurred (key wasn't there), presumably
268 # user gave default, so we can short-circuit return here - no need to
269 # track a deletion that did not happen.
270 if not key_existed:
271 return ret
272 # Here, we can assume at least the 1st posarg (key) existed.
273 self._track_removal_of(args[0])
274 # In all cases, return the popped value.
275 return ret
277 def popitem(self) -> Any:
278 ret = self._config.popitem()
279 self._track_removal_of(ret[0])
280 return ret
282 def setdefault(self, *args: Any) -> Any:
283 # Must test up front whether the key existed beforehand
284 key_existed = args and args[0] in self._config
285 # Run locally
286 ret = self._config.setdefault(*args)
287 # Key already existed -> nothing was mutated, short-circuit
288 if key_existed:
289 return ret
290 # Here, we can assume the key did not exist and thus user must have
291 # supplied a 'default' (if they did not, the real setdefault() above
292 # would have excepted.)
293 key, default = args
294 self._track_modification_of(key, default)
295 return ret
297 def update(self, *args: Any, **kwargs: Any) -> None:
298 if kwargs:
299 for key, value in kwargs.items():
300 self[key] = value
301 elif args:
302 # TODO: complain if arity>1
303 arg = args[0]
304 if isinstance(arg, dict):
305 for key in arg:
306 self[key] = arg[key]
307 else:
308 # TODO: be stricter about input in this case
309 for pair in arg:
310 self[pair[0]] = pair[1]
313class Config(DataProxy):
314 """
315 Invoke's primary configuration handling class.
317 See :doc:`/concepts/configuration` for details on the configuration system
318 this class implements, including the :ref:`configuration hierarchy
319 <config-hierarchy>`. The rest of this class' documentation assumes
320 familiarity with that document.
322 **Access**
324 Configuration values may be accessed and/or updated using dict syntax::
326 config['foo']
328 or attribute syntax::
330 config.foo
332 Nesting works the same way - dict config values are turned into objects
333 which honor both the dictionary protocol and the attribute-access method::
335 config['foo']['bar']
336 config.foo.bar
338 **A note about attribute access and methods**
340 This class implements the entire dictionary protocol: methods such as
341 ``keys``, ``values``, ``items``, ``pop`` and so forth should all function
342 as they do on regular dicts. It also implements new config-specific methods
343 such as `load_system`, `load_collection`, `merge`, `clone`, etc.
345 .. warning::
346 Accordingly, this means that if you have configuration options sharing
347 names with these methods, you **must** use dictionary syntax (e.g.
348 ``myconfig['keys']``) to access the configuration data.
350 **Lifecycle**
352 At initialization time, `.Config`:
354 - creates per-level data structures;
355 - stores any levels supplied to `__init__`, such as defaults or overrides,
356 as well as the various config file paths/filename patterns;
357 - and loads config files, if found (though typically this just means system
358 and user-level files, as project and runtime files need more info before
359 they can be found and loaded.)
361 - This step can be skipped by specifying ``lazy=True``.
363 At this point, `.Config` is fully usable - and because it pre-emptively
364 loads some config files, those config files can affect anything that
365 comes after, like CLI parsing or loading of task collections.
367 In the CLI use case, further processing is done after instantiation, using
368 the ``load_*`` methods such as `load_overrides`, `load_project`, etc:
370 - the result of argument/option parsing is applied to the overrides level;
371 - a project-level config file is loaded, as it's dependent on a loaded
372 tasks collection;
373 - a runtime config file is loaded, if its flag was supplied;
374 - then, for each task being executed:
376 - per-collection data is loaded (only possible now that we have
377 collection & task in hand);
378 - shell environment data is loaded (must be done at end of process due
379 to using the rest of the config as a guide for interpreting env var
380 names.)
382 At this point, the config object is handed to the task being executed, as
383 part of its execution `.Context`.
385 Any modifications made directly to the `.Config` itself after this point
386 end up stored in their own (topmost) config level, making it easier to
387 debug final values.
389 Finally, any *deletions* made to the `.Config` (e.g. applications of
390 dict-style mutators like ``pop``, ``clear`` etc) are also tracked in their
391 own structure, allowing the config object to honor such method calls
392 without mutating the underlying source data.
394 **Special class attributes**
396 The following class-level attributes are used for low-level configuration
397 of the config system itself, such as which file paths to load. They are
398 primarily intended for overriding by subclasses.
400 - ``prefix``: Supplies the default value for ``file_prefix`` (directly) and
401 ``env_prefix`` (uppercased). See their descriptions for details. Its
402 default value is ``"invoke"``.
403 - ``file_prefix``: The config file 'basename' default (though it is not a
404 literal basename; it can contain path parts if desired) which is appended
405 to the configured values of ``system_prefix``, ``user_prefix``, etc, to
406 arrive at the final (pre-extension) file paths.
408 Thus, by default, a system-level config file path concatenates the
409 ``system_prefix`` of ``/etc/`` with the ``file_prefix`` of ``invoke`` to
410 arrive at paths like ``/etc/invoke.json``.
412 Defaults to ``None``, meaning to use the value of ``prefix``.
414 - ``env_prefix``: A prefix used (along with a joining underscore) to
415 determine which environment variables are loaded as the env var
416 configuration level. Since its default is the value of ``prefix``
417 capitalized, this means env vars like ``INVOKE_RUN_ECHO`` are sought by
418 default.
420 Defaults to ``None``, meaning to use the value of ``prefix``.
422 .. versionadded:: 1.0
423 """
425 prefix = "invoke"
426 file_prefix = None
427 env_prefix = None
429 @staticmethod
430 def global_defaults() -> Dict[str, Any]:
431 """
432 Return the core default settings for Invoke.
434 Generally only for use by `.Config` internals. For descriptions of
435 these values, see :ref:`default-values`.
437 Subclasses may choose to override this method, calling
438 ``Config.global_defaults`` and applying `.merge_dicts` to the result,
439 to add to or modify these values.
441 .. versionadded:: 1.0
442 """
443 # On Windows, which won't have /bin/bash, check for a set COMSPEC env
444 # var (https://en.wikipedia.org/wiki/COMSPEC) or fallback to an
445 # unqualified cmd.exe otherwise.
446 if WINDOWS:
447 shell = os.environ.get("COMSPEC", "cmd.exe")
448 # Else, assume Unix, most distros of which have /bin/bash available.
449 # TODO: consider an automatic fallback to /bin/sh for systems lacking
450 # /bin/bash; however users may configure run.shell quite easily, so...
451 else:
452 shell = "/bin/bash"
454 return {
455 # TODO: we document 'debug' but it's not truly implemented outside
456 # of env var and CLI flag. If we honor it, we have to go around and
457 # figure out at what points we might want to call
458 # `util.enable_logging`:
459 # - just using it as a fallback default for arg parsing isn't much
460 # use, as at that point the config holds nothing but defaults & CLI
461 # flag values
462 # - doing it at file load time might be somewhat useful, though
463 # where this happens may be subject to change soon
464 # - doing it at env var load time seems a bit silly given the
465 # existing support for at-startup testing for INVOKE_DEBUG
466 # 'debug': False,
467 # TODO: I feel like we want these to be more consistent re: default
468 # values stored here vs 'stored' as logic where they are
469 # referenced, there are probably some bits that are all "if None ->
470 # default" that could go here. Alternately, make _more_ of these
471 # default to None?
472 "run": {
473 "asynchronous": False,
474 "disown": False,
475 "dry": False,
476 "echo": False,
477 "echo_stdin": None,
478 "encoding": None,
479 "env": {},
480 "err_stream": None,
481 "fallback": True,
482 "hide": None,
483 "in_stream": None,
484 "out_stream": None,
485 "echo_format": "\033[1;37m{command}\033[0m",
486 "pty": False,
487 "replace_env": False,
488 "shell": shell,
489 "warn": False,
490 "watchers": [],
491 },
492 # This doesn't live inside the 'run' tree; otherwise it'd make it
493 # somewhat harder to extend/override in Fabric 2 which has a split
494 # local/remote runner situation.
495 "runners": {"local": Local},
496 "sudo": {
497 "password": None,
498 "prompt": "[sudo] password: ",
499 "user": None,
500 },
501 "tasks": {
502 "auto_dash_names": True,
503 "collection_name": "tasks",
504 "dedupe": True,
505 "executor_class": None,
506 "ignore_unknown_help": False,
507 "search_root": None,
508 },
509 "timeouts": {"command": None},
510 }
512 def __init__(
513 self,
514 overrides: Optional[Dict[str, Any]] = None,
515 defaults: Optional[Dict[str, Any]] = None,
516 system_prefix: Optional[str] = None,
517 user_prefix: Optional[str] = None,
518 project_location: Optional[PathLike] = None,
519 runtime_path: Optional[PathLike] = None,
520 lazy: bool = False,
521 ):
522 """
523 Creates a new config object.
525 :param dict defaults:
526 A dict containing default (lowest level) config data. Default:
527 `global_defaults`.
529 :param dict overrides:
530 A dict containing override-level config data. Default: ``{}``.
532 :param str system_prefix:
533 Base path for the global config file location; combined with the
534 prefix and file suffixes to arrive at final file path candidates.
536 Default: ``/etc/`` (thus e.g. ``/etc/invoke.yaml`` or
537 ``/etc/invoke.json``).
539 :param str user_prefix:
540 Like ``system_prefix`` but for the per-user config file. These
541 variables are joined as strings, not via path-style joins, so they
542 may contain partial file paths; for the per-user config file this
543 often means a leading dot, to make the final result a hidden file
544 on most systems.
546 Default: ``~/.`` (e.g. ``~/.invoke.yaml``).
548 :param str project_location:
549 Optional directory path of the currently loaded `.Collection` (as
550 loaded by `.Loader`). When non-empty, will trigger seeking of
551 per-project config files in this directory.
553 :param str runtime_path:
554 Optional file path to a runtime configuration file.
556 Used to fill the penultimate slot in the config hierarchy. Should
557 be a full file path to an existing file, not a directory path or a
558 prefix.
560 :param bool lazy:
561 Whether to automatically load some of the lower config levels.
563 By default (``lazy=False``), ``__init__`` automatically calls
564 `load_system` and `load_user` to load system and user config files,
565 respectively.
567 For more control over what is loaded when, you can say
568 ``lazy=True``, and no automatic loading is done.
570 .. note::
571 If you give ``defaults`` and/or ``overrides`` as ``__init__``
572 kwargs instead of waiting to use `load_defaults` or
573 `load_overrides` afterwards, those *will* still end up 'loaded'
574 immediately.
575 """
576 # Technically an implementation detail - do not expose in public API.
577 # Stores merged configs and is accessed via DataProxy.
578 self._set(_config={})
580 # Config file suffixes to search, in preference order.
581 self._set(_file_suffixes=("yaml", "yml", "json", "py"))
583 # Default configuration values, typically a copy of `global_defaults`.
584 if defaults is None:
585 defaults = copy_dict(self.global_defaults())
586 self._set(_defaults=defaults)
588 # Collection-driven config data, gathered from the collection tree
589 # containing the currently executing task.
590 self._set(_collection={})
592 # Path prefix searched for the system config file.
593 # NOTE: There is no default system prefix on Windows.
594 if system_prefix is None and not WINDOWS:
595 system_prefix = "/etc/"
596 self._set(_system_prefix=system_prefix)
597 # Path to loaded system config file, if any.
598 self._set(_system_path=None)
599 # Whether the system config file has been loaded or not (or ``None`` if
600 # no loading has been attempted yet.)
601 self._set(_system_found=None)
602 # Data loaded from the system config file.
603 self._set(_system={})
605 # Path prefix searched for per-user config files.
606 if user_prefix is None:
607 user_prefix = "~/."
608 self._set(_user_prefix=user_prefix)
609 # Path to loaded user config file, if any.
610 self._set(_user_path=None)
611 # Whether the user config file has been loaded or not (or ``None`` if
612 # no loading has been attempted yet.)
613 self._set(_user_found=None)
614 # Data loaded from the per-user config file.
615 self._set(_user={})
617 # As it may want to be set post-init, project conf file related attrs
618 # get initialized or overwritten via a specific method.
619 self.set_project_location(project_location)
621 # Environment variable name prefix
622 env_prefix = self.env_prefix
623 if env_prefix is None:
624 env_prefix = self.prefix
625 env_prefix = "{}_".format(env_prefix.upper())
626 self._set(_env_prefix=env_prefix)
627 # Config data loaded from the shell environment.
628 self._set(_env={})
630 # As it may want to be set post-init, runtime conf file related attrs
631 # get initialized or overwritten via a specific method.
632 self.set_runtime_path(runtime_path)
634 # Overrides - highest normal config level. Typically filled in from
635 # command-line flags.
636 if overrides is None:
637 overrides = {}
638 self._set(_overrides=overrides)
640 # Absolute highest level: user modifications.
641 self._set(_modifications={})
642 # And its sibling: user deletions. (stored as a flat dict of keypath
643 # keys and dummy values, for constant-time membership testing/removal
644 # w/ no messy recursion. TODO: maybe redo _everything_ that way? in
645 # _modifications and other levels, the values would of course be
646 # valuable and not just None)
647 self._set(_deletions={})
649 # Convenience loading of user and system files, since those require no
650 # other levels in order to function.
651 if not lazy:
652 self.load_base_conf_files()
653 # Always merge, otherwise defaults, etc are not usable until creator or
654 # a subroutine does so.
655 self.merge()
657 def load_base_conf_files(self) -> None:
658 # Just a refactor of something done in unlazy init or in clone()
659 self.load_system(merge=False)
660 self.load_user(merge=False)
662 def load_defaults(self, data: Dict[str, Any], merge: bool = True) -> None:
663 """
664 Set or replace the 'defaults' configuration level, from ``data``.
666 :param dict data: The config data to load as the defaults level.
668 :param bool merge:
669 Whether to merge the loaded data into the central config. Default:
670 ``True``.
672 :returns: ``None``.
674 .. versionadded:: 1.0
675 """
676 self._set(_defaults=data)
677 if merge:
678 self.merge()
680 def load_overrides(self, data: Dict[str, Any], merge: bool = True) -> None:
681 """
682 Set or replace the 'overrides' configuration level, from ``data``.
684 :param dict data: The config data to load as the overrides level.
686 :param bool merge:
687 Whether to merge the loaded data into the central config. Default:
688 ``True``.
690 :returns: ``None``.
692 .. versionadded:: 1.0
693 """
694 self._set(_overrides=data)
695 if merge:
696 self.merge()
698 def load_system(self, merge: bool = True) -> None:
699 """
700 Load a system-level config file, if possible.
702 Checks the configured ``_system_prefix`` path, which defaults to
703 ``/etc``, and will thus load files like ``/etc/invoke.yml``.
705 :param bool merge:
706 Whether to merge the loaded data into the central config. Default:
707 ``True``.
709 :returns: ``None``.
711 .. versionadded:: 1.0
712 """
713 self._load_file(prefix="system", merge=merge)
715 def load_user(self, merge: bool = True) -> None:
716 """
717 Load a user-level config file, if possible.
719 Checks the configured ``_user_prefix`` path, which defaults to ``~/.``,
720 and will thus load files like ``~/.invoke.yml``.
722 :param bool merge:
723 Whether to merge the loaded data into the central config. Default:
724 ``True``.
726 :returns: ``None``.
728 .. versionadded:: 1.0
729 """
730 self._load_file(prefix="user", merge=merge)
732 def load_project(self, merge: bool = True) -> None:
733 """
734 Load a project-level config file, if possible.
736 Checks the configured ``_project_prefix`` value derived from the path
737 given to `set_project_location`, which is typically set to the
738 directory containing the loaded task collection.
740 Thus, if one were to run the CLI tool against a tasks collection
741 ``/home/myuser/code/tasks.py``, `load_project` would seek out files
742 like ``/home/myuser/code/invoke.yml``.
744 :param bool merge:
745 Whether to merge the loaded data into the central config. Default:
746 ``True``.
748 :returns: ``None``.
750 .. versionadded:: 1.0
751 """
752 self._load_file(prefix="project", merge=merge)
754 def set_runtime_path(self, path: Optional[PathLike]) -> None:
755 """
756 Set the runtime config file path.
758 .. versionadded:: 1.0
759 """
760 # Path to the user-specified runtime config file.
761 self._set(_runtime_path=path)
762 # Data loaded from the runtime config file.
763 self._set(_runtime={})
764 # Whether the runtime config file has been loaded or not (or ``None``
765 # if no loading has been attempted yet.)
766 self._set(_runtime_found=None)
768 def load_runtime(self, merge: bool = True) -> None:
769 """
770 Load a runtime-level config file, if one was specified.
772 When the CLI framework creates a `Config`, it sets ``_runtime_path``,
773 which is a full path to the requested config file. This method attempts
774 to load that file.
776 :param bool merge:
777 Whether to merge the loaded data into the central config. Default:
778 ``True``.
780 :returns: ``None``.
782 .. versionadded:: 1.0
783 """
784 self._load_file(prefix="runtime", absolute=True, merge=merge)
786 def load_shell_env(self) -> None:
787 """
788 Load values from the shell environment.
790 `.load_shell_env` is intended for execution late in a `.Config`
791 object's lifecycle, once all other sources (such as a runtime config
792 file or per-collection configurations) have been loaded. Loading from
793 the shell is not terrifically expensive, but must be done at a specific
794 point in time to ensure the "only known config keys are loaded from the
795 env" behavior works correctly.
797 See :ref:`env-vars` for details on this design decision and other info
798 re: how environment variables are scanned and loaded.
800 .. versionadded:: 1.0
801 """
802 # Force merge of existing data to ensure we have an up to date picture
803 debug("Running pre-merge for shell env loading...")
804 self.merge()
805 debug("Done with pre-merge.")
806 loader = Environment(config=self._config, prefix=self._env_prefix)
807 self._set(_env=loader.load())
808 debug("Loaded shell environment, triggering final merge")
809 self.merge()
811 def load_collection(
812 self, data: Dict[str, Any], merge: bool = True
813 ) -> None:
814 """
815 Update collection-driven config data.
817 `.load_collection` is intended for use by the core task execution
818 machinery, which is responsible for obtaining collection-driven data.
819 See :ref:`collection-configuration` for details.
821 .. versionadded:: 1.0
822 """
823 debug("Loading collection configuration")
824 self._set(_collection=data)
825 if merge:
826 self.merge()
828 def set_project_location(self, path: Union[PathLike, str, None]) -> None:
829 """
830 Set the directory path where a project-level config file may be found.
832 Does not do any file loading on its own; for that, see `load_project`.
834 .. versionadded:: 1.0
835 """
836 # 'Prefix' to match the other sets of attrs
837 project_prefix = None
838 if path is not None:
839 # Ensure the prefix is normalized to a directory-like path string
840 project_prefix = join(path, "")
841 self._set(_project_prefix=project_prefix)
842 # Path to loaded per-project config file, if any.
843 self._set(_project_path=None)
844 # Whether the project config file has been loaded or not (or ``None``
845 # if no loading has been attempted yet.)
846 self._set(_project_found=None)
847 # Data loaded from the per-project config file.
848 self._set(_project={})
850 def _load_file(
851 self, prefix: str, absolute: bool = False, merge: bool = True
852 ) -> None:
853 # Setup
854 found = "_{}_found".format(prefix)
855 path = "_{}_path".format(prefix)
856 data = "_{}".format(prefix)
857 midfix = self.file_prefix
858 if midfix is None:
859 midfix = self.prefix
860 # Short-circuit if loading appears to have occurred already
861 if getattr(self, found) is not None:
862 return
863 # Moar setup
864 if absolute:
865 absolute_path = getattr(self, path)
866 # None -> expected absolute path but none set, short circuit
867 if absolute_path is None:
868 return
869 paths = [absolute_path]
870 else:
871 path_prefix = getattr(self, "_{}_prefix".format(prefix))
872 # Short circuit if loading seems unnecessary (eg for project config
873 # files when not running out of a project)
874 if path_prefix is None:
875 return
876 paths = [
877 ".".join((path_prefix + midfix, x))
878 for x in self._file_suffixes
879 ]
880 # Poke 'em
881 for filepath in paths:
882 # Normalize
883 filepath = expanduser(filepath)
884 try:
885 try:
886 type_ = splitext(filepath)[1].lstrip(".")
887 loader = getattr(self, "_load_{}".format(type_))
888 except AttributeError:
889 msg = "Config files of type {!r} (from file {!r}) are not supported! Please use one of: {!r}" # noqa
890 raise UnknownFileType(
891 msg.format(type_, filepath, self._file_suffixes)
892 )
893 # Store data, the path it was found at, and fact that it was
894 # found
895 self._set(data, loader(filepath))
896 self._set(path, filepath)
897 self._set(found, True)
898 break
899 # Typically means 'no such file', so just note & skip past.
900 except IOError as e:
901 if e.errno == 2:
902 err = "Didn't see any {}, skipping."
903 debug(err.format(filepath))
904 else:
905 raise
906 # Still None -> no suffixed paths were found, record this fact
907 if getattr(self, path) is None:
908 self._set(found, False)
909 # Merge loaded data in if any was found
910 elif merge:
911 self.merge()
913 def _load_yaml(self, path: PathLike) -> Any:
914 with open(path) as fd:
915 return yaml.safe_load(fd)
917 _load_yml = _load_yaml
919 def _load_json(self, path: PathLike) -> Any:
920 with open(path) as fd:
921 return json.load(fd)
923 def _load_py(self, path: str) -> Dict[str, Any]:
924 data = {}
925 for key, value in (load_source("mod", path)).items():
926 # Strip special members, as these are always going to be builtins
927 # and other special things a user will not want in their config.
928 if key.startswith("__"):
929 continue
930 # Raise exceptions on module values; they are unpicklable.
931 # TODO: suck it up and reimplement copy() without pickling? Then
932 # again, a user trying to stuff a module into their config is
933 # probably doing something better done in runtime/library level
934 # code and not in a "config file"...right?
935 if isinstance(value, types.ModuleType):
936 err = "'{}' is a module, which can't be used as a config value. (Are you perhaps giving a tasks file instead of a config file by mistake?)" # noqa
937 raise UnpicklableConfigMember(err.format(key))
938 data[key] = value
939 return data
941 def merge(self) -> None:
942 """
943 Merge all config sources, in order.
945 .. versionadded:: 1.0
946 """
947 debug("Merging config sources in order onto new empty _config...")
948 self._set(_config={})
949 debug("Defaults: {!r}".format(self._defaults))
950 merge_dicts(self._config, self._defaults)
951 debug("Collection-driven: {!r}".format(self._collection))
952 merge_dicts(self._config, self._collection)
953 self._merge_file("system", "System-wide")
954 self._merge_file("user", "Per-user")
955 self._merge_file("project", "Per-project")
956 debug("Environment variable config: {!r}".format(self._env))
957 merge_dicts(self._config, self._env)
958 self._merge_file("runtime", "Runtime")
959 debug("Overrides: {!r}".format(self._overrides))
960 merge_dicts(self._config, self._overrides)
961 debug("Modifications: {!r}".format(self._modifications))
962 merge_dicts(self._config, self._modifications)
963 debug("Deletions: {!r}".format(self._deletions))
964 obliterate(self._config, self._deletions)
966 def _merge_file(self, name: str, desc: str) -> None:
967 # Setup
968 desc += " config file" # yup
969 found = getattr(self, "_{}_found".format(name))
970 path = getattr(self, "_{}_path".format(name))
971 data = getattr(self, "_{}".format(name))
972 # None -> no loading occurred yet
973 if found is None:
974 debug("{} has not been loaded yet, skipping".format(desc))
975 # True -> hooray
976 elif found:
977 debug("{} ({}): {!r}".format(desc, path, data))
978 merge_dicts(self._config, data)
979 # False -> did try, did not succeed
980 else:
981 # TODO: how to preserve what was tried for each case but only for
982 # the negative? Just a branch here based on 'name'?
983 debug("{} not found, skipping".format(desc))
985 def clone(self, into: Optional[Type["Config"]] = None) -> "Config":
986 """
987 Return a copy of this configuration object.
989 The new object will be identical in terms of configured sources and any
990 loaded (or user-manipulated) data, but will be a distinct object with
991 as little shared mutable state as possible.
993 Specifically, all `dict` values within the config are recursively
994 recreated, with non-dict leaf values subjected to `copy.copy` (note:
995 *not* `copy.deepcopy`, as this can cause issues with various objects
996 such as compiled regexen or threading locks, often found buried deep
997 within rich aggregates like API or DB clients).
999 The only remaining config values that may end up shared between a
1000 config and its clone are thus those 'rich' objects that do not
1001 `copy.copy` cleanly, or compound non-dict objects (such as lists or
1002 tuples).
1004 :param into:
1005 A `.Config` subclass that the new clone should be "upgraded" to.
1007 Used by client libraries which have their own `.Config` subclasses
1008 that e.g. define additional defaults; cloning "into" one of these
1009 subclasses ensures that any new keys/subtrees are added gracefully,
1010 without overwriting anything that may have been pre-defined.
1012 Default: ``None`` (just clone into another regular `.Config`).
1014 :returns:
1015 A `.Config`, or an instance of the class given to ``into``.
1017 .. versionadded:: 1.0
1018 """
1019 # Construct new object
1020 klass = self.__class__ if into is None else into
1021 # Also allow arbitrary constructor kwargs, for subclasses where passing
1022 # (some) data in at init time is desired (vs post-init copying)
1023 # TODO: probably want to pivot the whole class this way eventually...?
1024 # No longer recall exactly why we went with the 'fresh init + attribute
1025 # setting' approach originally...tho there's clearly some impedance
1026 # mismatch going on between "I want stuff to happen in my config's
1027 # instantiation" and "I want cloning to not trigger certain things like
1028 # external data source loading".
1029 # NOTE: this will include lazy=True, see end of method
1030 new = klass(**self._clone_init_kwargs(into=into))
1031 # Copy/merge/etc all 'private' data sources and attributes
1032 for name in """
1033 collection
1034 system_prefix
1035 system_path
1036 system_found
1037 system
1038 user_prefix
1039 user_path
1040 user_found
1041 user
1042 project_prefix
1043 project_path
1044 project_found
1045 project
1046 env_prefix
1047 env
1048 runtime_path
1049 runtime_found
1050 runtime
1051 overrides
1052 modifications
1053 """.split():
1054 name = "_{}".format(name)
1055 my_data = getattr(self, name)
1056 # Non-dict data gets carried over straight (via a copy())
1057 # NOTE: presumably someone could really screw up and change these
1058 # values' types, but at that point it's on them...
1059 if not isinstance(my_data, dict):
1060 new._set(name, copy.copy(my_data))
1061 # Dict data gets merged (which also involves a copy.copy
1062 # eventually)
1063 else:
1064 merge_dicts(getattr(new, name), my_data)
1065 # Do what __init__ would've done if not lazy, i.e. load user/system
1066 # conf files.
1067 new.load_base_conf_files()
1068 # Finally, merge() for reals (_load_base_conf_files doesn't do so
1069 # internally, so that data wouldn't otherwise show up.)
1070 new.merge()
1071 return new
1073 def _clone_init_kwargs(
1074 self, into: Optional[Type["Config"]] = None
1075 ) -> Dict[str, Any]:
1076 """
1077 Supply kwargs suitable for initializing a new clone of this object.
1079 Note that most of the `.clone` process involves copying data between
1080 two instances instead of passing init kwargs; however, sometimes you
1081 really do want init kwargs, which is why this method exists.
1083 :param into: The value of ``into`` as passed to the calling `.clone`.
1085 :returns: A `dict`.
1086 """
1087 # NOTE: must pass in defaults fresh or otherwise global_defaults() gets
1088 # used instead. Except when 'into' is in play, in which case we truly
1089 # want the union of the two.
1090 new_defaults = copy_dict(self._defaults)
1091 if into is not None:
1092 merge_dicts(new_defaults, into.global_defaults())
1093 # The kwargs.
1094 return dict(
1095 defaults=new_defaults,
1096 # TODO: consider making this 'hardcoded' on the calling end (ie
1097 # inside clone()) to make sure nobody accidentally nukes it via
1098 # subclassing?
1099 lazy=True,
1100 )
1102 def _modify(self, keypath: Tuple[str, ...], key: str, value: str) -> None:
1103 """
1104 Update our user-modifications config level with new data.
1106 :param tuple keypath:
1107 The key path identifying the sub-dict being updated. May be an
1108 empty tuple if the update is occurring at the topmost level.
1110 :param str key:
1111 The actual key receiving an update.
1113 :param value:
1114 The value being written.
1115 """
1116 # First, ensure we wipe the keypath from _deletions, in case it was
1117 # previously deleted.
1118 excise(self._deletions, keypath + (key,))
1119 # Now we can add it to the modifications structure.
1120 data = self._modifications
1121 keypath_list = list(keypath)
1122 while keypath_list:
1123 subkey = keypath_list.pop(0)
1124 # TODO: could use defaultdict here, but...meh?
1125 if subkey not in data:
1126 # TODO: generify this and the subsequent 3 lines...
1127 data[subkey] = {}
1128 data = data[subkey]
1129 data[key] = value
1130 self.merge()
1132 def _remove(self, keypath: Tuple[str, ...], key: str) -> None:
1133 """
1134 Like `._modify`, but for removal.
1135 """
1136 # NOTE: because deletions are processed in merge() last, we do not need
1137 # to remove things from _modifications on removal; but we *do* do the
1138 # inverse - remove from _deletions on modification.
1139 # TODO: may be sane to push this step up to callers?
1140 data = self._deletions
1141 keypath_list = list(keypath)
1142 while keypath_list:
1143 subkey = keypath_list.pop(0)
1144 if subkey in data:
1145 data = data[subkey]
1146 # If we encounter None, it means something higher up than our
1147 # requested keypath is already marked as deleted; so we don't
1148 # have to do anything or go further.
1149 if data is None:
1150 return
1151 # Otherwise it's presumably another dict, so keep looping...
1152 else:
1153 # Key not found -> nobody's marked anything along this part of
1154 # the path for deletion, so we'll start building it out.
1155 data[subkey] = {}
1156 # Then prep for next iteration
1157 data = data[subkey]
1158 # Exited loop -> data must be the leafmost dict, so we can now set our
1159 # deleted key to None
1160 data[key] = None
1161 self.merge()
1164class AmbiguousMergeError(ValueError):
1165 pass
1168def merge_dicts(
1169 base: Dict[str, Any], updates: Dict[str, Any]
1170) -> Dict[str, Any]:
1171 """
1172 Recursively merge dict ``updates`` into dict ``base`` (mutating ``base``.)
1174 * Values which are themselves dicts will be recursed into.
1175 * Values which are a dict in one input and *not* a dict in the other input
1176 (e.g. if our inputs were ``{'foo': 5}`` and ``{'foo': {'bar': 5}}``) are
1177 irreconciliable and will generate an exception.
1178 * Non-dict leaf values are run through `copy.copy` to avoid state bleed.
1180 .. note::
1181 This is effectively a lightweight `copy.deepcopy` which offers
1182 protection from mismatched types (dict vs non-dict) and avoids some
1183 core deepcopy problems (such as how it explodes on certain object
1184 types).
1186 :returns:
1187 The value of ``base``, which is mostly useful for wrapper functions
1188 like `copy_dict`.
1190 .. versionadded:: 1.0
1191 """
1192 # TODO: for chrissakes just make it return instead of mutating?
1193 for key, value in (updates or {}).items():
1194 # Dict values whose keys also exist in 'base' -> recurse
1195 # (But only if both types are dicts.)
1196 if key in base:
1197 if isinstance(value, dict):
1198 if isinstance(base[key], dict):
1199 merge_dicts(base[key], value)
1200 else:
1201 raise _merge_error(base[key], value)
1202 else:
1203 if isinstance(base[key], dict):
1204 raise _merge_error(base[key], value)
1205 # Fileno-bearing objects are probably 'real' files which do not
1206 # copy well & must be passed by reference. Meh.
1207 elif hasattr(value, "fileno"):
1208 base[key] = value
1209 else:
1210 base[key] = copy.copy(value)
1211 # New values get set anew
1212 else:
1213 # Dict values get reconstructed to avoid being references to the
1214 # updates dict, which can lead to nasty state-bleed bugs otherwise
1215 if isinstance(value, dict):
1216 base[key] = copy_dict(value)
1217 # Fileno-bearing objects are probably 'real' files which do not
1218 # copy well & must be passed by reference. Meh.
1219 elif hasattr(value, "fileno"):
1220 base[key] = value
1221 # Non-dict values just get set straight
1222 else:
1223 base[key] = copy.copy(value)
1224 return base
1227def _merge_error(orig: object, new: object) -> AmbiguousMergeError:
1228 return AmbiguousMergeError(
1229 "Can't cleanly merge {} with {}".format(
1230 _format_mismatch(orig), _format_mismatch(new)
1231 )
1232 )
1235def _format_mismatch(x: object) -> str:
1236 return "{} ({!r})".format(type(x), x)
1239def copy_dict(source: Dict[str, Any]) -> Dict[str, Any]:
1240 """
1241 Return a fresh copy of ``source`` with as little shared state as possible.
1243 Uses `merge_dicts` under the hood, with an empty ``base`` dict; see its
1244 documentation for details on behavior.
1246 .. versionadded:: 1.0
1247 """
1248 return merge_dicts({}, source)
1251def excise(dict_: Dict[str, Any], keypath: Tuple[str, ...]) -> None:
1252 """
1253 Remove key pointed at by ``keypath`` from nested dict ``dict_``, if exists.
1255 .. versionadded:: 1.0
1256 """
1257 data = dict_
1258 keypath_list = list(keypath)
1259 leaf_key = keypath_list.pop()
1260 while keypath_list:
1261 key = keypath_list.pop(0)
1262 if key not in data:
1263 # Not there, nothing to excise
1264 return
1265 data = data[key]
1266 if leaf_key in data:
1267 del data[leaf_key]
1270def obliterate(base: Dict[str, Any], deletions: Dict[str, Any]) -> None:
1271 """
1272 Remove all (nested) keys mentioned in ``deletions``, from ``base``.
1274 .. versionadded:: 1.0
1275 """
1276 for key, value in deletions.items():
1277 if isinstance(value, dict):
1278 # NOTE: not testing for whether base[key] exists; if something's
1279 # listed in a deletions structure, it must exist in some source
1280 # somewhere, and thus also in the cache being obliterated.
1281 obliterate(base[key], deletions[key])
1282 else: # implicitly None
1283 del base[key]