From f80b7dff65c5df4e9c9364b6e1ae97ff55a6554c Mon Sep 17 00:00:00 2001 From: novirium Date: Thu, 19 Dec 2019 15:34:49 +0800 Subject: [PATCH] Docs, added layer system to config, updated plugins --- shepherd/config.py | 93 +++++++++++++++++++++++++++++++++++----------- 1 file changed, 71 insertions(+), 22 deletions(-) diff --git a/shepherd/config.py b/shepherd/config.py index 17cd6c1..32cfb8d 100644 --- a/shepherd/config.py +++ b/shepherd/config.py @@ -38,6 +38,7 @@ Root items that are not dicts are not supported, for instance both the following import re import toml from abc import ABC, abstractmethod +from copy import deepcopy from .freezedry import freezedryable, rehydrate @@ -58,6 +59,10 @@ class _ConfigDefinition(ABC): @abstractmethod def validate(self, value): + """ + Checks the supplied value to confirm that it complies with this ConfigDefinition. + Raises InvalidConfigError on failure. + """ pass @@ -121,7 +126,14 @@ class DictDef(_ConfigDefinition): self.def_dict[name] = newdef return newdef - def validate(self, value_dict): # pylint: disable=W0221 + def validate(self, value_dict): + """ + Checks the supplied value to confirm that it complies with this ConfigDefinition. + Raises InvalidConfigError on failure. + + This *can* modify the supplied value dict, inserting defaults for any child + ConfigDefinitions that are marked as optional. + """ def_set = set(self.def_dict.keys()) value_set = set(value_dict.keys()) @@ -210,6 +222,22 @@ class ConfigManager(): def __init__(self): self.root_config = {} self.confdefs = {} + self.frozen_config = {} + + @staticmethod + def _load_source(source): + """ + Accept a filepath or opened file representing a TOML file, or a direct dict, + and return a plain parsed dict. + """ + if isinstance(source, dict): # load from dict + return source + elif isinstance(source, str): # load from pathname + with open(source, 'r') as conf_file: + return toml.load(conf_file) + else: # load from file + return toml.load(source) + def load(self, source): """ @@ -219,13 +247,8 @@ class ConfigManager(): source: Either a dict config to load directly, a filepath to a TOML file, or an open file. """ - if isinstance(source, dict): # load from dict - self.root_config = source - elif isinstance(source, str): # load from pathname - with open(source, 'r') as conf_file: - self.root_config = toml.load(conf_file) - else: # load from file - self.root_config = toml.load(source) + self.root_config = self._load_source(source) + self._overlay(self.frozen_config, self.root_config) def load_overlay(self, source): """ @@ -237,17 +260,40 @@ class ConfigManager(): Args: source: Either the root dict of a data structure to load directly, a filepath to a TOML file, or an open TOML file. - """ + """ + self._overlay(self._load_source(source), self.root_config) + self._overlay(self.frozen_config, self.root_config) - if isinstance(source, dict): # load from dict - new_source = source - elif isinstance(source, str): # load from pathname - with open(source, 'r') as conf_file: - new_source = toml.load(conf_file) - else: # load from file - new_source = toml.load(source) + + def freeze_value(self, bundle_name, *field_names): + """ + Freeze the given config field so that subsequent calls to ``load`` and ``load_overlay`` + cannot change it. Can only be used for dict values or dict values nested in parent dicts. - self._overlay(new_source, self.root_config) + Args: + bundle_name: The name of the bundle to look for the field in. + *field_names: a series of strings that locate the config field, either a single + key or series of nested keys. + """ + + #Bundle names are really no different from any other nested dict + names = (bundle_name,) + field_names + + target_field = self.root_config + frozen_value = self.frozen_config + + # Cycle through nested names, creating frozen_config nested dicts as necessary + for name in names[:-1]: + target_field = target_field[name] + if name not in frozen_value: + frozen_value[name] = {} + frozen_value = frozen_value[name] + + + frozen_value[names[-1]] = target_field[names[-1]] + + + def add_confdef(self, bundle_name, confdef): """ @@ -290,10 +336,13 @@ class ConfigManager(): Get a config bundle called ``bundle_name`` and validate it against the corresponding config definition stored in the ConfigManager. If ``conf_def`` is supplied, it gets used instead. Returns a validated - config bundle dict. + config bundle dict. Note that as part of validation, optional keys that are missing will be - filled in with their default values (see ``DictDef``). + filled in with their default values (see ``DictDef``). This function will copy + the config bundle *after* validation, and so config loaded in the ConfManager will + be modified, but future ConfigManager manipulations won't change the returned config + bundle. Args: config_name: (str) Name of the config dict to find. @@ -308,14 +357,14 @@ class ConfigManager(): try: conf_def.validate(self.root_config[bundle_name]) except InvalidConfigError as e: - e.args = ("Module: " + bundle_name,) + e.args + e.args = ("Bundle: " + bundle_name,) + e.args raise - return self.root_config[bundle_name] + return deepcopy(self.root_config[bundle_name]) def get_config_bundles(self, bundle_names): """ Get multiple config bundles from the root dict at once, validating each one with the - corresponding confdef stored in the ConfigManager. + corresponding confdef stored in the ConfigManager. See ``get_config_bundle`` Args: bundle_names: A list of config bundle names to get. If dictionary is supplied, uses the values