Docs, added layer system to config, updated plugins

shepherd/config.py

@@ -38,6 +38,7 @@ Root items that are not dicts are not supported, for instance both the following
 import re
 import toml
 from abc import ABC, abstractmethod
+from copy import deepcopy
 
 from .freezedry import freezedryable, rehydrate
 
@@ -58,6 +59,10 @@ class _ConfigDefinition(ABC):
 
     @abstractmethod
     def validate(self, value):
+        """
+        Checks the supplied value to confirm that it complies with this ConfigDefinition.
+        Raises InvalidConfigError on failure.
+        """
         pass
         
 
@@ -121,7 +126,14 @@ class DictDef(_ConfigDefinition):
         self.def_dict[name] = newdef
         return newdef
 
-    def validate(self, value_dict):  # pylint: disable=W0221
+    def validate(self, value_dict):
+        """
+        Checks the supplied value to confirm that it complies with this ConfigDefinition.
+        Raises InvalidConfigError on failure.
+
+        This *can* modify the supplied value dict, inserting defaults for any child
+        ConfigDefinitions that are marked as optional.
+        """
         def_set = set(self.def_dict.keys())
         value_set = set(value_dict.keys())
 
@@ -210,6 +222,22 @@ class ConfigManager():
     def __init__(self):
         self.root_config = {}
         self.confdefs = {}
+        self.frozen_config = {}
+
+    @staticmethod
+    def _load_source(source):
+        """
+        Accept a filepath or opened file representing a TOML file, or a direct dict,
+        and return a plain parsed dict.
+        """
+        if isinstance(source, dict):  # load from dict
+           return source
+        elif isinstance(source, str):  # load from pathname
+            with open(source, 'r') as conf_file:
+                return toml.load(conf_file) 
+        else:  # load from file
+            return toml.load(source)
+
 
     def load(self, source):
         """
@@ -219,13 +247,8 @@ class ConfigManager():
             source: Either a dict config to load directly, a filepath to a TOML file,
                 or an open file.
         """
-        if isinstance(source, dict):  # load from dict
+        self.root_config = self._load_source(source)
-            self.root_config = source
+        self._overlay(self.frozen_config, self.root_config)
-        elif isinstance(source, str):  # load from pathname
-            with open(source, 'r') as conf_file:
-                self.root_config = toml.load(conf_file)
-        else:  # load from file
-            self.root_config = toml.load(source)
 
     def load_overlay(self, source):
         """
@@ -237,17 +260,40 @@ class ConfigManager():
         Args:
             source: Either the root dict of a data structure to load directly, a filepath to a TOML file,
                 or an open TOML file.
-        """
+        """        
+        self._overlay(self._load_source(source), self.root_config)
+        self._overlay(self.frozen_config, self.root_config)
 
-        if isinstance(source, dict):  # load from dict
+
-             new_source = source
+    def freeze_value(self, bundle_name, *field_names):
-        elif isinstance(source, str):  # load from pathname
+        """
-            with open(source, 'r') as conf_file:
+        Freeze the given config field so that subsequent calls to ``load`` and ``load_overlay``
-                new_source = toml.load(conf_file)
+        cannot change it. Can only be used for dict values or dict values nested in parent dicts.
-        else:  # load from file
-            new_source = toml.load(source)
         
-        self._overlay(new_source, self.root_config)
+        Args:
+            bundle_name: The name of the bundle to look for the field in.
+            *field_names: a series of strings that locate the config field, either a single
+                key or series of nested keys.
+        """
+
+        #Bundle names are really no different from any other nested dict
+        names = (bundle_name,) + field_names
+
+        target_field = self.root_config
+        frozen_value = self.frozen_config
+
+        # Cycle through nested names, creating frozen_config nested dicts as necessary
+        for name in names[:-1]:
+            target_field = target_field[name]
+            if name not in frozen_value:
+                frozen_value[name] = {}
+            frozen_value = frozen_value[name]
+
+
+        frozen_value[names[-1]] = target_field[names[-1]]
+
+
+
 
     def add_confdef(self, bundle_name, confdef):
         """
@@ -290,10 +336,13 @@ class ConfigManager():
         Get a config bundle called ``bundle_name`` and validate
         it against the corresponding config definition stored in the ConfigManager.
         If ``conf_def`` is supplied, it gets used instead. Returns a validated
-        config bundle dict.
+        config bundle dict. 
 
         Note that as part of validation, optional keys that are missing will be
-        filled in with their default values (see ``DictDef``).
+        filled in with their default values (see ``DictDef``). This function will copy
+        the config bundle *after* validation, and so config loaded in the ConfManager will
+        be modified, but future ConfigManager manipulations won't change the returned config
+        bundle.
 
         Args:
             config_name: (str) Name of the config dict to find.
@@ -308,14 +357,14 @@ class ConfigManager():
         try:
             conf_def.validate(self.root_config[bundle_name])
         except InvalidConfigError as e:
-            e.args = ("Module: " + bundle_name,) + e.args
+            e.args = ("Bundle: " + bundle_name,) + e.args
             raise
-        return self.root_config[bundle_name]
+        return deepcopy(self.root_config[bundle_name])
 
     def get_config_bundles(self, bundle_names):
         """
         Get multiple config bundles from the root dict at once, validating each one with the 
-        corresponding confdef stored in the ConfigManager.
+        corresponding confdef stored in the ConfigManager. See ``get_config_bundle``
 
         Args:
             bundle_names: A list of config bundle names to get. If dictionary is supplied, uses the values