commit 4545e86a9fb6508a2098a2eaff8e04e3be6bca42
Author: novirium <t.wilson.au@gmail.com>
Date:   Thu Jun 13 15:42:16 2019 +0800

    Restructure

diff --git a/shepherd/config.py b/shepherd/config.py
new file mode 100644
index 0000000..6c7a8bc
--- /dev/null
+++ b/shepherd/config.py
@@ -0,0 +1,223 @@
+import re
+import toml
+
+
+class InvalidConfigError(Exception):
+    pass
+
+# On start,
+# get conf_defs via imports in core
+# Load a conf file
+# Validate the Shepherd table in conf file, then load its values to get module list
+# Validate other loaded tables/modules, then load their values
+
+# How do modules wind up with their instance of ConfigValues? Return val instance
+# from validation function - as it needs to build the vals instance while validating anyway
+
+# Validate a conf file given a module or config_def list
+
+
+# idea is to create these similar to how arg parser works
+
+# how to store individual items in def? Dict of...
+# Tables need a dict, lists/arrays need a list of dicts, but what's in the dict?
+# if it's another instacnce of ConfigDef, then each Config Def needs to handle
+# one item, but cater for all the different types of items - and many of those
+# should be able to add a new item.
+
+# could have a separate class for the root, but lower tables really need to
+# perform exactly the same...
+
+# config def required interface:
+# Validate values.
+
+class _ConfigDefinition():
+    def __init__(self, default=None, optional=False):
+        self.default = default
+        self.optional = optional
+
+    def validate(self, value): # pylint: disable=W0613
+        raise TypeError("_ConfigDefinition.validate() is an abstract method")
+
+
+class BoolDef(_ConfigDefinition):
+    def __init__(self, default=None, optional=False): # pylint: disable=W0235
+        super().__init__(default, optional)
+
+    def validate(self, value):
+        if not isinstance(value, bool):
+            raise InvalidConfigError("Config value must be a boolean")
+
+
+class IntDef(_ConfigDefinition):
+    def __init__(self, default=None, minval=None, maxval=None,
+                 optional=False):
+        super().__init__(default, optional)
+        self.minval = minval
+        self.maxval = maxval
+
+    def validate(self, value):
+        if not isinstance(value, int):
+            raise InvalidConfigError("Config value must be an integer")
+        if self.minval is not None and value < self.minval:
+            raise InvalidConfigError("Config value must be >= " +
+                                     str(self.minval))
+        if self.maxval is not None and value > self.maxval:
+            raise InvalidConfigError("Config value must be <= " +
+                                     str(self.maxval))
+
+
+class StringDef(_ConfigDefinition):
+    def __init__(self, default=None, minlength=None, maxlength=None,
+                 optional=False):
+        super().__init__(default, optional)
+        self.minlength = minlength
+        self.maxlength = maxlength
+
+    def validate(self, value):
+        if not isinstance(value, str):
+            raise InvalidConfigError("Config value must be a string")
+        if self.minlength is not None and len(value) < self.minlength:
+            raise InvalidConfigError("Config string length must be >= " +
+                                     str(self.minlength))
+        if self.maxlength is not None and len(value) > self.maxlength:
+            raise InvalidConfigError("Config string length must be <= " +
+                                     str(self.maxlength))
+
+
+class TableDef(_ConfigDefinition):
+    def __init__(self, default=None, optional=False):
+        super().__init__(default, optional)
+        self.def_table = {}
+
+    def add_def(self, name, newdef):
+        if not isinstance(newdef, _ConfigDefinition):
+            raise TypeError("Config definiton must be an instance of a "
+                            "ConfigDefinition subclass")
+        if not isinstance(name, str):
+            raise TypeError("Config definition name must be a string")
+        self.def_table[name] = newdef
+        return newdef
+
+    def validate(self, value_table): # pylint: disable=W0221
+        def_set = set(self.def_table.keys())
+        value_set = set(value_table.keys())
+
+        for missing_key in def_set - value_set:
+            if not self.def_table[missing_key].optional:
+                raise InvalidConfigError("Table must contain key: " +
+                                         missing_key)
+            else:
+                value_table[missing_key] = self.def_table[missing_key].default
+
+        for extra_key in value_set - def_set:
+            raise InvalidConfigError("Table contains unknown key: " +
+                                     extra_key)
+
+        for key, value in value_table.items():
+            try:
+                self.def_table[key].validate(value)
+            except InvalidConfigError as e:
+                e.args = ("Key: " + key,) + e.args
+                raise
+
+
+class _ArrayDefMixin():
+    def validate(self, value_array):
+        if not isinstance(value_array, list):
+            raise InvalidConfigError("Config item must be an array")
+        for index, value in enumerate(value_array):
+            try:
+                super().validate(value)
+            except InvalidConfigError as e:
+                e.args = ("Array index: " + str(index),) + e.args
+                raise
+
+
+class BoolArrayDef(_ArrayDefMixin, BoolDef):
+    pass
+
+
+class IntArrayDef(_ArrayDefMixin, IntDef):
+    pass
+
+
+class StringArrayDef(_ArrayDefMixin, StringDef):
+    pass
+
+
+class TableArrayDef(_ArrayDefMixin, TableDef):
+    pass
+
+
+class ConfDefinition(TableDef):
+    pass
+
+
+class ConfigManager():
+    def __init__(self):
+        self.root_config = {}
+
+    def load(self, source):
+        if isinstance(source, dict):  # load from dict
+            self.root_config = source
+        elif isinstance(source, str):  # load from pathname
+            with open(source, 'r') as conf_file:
+                self.root_config = toml.load(conf_file)
+        else:  # load from file
+            self.root_config = toml.load(source)
+
+    def get_config(self, table_name, conf_def):
+        if not isinstance(conf_def, ConfDefinition):
+            raise TypeError("Supplied config definition must be an instance "
+                            "of ConfDefinition")
+        if table_name not in self.root_config:
+            raise InvalidConfigError("Config must contain table: " + table_name)
+        try:
+            conf_def.validate(self.root_config[table_name])
+        except InvalidConfigError as e:
+            e.args = ("Module: " + table_name,) + e.args
+            raise
+        return self.root_config[table_name]
+
+    def get_configs(self, conf_defs):
+        config_values = {}
+        for name, conf_def in conf_defs.items():
+            config_values[name] = self.get_config(name, conf_def)
+        return config_values
+
+    def get_module_configs(self, modules):
+        config_values = {}
+        for name, module in modules.items():
+            config_values[name] = self.get_config(name, module.conf_def)
+        return config_values
+
+    def dump_toml(self):
+        return toml.dumps(self.root_config)
+
+    def dump_to_file(self, filepath, message=None):
+        with open(filepath, 'w+') as f:
+            content = self.dump_toml()
+            if message is not None:
+                content = content.rstrip() + gen_comment(message)
+            f.write(content)
+
+
+def strip_toml_message(string):
+    print("stripping...")
+    return re.sub("(?m)^#\\ shepherd_message:[^\\n]*$\\n?(?:^#[^\\n]+$\\n?)*",
+                  '', string)
+
+
+def update_toml_message(filepath, message):
+    with open(filepath, 'r+') as f:
+        content = f.read()
+        content = strip_toml_message(content).rstrip()
+        content += gen_comment(message)
+        f.seek(0)
+        f.write(content)
+        f.truncate()
+
+
+def gen_comment(string):
+    return '\n# shepherd_message: ' + '\n# '.join(string.replace('#', '').splitlines()) + '\n'