From 5f1d3be59fd7978ec3cf2ee9d86e742c9f664eab Mon Sep 17 00:00:00 2001 From: novirium Date: Mon, 23 Sep 2019 12:08:36 +0800 Subject: [PATCH] Update config to use Python terminology --- shepherd/config.py | 176 +++++++++++++++++++++++---------------------- 1 file changed, 90 insertions(+), 86 deletions(-) diff --git a/shepherd/config.py b/shepherd/config.py index ac732fe..0e43298 100644 --- a/shepherd/config.py +++ b/shepherd/config.py @@ -1,66 +1,69 @@ """ -Configuration managment module. Enables configuration to be validated against +Configuration management module. Enables configuration to be validated against requirement definitions before being loaded and used. -""" +Compatible with both raw config data structures and TOML files, config data must +start with a root dict containing named "config bundles". These are intended to +refer to different modular parts of the application needing configuration, and the +config data structure must contain at least one. -import re -import toml - -from .freezedry import freezedryable, rehydrate +Each config bundle itself needs to have a dict at the root, and so in practice a minimal +TOML config file would look like:: + [myapp] + config_thingy_a = "foooooo!" + important_number = 8237 +This would resolve to a config bundle named "myapp" that results in the dict:: + {"config_thingy_a": "foooooo!", "important_number": 8237} -class InvalidConfigError(Exception): - pass +Root items that are not dicts are not supported, for instance both the following TOML files would fail:: -# On start, -# get conf_defs via imports in core -# Load a conf file -# Validate the Shepherd table in conf file, then load its values to get module list -# Validate other loaded tables/modules, then load their values + [[myapp]] + important_number = 8237 + [[myapp]] + another_important_number = 2963 -# How do modules wind up with their instance of ConfigValues? Return val instance -# from validation function - as it needs to build the vals instance while validating anyway +(root object in bundle is a list) -# Validate a conf file given a module or config_def list +:: + root_thingy = 46 -# idea is to create these similar to how arg parser works +(root object in config is a single value) +""" -# how to store individual items in def? Dict of... -# Tables need a dict, lists/arrays need a list of dicts, but what's in the dict? -# if it's another instacnce of ConfigDef, then each Config Def needs to handle -# one item, but cater for all the different types of items - and many of those -# should be able to add a new item. -# could have a separate class for the root, but lower tables really need to -# perform exactly the same... +import re +import toml +from abc import ABC, abstractmethod -# config def required interface: -# Validate values. +from .freezedry import freezedryable, rehydrate -# The Table and Array terms used here are directly from the TOML convention, but they essentially -# map directly to Dictionaries (Tables), and Lists (Arrays) +class InvalidConfigError(Exception): + pass +# The Table and Array terms from the TOML convention essentially +# map directly to Dictionaries (Tables), and Lists (Arrays) -class _ConfigDefinition(): +class _ConfigDefinition(ABC): def __init__(self, default=None, optional=False, helptext=""): self.default = default self.optional = optional self.helptext = helptext - def validate(self, value): # pylint: disable=W0613 - raise TypeError("_ConfigDefinition.validate() is an abstract method") + @abstractmethod + def validate(self, value): + pass @freezedryable class BoolDef(_ConfigDefinition): - def __init__(self, default=None, optional=False, helptext=""): # pylint: disable=W0235 + def __init__(self, default=None, optional=False, helptext=""): super().__init__(default, optional, helptext) def validate(self, value): @@ -104,10 +107,10 @@ class StringDef(_ConfigDefinition): str(self.maxlength)) @freezedryable -class TableDef(_ConfigDefinition): +class DictDef(_ConfigDefinition): def __init__(self, default=None, optional=False, helptext=""): super().__init__(default, optional, helptext) - self.def_table = {} + self.def_dict = {} def add_def(self, name, newdef): if not isinstance(newdef, _ConfigDefinition): @@ -115,61 +118,62 @@ class TableDef(_ConfigDefinition): "ConfigDefinition subclass") if not isinstance(name, str): raise TypeError("Config definition name must be a string") - self.def_table[name] = newdef + self.def_dict[name] = newdef return newdef - def validate(self, value_table): # pylint: disable=W0221 - def_set = set(self.def_table.keys()) - value_set = set(value_table.keys()) + def validate(self, value_dict): # pylint: disable=W0221 + def_set = set(self.def_dict.keys()) + value_set = set(value_dict.keys()) for missing_key in def_set - value_set: - if not self.def_table[missing_key].optional: - raise InvalidConfigError("Table must contain key: " + + if not self.def_dict[missing_key].optional: + raise InvalidConfigError("Dict must contain key: " + missing_key) else: - value_table[missing_key] = self.def_table[missing_key].default + value_dict[missing_key] = self.def_dict[missing_key].default for extra_key in value_set - def_set: - raise InvalidConfigError("Table contains unknown key: " + + raise InvalidConfigError("Dict contains unknown key: " + extra_key) - for key, value in value_table.items(): + for key, value in value_dict.items(): try: - self.def_table[key].validate(value) + self.def_dict[key].validate(value) except InvalidConfigError as e: e.args = ("Key: " + key,) + e.args raise -class _ArrayDefMixin(): - def validate(self, value_array): - if not isinstance(value_array, list): - raise InvalidConfigError("Config item must be an array") - for index, value in enumerate(value_array): + +class _ListDefMixin(): + def validate(self, value_list): + if not isinstance(value_list, list): + raise InvalidConfigError("Config item must be a list") + for index, value in enumerate(value_list): try: super().validate(value) except InvalidConfigError as e: - e.args = ("Array index: " + str(index),) + e.args + e.args = ("List index: " + str(index),) + e.args raise @freezedryable -class BoolArrayDef(_ArrayDefMixin, BoolDef): +class BoolListDef(_ListDefMixin, BoolDef): pass @freezedryable -class IntArrayDef(_ArrayDefMixin, IntDef): +class IntListDef(_ListDefMixin, IntDef): pass @freezedryable -class StringArrayDef(_ArrayDefMixin, StringDef): +class StringListDef(_ListDefMixin, StringDef): pass @freezedryable -class TableArrayDef(_ArrayDefMixin, TableDef): +class DictListDef(_ListDefMixin, DictDef): pass @freezedryable -class ConfDefinition(TableDef): +class ConfDefinition(DictDef): pass @@ -202,8 +206,8 @@ class ConfigManager(): value and completely replaced. Args: - source: Either a dict config to load directly, a filepath to a TOML file, - or an open file. + source: Either the root dict of a data structure to load directly, a filepath to a TOML file, + or an open TOML file. """ if isinstance(source, dict): # load from dict @@ -216,28 +220,28 @@ class ConfigManager(): self._overlay(new_source, self.root_config) - def add_confdef(self, name, confdef): + def add_confdef(self, bundle_name, confdef): """ - Stores a ConfigDefinition for future use when validating the corresponding config table + Stores a ConfigDefinition for future use when validating the corresponding config bundle Args: - name (str) : Then name to store the config definition under. + bundle_name (str) : The name to store the config definition under. confdef (ConfigDefinition): The populated ConfigDefinition to store. """ - self.confdefs[name]=confdef + self.confdefs[bundle_name]=confdef def add_confdefs(self, confdefs): """ - Stores multiple ConfigDefinitions at once for future use when validating the corresponding config tables + Stores multiple ConfigDefinitions at once for future use when validating the corresponding config bundles Args: confdefs : A dict of populated ConfigDefinitions to store, using their keys as names. """ self.confdefs.update(confdefs) - def get_missing_confdefs(self): + def list_missing_confdefs(self): """ - Returns a list of config table names that do not have a corresponding ConfigDefinition + Returns a list of config bundle names that do not have a corresponding ConfigDefinition stored in the ConfigManager. """ return list(self.root_config.keys() - self.confdefs.keys()) @@ -252,57 +256,57 @@ class ConfigManager(): # Otherwise it's either an existing value to be replaced or needs to be added. dest[key] = src[key] - def validate_and_get_config(self, config_name, conf_def=None): + def get_config_bundle(self, bundle_name, conf_def=None): """ - Get a config dict called ``table_name`` and validate + Get a config bundle called ``bundle_name`` and validate it against the corresponding config definition stored in the ConfigManager. If ``conf_def`` is supplied, it gets used instead. Returns a validated - config dictionary. + config bundle dict. Note that as part of validation, optional keys that are missing will be - filled in with their default values (see ``TableDef``). + filled in with their default values (see ``DictDef``). Args: - table_name: (str) Name of the config dict to find. + config_name: (str) Name of the config dict to find. conf_def: (ConfDefinition) Optional config definition to validate against. """ if not isinstance(conf_def, ConfDefinition): - conf_def = self.confdefs[config_name] + conf_def = self.confdefs[bundle_name] - if config_name not in self.root_config: + if bundle_name not in self.root_config: raise InvalidConfigError( - "Config must contain table: " + config_name) + "Config must contain dict: " + bundle_name) try: - conf_def.validate(self.root_config[config_name]) + conf_def.validate(self.root_config[bundle_name]) except InvalidConfigError as e: - e.args = ("Module: " + config_name,) + e.args + e.args = ("Module: " + bundle_name,) + e.args raise - return self.root_config[config_name] + return self.root_config[bundle_name] - def validate_and_get_configs(self, config_names): + def get_config_bundles(self, bundle_names): """ - Get multiple configs from the root table at once, validating each one with the + Get multiple config bundles from the root dict at once, validating each one with the corresponding confdef stored in the ConfigManager. Args: - conf_defs: A list of config names to get. If dictionary is supplied, uses the values + bundle_names: A list of config bundle names to get. If dictionary is supplied, uses the values as ConfigDefinitions rather than looking up a stored one in the ConfigManager. Returns: - A dict of config dicts, with keys matching those passed in ``config_names``. + A dict of config dicts, with keys matching those passed in ``bundle_names``. """ config_values = {} - if isinstance(config_names, dict): - for name, conf_def in config_names.items(): - config_values[name] = self.validate_and_get_config(name, conf_def) + if isinstance(bundle_names, dict): + for name, conf_def in bundle_names.items(): + config_values[name] = self.get_config_bundle(name, conf_def) else: - for name in config_names: - config_values[name] = self.validate_and_get_config(name) + for name in bundle_names: + config_values[name] = self.get_config_bundle(name) return config_values - def get_config_names(self): + def get_bundle_names(self): """ - Returns a list of names of top level config tables + Returns a list of names of top level config bundles """ return list(self.root_config.keys())