Distreon-Systems
/
shepherd-agent
7
0
Fork 0
Code Issues 8 Pull Requests Releases Wiki Activity

Update config to use Python terminology

Browse Source
fix-v0.2
Tom Wilson 6 years ago
parent 3e5117d9ff
commit 5f1d3be59f
1 changed files with 90 additions and 86 deletions
Show Stats Download Patch File Download Diff File

176
shepherd/config.py
Unescape View File

@ -1,66 +1,69 @@
""" """
Configuration managment module. Enables configuration to be validated against Configuration management module. Enables configuration to be validated against
requirement definitions before being loaded and used. requirement definitions before being loaded and used.
"""
Compatible with both raw config data structures and TOML files, config data must
start with a root dict containing named "config bundles". These are intended to
refer to different modular parts of the application needing configuration, and the
config data structure must contain at least one.
import re Each config bundle itself needs to have a dict at the root, and so in practice a minimal
import toml TOML config file would look like::
from .freezedry import freezedryable, rehydrate
[myapp]
config_thingy_a = "foooooo!"
important_number = 8237
This would resolve to a config bundle named "myapp" that results in the dict::
{"config_thingy_a": "foooooo!", "important_number": 8237}
class InvalidConfigError(Exception): Root items that are not dicts are not supported, for instance both the following TOML files would fail::
pass
# On start, [[myapp]]
# get conf_defs via imports in core important_number = 8237
# Load a conf file [[myapp]]
# Validate the Shepherd table in conf file, then load its values to get module list another_important_number = 2963
# Validate other loaded tables/modules, then load their values
# How do modules wind up with their instance of ConfigValues? Return val instance (root object in bundle is a list)
# from validation function - as it needs to build the vals instance while validating anyway
# Validate a conf file given a module or config_def list ::
root_thingy = 46
# idea is to create these similar to how arg parser works (root object in config is a single value)
"""
# how to store individual items in def? Dict of...
# Tables need a dict, lists/arrays need a list of dicts, but what's in the dict?
# if it's another instacnce of ConfigDef, then each Config Def needs to handle
# one item, but cater for all the different types of items - and many of those
# should be able to add a new item.
# could have a separate class for the root, but lower tables really need to import re
# perform exactly the same... import toml
from abc import ABC, abstractmethod
# config def required interface: from .freezedry import freezedryable, rehydrate
# Validate values.
# The Table and Array terms used here are directly from the TOML convention, but they essentially
# map directly to Dictionaries (Tables), and Lists (Arrays)
class InvalidConfigError(Exception):
pass
# The Table and Array terms from the TOML convention essentially
# map directly to Dictionaries (Tables), and Lists (Arrays)
class _ConfigDefinition(): class _ConfigDefinition(ABC):
def __init__(self, default=None, optional=False, helptext=""): def __init__(self, default=None, optional=False, helptext=""):
self.default = default self.default = default
self.optional = optional self.optional = optional
self.helptext = helptext self.helptext = helptext
def validate(self, value): # pylint: disable=W0613 @abstractmethod
raise TypeError("_ConfigDefinition.validate() is an abstract method") def validate(self, value):
pass
@freezedryable @freezedryable
class BoolDef(_ConfigDefinition): class BoolDef(_ConfigDefinition):
def __init__(self, default=None, optional=False, helptext=""): # pylint: disable=W0235 def __init__(self, default=None, optional=False, helptext=""):
super().__init__(default, optional, helptext) super().__init__(default, optional, helptext)
def validate(self, value): def validate(self, value):
@ -104,10 +107,10 @@ class StringDef(_ConfigDefinition):
str(self.maxlength)) str(self.maxlength))
@freezedryable @freezedryable
class TableDef(_ConfigDefinition): class DictDef(_ConfigDefinition):
def __init__(self, default=None, optional=False, helptext=""): def __init__(self, default=None, optional=False, helptext=""):
super().__init__(default, optional, helptext) super().__init__(default, optional, helptext)
self.def_table = {} self.def_dict = {}
def add_def(self, name, newdef): def add_def(self, name, newdef):
if not isinstance(newdef, _ConfigDefinition): if not isinstance(newdef, _ConfigDefinition):
@ -115,61 +118,62 @@ class TableDef(_ConfigDefinition):
"ConfigDefinition subclass") "ConfigDefinition subclass")
if not isinstance(name, str): if not isinstance(name, str):
raise TypeError("Config definition name must be a string") raise TypeError("Config definition name must be a string")
self.def_table[name] = newdef self.def_dict[name] = newdef
return newdef return newdef
def validate(self, value_table): # pylint: disable=W0221 def validate(self, value_dict): # pylint: disable=W0221
def_set = set(self.def_table.keys()) def_set = set(self.def_dict.keys())
value_set = set(value_table.keys()) value_set = set(value_dict.keys())
for missing_key in def_set - value_set: for missing_key in def_set - value_set:
if not self.def_table[missing_key].optional: if not self.def_dict[missing_key].optional:
raise InvalidConfigError("Table must contain key: " + raise InvalidConfigError("Dict must contain key: " +
missing_key) missing_key)
else: else:
value_table[missing_key] = self.def_table[missing_key].default value_dict[missing_key] = self.def_dict[missing_key].default
for extra_key in value_set - def_set: for extra_key in value_set - def_set:
raise InvalidConfigError("Table contains unknown key: " + raise InvalidConfigError("Dict contains unknown key: " +
extra_key) extra_key)
for key, value in value_table.items(): for key, value in value_dict.items():
try: try:
self.def_table[key].validate(value) self.def_dict[key].validate(value)
except InvalidConfigError as e: except InvalidConfigError as e:
e.args = ("Key: " + key,) + e.args e.args = ("Key: " + key,) + e.args
raise raise
class _ArrayDefMixin():
def validate(self, value_array): class _ListDefMixin():
if not isinstance(value_array, list): def validate(self, value_list):
raise InvalidConfigError("Config item must be an array") if not isinstance(value_list, list):
for index, value in enumerate(value_array): raise InvalidConfigError("Config item must be a list")
for index, value in enumerate(value_list):
try: try:
super().validate(value) super().validate(value)
except InvalidConfigError as e: except InvalidConfigError as e:
e.args = ("Array index: " + str(index),) + e.args e.args = ("List index: " + str(index),) + e.args
raise raise
@freezedryable @freezedryable
class BoolArrayDef(_ArrayDefMixin, BoolDef): class BoolListDef(_ListDefMixin, BoolDef):
pass pass
@freezedryable @freezedryable
class IntArrayDef(_ArrayDefMixin, IntDef): class IntListDef(_ListDefMixin, IntDef):
pass pass
@freezedryable @freezedryable
class StringArrayDef(_ArrayDefMixin, StringDef): class StringListDef(_ListDefMixin, StringDef):
pass pass
@freezedryable @freezedryable
class TableArrayDef(_ArrayDefMixin, TableDef): class DictListDef(_ListDefMixin, DictDef):
pass pass
@freezedryable @freezedryable
class ConfDefinition(TableDef): class ConfDefinition(DictDef):
pass pass
@ -202,8 +206,8 @@ class ConfigManager():
value and completely replaced. value and completely replaced.
Args: Args:
source: Either a dict config to load directly, a filepath to a TOML file, source: Either the root dict of a data structure to load directly, a filepath to a TOML file,
or an open file. or an open TOML file.
""" """
if isinstance(source, dict): # load from dict if isinstance(source, dict): # load from dict
@ -216,28 +220,28 @@ class ConfigManager():
self._overlay(new_source, self.root_config) self._overlay(new_source, self.root_config)
def add_confdef(self, name, confdef): def add_confdef(self, bundle_name, confdef):
""" """
Stores a ConfigDefinition for future use when validating the corresponding config table Stores a ConfigDefinition for future use when validating the corresponding config bundle
Args: Args:
name (str) : Then name to store the config definition under. bundle_name (str) : The name to store the config definition under.
confdef (ConfigDefinition): The populated ConfigDefinition to store. confdef (ConfigDefinition): The populated ConfigDefinition to store.
""" """
self.confdefs[name]=confdef self.confdefs[bundle_name]=confdef
def add_confdefs(self, confdefs): def add_confdefs(self, confdefs):
""" """
Stores multiple ConfigDefinitions at once for future use when validating the corresponding config tables Stores multiple ConfigDefinitions at once for future use when validating the corresponding config bundles
Args: Args:
confdefs : A dict of populated ConfigDefinitions to store, using their keys as names. confdefs : A dict of populated ConfigDefinitions to store, using their keys as names.
""" """
self.confdefs.update(confdefs) self.confdefs.update(confdefs)
def get_missing_confdefs(self): def list_missing_confdefs(self):
""" """
Returns a list of config table names that do not have a corresponding ConfigDefinition Returns a list of config bundle names that do not have a corresponding ConfigDefinition
stored in the ConfigManager. stored in the ConfigManager.
""" """
return list(self.root_config.keys() - self.confdefs.keys()) return list(self.root_config.keys() - self.confdefs.keys())
@ -252,57 +256,57 @@ class ConfigManager():
# Otherwise it's either an existing value to be replaced or needs to be added. # Otherwise it's either an existing value to be replaced or needs to be added.
dest[key] = src[key] dest[key] = src[key]
def validate_and_get_config(self, config_name, conf_def=None): def get_config_bundle(self, bundle_name, conf_def=None):
""" """
Get a config dict called ``table_name`` and validate Get a config bundle called ``bundle_name`` and validate
it against the corresponding config definition stored in the ConfigManager. it against the corresponding config definition stored in the ConfigManager.
If ``conf_def`` is supplied, it gets used instead. Returns a validated If ``conf_def`` is supplied, it gets used instead. Returns a validated
config dictionary. config bundle dict.
Note that as part of validation, optional keys that are missing will be Note that as part of validation, optional keys that are missing will be
filled in with their default values (see ``TableDef``). filled in with their default values (see ``DictDef``).
Args: Args:
table_name: (str) Name of the config dict to find. config_name: (str) Name of the config dict to find.
conf_def: (ConfDefinition) Optional config definition to validate against. conf_def: (ConfDefinition) Optional config definition to validate against.
""" """
if not isinstance(conf_def, ConfDefinition): if not isinstance(conf_def, ConfDefinition):
conf_def = self.confdefs[config_name] conf_def = self.confdefs[bundle_name]
if config_name not in self.root_config: if bundle_name not in self.root_config:
raise InvalidConfigError( raise InvalidConfigError(
"Config must contain table: " + config_name) "Config must contain dict: " + bundle_name)
try: try:
conf_def.validate(self.root_config[config_name]) conf_def.validate(self.root_config[bundle_name])
except InvalidConfigError as e: except InvalidConfigError as e:
e.args = ("Module: " + config_name,) + e.args e.args = ("Module: " + bundle_name,) + e.args
raise raise
return self.root_config[config_name] return self.root_config[bundle_name]
def validate_and_get_configs(self, config_names): def get_config_bundles(self, bundle_names):
""" """
Get multiple configs from the root table at once, validating each one with the Get multiple config bundles from the root dict at once, validating each one with the
corresponding confdef stored in the ConfigManager. corresponding confdef stored in the ConfigManager.
Args: Args:
conf_defs: A list of config names to get. If dictionary is supplied, uses the values bundle_names: A list of config bundle names to get. If dictionary is supplied, uses the values
as ConfigDefinitions rather than looking up a stored one in the ConfigManager. as ConfigDefinitions rather than looking up a stored one in the ConfigManager.
Returns: Returns:
A dict of config dicts, with keys matching those passed in ``config_names``. A dict of config dicts, with keys matching those passed in ``bundle_names``.
""" """
config_values = {} config_values = {}
if isinstance(config_names, dict): if isinstance(bundle_names, dict):
for name, conf_def in config_names.items(): for name, conf_def in bundle_names.items():
config_values[name] = self.validate_and_get_config(name, conf_def) config_values[name] = self.get_config_bundle(name, conf_def)
else: else:
for name in config_names: for name in bundle_names:
config_values[name] = self.validate_and_get_config(name) config_values[name] = self.get_config_bundle(name)
return config_values return config_values
def get_config_names(self): def get_bundle_names(self):
""" """
Returns a list of names of top level config tables Returns a list of names of top level config bundles
""" """
return list(self.root_config.keys()) return list(self.root_config.keys())

Write Preview
Loading…
Cancel
Save