From 9a42121007f844f38e12c8d7fadd4845a93277f8 Mon Sep 17 00:00:00 2001 From: Thomas Wilson Date: Thu, 21 May 2020 15:17:50 +0800 Subject: [PATCH] Implement core as plugin interface --- shepherd/agent/control.py | 7 ++-- shepherd/agent/core.py | 86 ++++++++++++++++++++++++--------------- shepherd/agent/plugin.py | 37 +++++++++++------ tests/test_control.py | 14 ++++++- tests/test_core.py | 8 ++++ 5 files changed, 103 insertions(+), 49 deletions(-) diff --git a/shepherd/agent/control.py b/shepherd/agent/control.py index 80a4ec8..0b9733e 100644 --- a/shepherd/agent/control.py +++ b/shepherd/agent/control.py @@ -30,15 +30,15 @@ def _update_required_callback(): _control_update_required.notify() -def control_confspec(): +def register(core_interface): """ - Returns the control config specification + Register the control confspec on the core interface. """ confspec = ConfigSpecification() confspec.add_spec("server", StringSpec()) confspec.add_spec("intro_key", StringSpec()) - return confspec + core_interface.confspec.add_spec("control", confspec, optional=True) class CoreUpdateState(): @@ -46,6 +46,7 @@ class CoreUpdateState(): A container for all state that might need communicating remotely to Control. Abstracts the Statesman topics away from other parts of the Agent. """ + def __init__(self, cmd_reader, cmd_result_writer): """ Control update handler for the `/update` core endpoint. Needs a reference to the CommandRunner diff --git a/shepherd/agent/core.py b/shepherd/agent/core.py index 3676e3e..5eaa8a2 100644 --- a/shepherd/agent/core.py +++ b/shepherd/agent/core.py @@ -30,12 +30,20 @@ class Agent(): self.applied_config = None # Split the applied_config up into core and plugins self.core_config = None - self.plugin_configs = None self.interface_functions = None self.control_enabled = control_enabled + self.core_interface = plugin.PluginInterface() + self.plugin_interfaces = None + + def root_dir(self): + return self.core_config["root_dir"] + + def device_name(self): + return self.core_config["name"] + def load(self, default_config_path, use_custom_config=True, new_device_mode=False): """ Load in the Shepherd Agent config and associated plugins. @@ -48,18 +56,45 @@ class Agent(): of ID, as if it were being run on a fresh system. """ + # Setup core interface + self.core_interface.register_confspec(core_confspec()) + self.core_interface.register_function(self.root_dir) + self.core_interface.register_function(self.device_name) + # Allows plugins to add delay for system time to stabilise + self.core_interface.register_hook("wait_for_stable_time") + + # Allow other modules to add to the core interface (confspec, hooks, interface functions) + # Having modules modify a confspec after it's registered here is a bit of a hack. + tasks.register(self.core_interface) + control.register(self.core_interface) + + # Because the plugin module caches interfaces, this will then get used when loading + # config layers and validating them + plugin.load_plugin_interface("shepherd", self.core_interface) + # Compile the config layers + confman = ConfigManager() + # Pre-seed confman with core confspec to bootstrap 'plugin_dir'. This is required even + # though 'load_config_layer_and_plugins()' will get the core confspec from the cached + # interface, as it needs 'plugin_dir' to list the plugins to load first. + confman.add_confspec("shepherd", self.core_interface._confspec) + compile_local_config(confman, default_config_path, use_custom_config) self.local_config = deepcopy(confman.get_config_bundles()) - # Check for new device mode - core_conf = confman.get_config_bundle('shepherd') + local_core_conf = confman.get_config_bundle('shepherd') - if new_device_mode or check_new_device_file(core_conf["custom_config_path"]): + # Check for new device mode + if new_device_mode or check_new_device_file(local_core_conf["custom_config_path"]): log.info("'new device' mode enabled, clearing old state...") - control.generate_device_identity(core_conf["root_dir"]) - control.clear_cached_config(core_conf["root_dir"]) + control.generate_device_identity(local_core_conf["root_dir"]) + control.clear_cached_config(local_core_conf["root_dir"]) + + if local_core_conf["control"] is None: + self.control_enabled = False + log.warning("Shepherd control config section not present. Will not attempt to" + " connect to Shepherd Control server.") if self.control_enabled: compile_remote_config(confman) @@ -67,21 +102,23 @@ class Agent(): log.info("Shepherd Control config layer disabled") self.applied_config = confman.get_config_bundles() - self.plugin_configs = confman.get_config_bundles() - self.core_config = self.plugin_configs.pop('shepherd') + self.core_config = confman.get_config_bundle('shepherd') log.debug("Compiled config: %s", confman.root_config) - if core_conf["compiled_config_path"]: + if self.core_config["compiled_config_path"]: message = F"Compiled Shepherd config at {datetime.now()}" - confman.dump_to_file(core_conf["compiled_config_path"], message=message) - log.info(F"Saved compiled config to {core_conf['compiled_config_path']}") + confman.dump_to_file(self.core_config["compiled_config_path"], message=message) + log.info(F"Saved compiled config to {self.core_config['compiled_config_path']}") + + def restart(self): + pass def start(self): # After this point, plugins may already have their own threads running if they create # them during init - plugin_interfaces, self.interface_functions = plugin.init_plugins( - self.plugin_configs, self.core_config, {}) + self.plugin_interfaces = plugin.init_plugins(self.applied_config) + self.interface_functions = self.core_interface.plugins cmd_runner = control.CommandRunner(self.interface_functions) core_update_state = control.CoreUpdateState(cmd_runner.cmd_reader, @@ -89,22 +126,11 @@ class Agent(): core_update_state.set_static_state(self.local_config, self.applied_config, core_confspec()) plugin_update_states = {name: iface._update_state - for name, iface in plugin_interfaces.items()} + for name, iface in self.plugin_interfaces.items()} if self.control_enabled: - if self.core_config["control"] is not None: - control.start_control(self.core_config["control"], self.core_config["root_dir"], + control.start_control(self.core_config["control"], self.root_dir(), core_update_state, plugin_update_states) - else: - log.warning("Shepherd control config section not present. Will not attempt to" - " connect to Shepherd Control server.") - - # Shift Control check to when it actually tries to find config, and just set - # control_enabled to false. - # Does all the other cmd_runner and update state stuff need to happen still if - # control is disabled? - # Should Core really need to know anything about the command runner, or should it - # just hand the interface functions in directly to Control? # Need somewhere to eventually pass in the hooks Tasks will need for the lowpower stuff, # probably just another init_plugins arg. @@ -139,8 +165,6 @@ def compile_local_config(confman, default_config_path, use_custom_config): As part of this, load the required plugins into cache (required to validate their config). """ - confman.add_confspec("shepherd", core_confspec()) - # ====Default Local Config Layer==== # This must validate to continue. default_config_path = Path(default_config_path).expanduser().resolve() @@ -202,8 +226,8 @@ def compile_remote_config(confman): # ====Control Remote Config Layer==== # Freeze Shepherd Control related config. - confman.freeze_value("shepherd", "control_server") - confman.freeze_value("shepherd", "control_api_key") + confman.freeze_value("shepherd", "control", "server") + confman.freeze_value("shepherd", "control", "intro_key") # Save current good local config confman.save_fallback() @@ -255,7 +279,6 @@ def core_confspec(): "./shepherd-plugins") }) - confspec.add_spec("control", control.control_confspec(), optional=True) return confspec @@ -300,7 +323,6 @@ def load_config_layer_and_plugins(confman: ConfigManager, config_source): # List other bundle names to get plugins we need to load plugin_names = confman.get_bundle_names() - plugin_names.remove("shepherd") # Load plugins to get their config specifications plugin_interfaces = {name: plugin.load_plugin(name, plugin_dir) for name in plugin_names} diff --git a/shepherd/agent/plugin.py b/shepherd/agent/plugin.py index 7827a39..ae29f9f 100644 --- a/shepherd/agent/plugin.py +++ b/shepherd/agent/plugin.py @@ -616,26 +616,41 @@ def load_plugin(plugin_name, plugin_dir=None): F" than one PluginInterface instance.") _, interface = interface_list[0] + + load_plugin_interface(plugin_name, interface, module) + return interface + + +def load_plugin_interface(plugin_name, interface, module=None): + """ + Load the plugin interface provided and add it to the plugin cache. If a module is provided or + the interface has a plugin class registered to it, scan them for plugin load markers and + perform the appropriate registrations on the interface. + + Usually called by `load_plugin()`, but allows a PluginInterface to be loaded directly, rather + than searching for it. + """ + interface._plugin_name = plugin_name # Only looks for implicit confspec if one isn't registered. Uses a blank one if none are # supplied. - if interface._confspec is None: - + if interface._confspec is None and module is not None: confspec_list = inspect.getmembers(module, is_instance_check(ConfigSpecification)) if confspec_list: if len(confspec_list) > 1: log.warning(F"Plugin {interface._plugin_name} has more" F" than one root ConfigSpecification instance.") interface.register_confspec(confspec_list[0][1]) - else: + + if interface._confspec is None: interface._confspec = ConfigSpecification() interface._update_state.set_confspec(interface.confspec) + if module is not None: # Scan module for load markers left by decorators and pass them over to register methods - for key, attr in module.__dict__.items(): if hasattr(attr, "_shepherd_load_marker"): if isinstance(attr._shepherd_load_marker, FunctionMarker): @@ -676,21 +691,22 @@ def load_plugin(plugin_name, plugin_dir=None): interface._loaded = True + # Add plugin interface to the cache _loaded_plugins[plugin_name] = interface return interface -def init_plugins(plugin_configs, core_config, core_interface_functions): +def init_plugins(plugin_configs, plugin_dir=None): """ - Initialise plugins named as keys in plugin_configs. Plugins must already be loaded. + Loads and initialise plugins named as keys in plugin_configs. Returns dict of initialised plugin interfaces, and a dict of interface function namedtuples (one for each plugin) """ - # Pick out plugins to init + # Pick out plugins to load (should already be loaded in cache) plugin_interfaces = {} for plugin_name in plugin_configs.keys(): - plugin_interfaces[plugin_name] = _loaded_plugins[plugin_name] + plugin_interfaces[plugin_name] = load_plugin(plugin_name, plugin_dir) interface_functions = {} interface_functions_proxy = MappingProxyType(interface_functions) @@ -731,13 +747,10 @@ def init_plugins(plugin_configs, core_config, core_interface_functions): # TODO We've run the object __init__, but not the init hook, that needs to be done interface._initialised = True - # Add core functions - interface_functions['shepherd'] = NamespaceProxy(core_interface_functions) - # Wait until all plugins have run their init before filling in and giving access # to all the interface functions. for plugin_name, interface in plugin_interfaces.items(): # Each plugin has a NamespaceProxy of it's interface functions for read-only attr access interface_functions[plugin_name] = NamespaceProxy(interface._functions) - return plugin_interfaces, interface_functions_proxy + return plugin_interfaces diff --git a/tests/test_control.py b/tests/test_control.py index 4a03bfb..2aa46ac 100644 --- a/tests/test_control.py +++ b/tests/test_control.py @@ -9,6 +9,8 @@ import responses import statesman from collections import namedtuple +from configspec import ConfigSpecification + from shepherd.agent import control from shepherd.agent import plugin @@ -35,8 +37,16 @@ def control_config(): return {'server': 'api.shepherd.test', 'intro_key': 'abcdefabcdefabcdef'} -def test_config(control_config): - control.control_confspec().validate(control_config) +@pytest.fixture +def registered_interface(): + interface = plugin.PluginInterface() + interface.register_confspec(ConfigSpecification()) + control.register(interface) + return interface + + +def test_config(control_config, registered_interface): + registered_interface.confspec.validate({'control': control_config}) def test_url(): diff --git a/tests/test_core.py b/tests/test_core.py index 3007d0b..ea3cefd 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -92,3 +92,11 @@ def test_local_agent_plugin_start(local_agent, plugin_config): local_agent.start() assert local_agent.interface_functions["classtestplugin"].instance_method( 3) == "instance method 3" + + +def test_core_interface(local_agent, plugin_config): + local_agent.load(plugin_config) + local_agent.start() + plugin_interface = local_agent.plugin_interfaces["classtestplugin"] + assert plugin_interface.plugins["shepherd"].device_name() == "shepherd-test" + assert plugin_interface.plugins["shepherd"].root_dir() == str(plugin_config.parent)