Implement core as plugin interface

shepherd/agent/control.py

@@ -30,15 +30,15 @@ def _update_required_callback():
         _control_update_required.notify()
 
 
-def control_confspec():
+def register(core_interface):
     """
-    Returns the control config specification
+    Register the control confspec on the core interface.
     """
     confspec = ConfigSpecification()
     confspec.add_spec("server", StringSpec())
     confspec.add_spec("intro_key", StringSpec())
 
-    return confspec
+    core_interface.confspec.add_spec("control", confspec, optional=True)
 
 
 class CoreUpdateState():
@@ -46,6 +46,7 @@ class CoreUpdateState():
     A container for all state that might need communicating remotely to Control. Abstracts the
     Statesman topics away from other parts of the Agent.
     """
+
     def __init__(self, cmd_reader, cmd_result_writer):
         """
         Control update handler for the `/update` core endpoint. Needs a reference to the CommandRunner

shepherd/agent/core.py

@@ -30,12 +30,20 @@ class Agent():
         self.applied_config = None
         # Split the applied_config up into core and plugins
         self.core_config = None
-        self.plugin_configs = None
 
         self.interface_functions = None
 
         self.control_enabled = control_enabled
 
+        self.core_interface = plugin.PluginInterface()
+        self.plugin_interfaces = None
+
+    def root_dir(self):
+        return self.core_config["root_dir"]
+
+    def device_name(self):
+        return self.core_config["name"]
+
     def load(self, default_config_path, use_custom_config=True,  new_device_mode=False):
         """
         Load in the Shepherd Agent config and associated plugins.
@@ -48,18 +56,45 @@ class Agent():
                 of ID, as if it were being run on a fresh system.
         """
 
+        # Setup core interface
+        self.core_interface.register_confspec(core_confspec())
+        self.core_interface.register_function(self.root_dir)
+        self.core_interface.register_function(self.device_name)
+        # Allows plugins to add delay for system time to stabilise
+        self.core_interface.register_hook("wait_for_stable_time")
+
+        # Allow other modules to add to the core interface (confspec, hooks, interface functions)
+        # Having modules modify a confspec after it's registered here is a bit of a hack.
+        tasks.register(self.core_interface)
+        control.register(self.core_interface)
+        
+        # Because the plugin module caches interfaces, this will then get used when loading
+        # config layers and validating them
+        plugin.load_plugin_interface("shepherd", self.core_interface)
+
         # Compile the config layers
+
         confman = ConfigManager()
+        # Pre-seed confman with core confspec to bootstrap 'plugin_dir'. This is required even
+        # though 'load_config_layer_and_plugins()' will get the core confspec from the cached
+        # interface, as it needs 'plugin_dir' to list the plugins to load first.
+        confman.add_confspec("shepherd", self.core_interface._confspec)
+
         compile_local_config(confman, default_config_path, use_custom_config)
         self.local_config = deepcopy(confman.get_config_bundles())
 
-        # Check for new device mode
+        local_core_conf = confman.get_config_bundle('shepherd')
-        core_conf = confman.get_config_bundle('shepherd')
 
-        if new_device_mode or check_new_device_file(core_conf["custom_config_path"]):
+        # Check for new device mode
+        if new_device_mode or check_new_device_file(local_core_conf["custom_config_path"]):
             log.info("'new device' mode enabled, clearing old state...")
-            control.generate_device_identity(core_conf["root_dir"])
+            control.generate_device_identity(local_core_conf["root_dir"])
-            control.clear_cached_config(core_conf["root_dir"])
+            control.clear_cached_config(local_core_conf["root_dir"])
+
+        if local_core_conf["control"] is None:
+            self.control_enabled = False
+            log.warning("Shepherd control config section not present. Will not attempt to"
+                        " connect to Shepherd Control server.")
 
         if self.control_enabled:
             compile_remote_config(confman)
@@ -67,21 +102,23 @@ class Agent():
             log.info("Shepherd Control config layer disabled")
 
         self.applied_config = confman.get_config_bundles()
-        self.plugin_configs = confman.get_config_bundles()
+        self.core_config = confman.get_config_bundle('shepherd')
-        self.core_config = self.plugin_configs.pop('shepherd')
 
         log.debug("Compiled config: %s", confman.root_config)
-        if core_conf["compiled_config_path"]:
+        if self.core_config["compiled_config_path"]:
             message = F"Compiled Shepherd config at {datetime.now()}"
-            confman.dump_to_file(core_conf["compiled_config_path"], message=message)
+            confman.dump_to_file(self.core_config["compiled_config_path"], message=message)
-            log.info(F"Saved compiled config to {core_conf['compiled_config_path']}")
+            log.info(F"Saved compiled config to {self.core_config['compiled_config_path']}")
+
+    def restart(self):
+        pass
 
     def start(self):
 
         # After this point, plugins may already have their own threads running if they create
         # them during init
-        plugin_interfaces, self.interface_functions = plugin.init_plugins(
+        self.plugin_interfaces = plugin.init_plugins(self.applied_config)
-            self.plugin_configs, self.core_config, {})
+        self.interface_functions = self.core_interface.plugins
 
         cmd_runner = control.CommandRunner(self.interface_functions)
         core_update_state = control.CoreUpdateState(cmd_runner.cmd_reader,
@@ -89,22 +126,11 @@ class Agent():
         core_update_state.set_static_state(self.local_config, self.applied_config, core_confspec())
 
         plugin_update_states = {name: iface._update_state
-                                for name, iface in plugin_interfaces.items()}
+                                for name, iface in self.plugin_interfaces.items()}
 
         if self.control_enabled:
-            if self.core_config["control"] is not None:
+            control.start_control(self.core_config["control"], self.root_dir(),
-                control.start_control(self.core_config["control"], self.core_config["root_dir"],
                                       core_update_state, plugin_update_states)
-            else:
-                log.warning("Shepherd control config section not present. Will not attempt to"
-                            " connect to Shepherd Control server.")
-
-        # Shift Control check to when it actually tries to find config, and just set 
-        # control_enabled to false. 
-        # Does all the other cmd_runner and update state stuff need to happen still if
-        # control is disabled?
-        # Should Core really need to know anything about the command runner, or should it
-        # just hand the interface functions in directly to Control?
 
         # Need somewhere to eventually pass in the hooks Tasks will need for the lowpower stuff, 
         # probably just another init_plugins arg.
@@ -139,8 +165,6 @@ def compile_local_config(confman, default_config_path, use_custom_config):
     As part of this, load the required plugins into cache (required to validate their config).
     """
 
-    confman.add_confspec("shepherd", core_confspec())
-
     # ====Default Local Config Layer====
     # This must validate to continue.
     default_config_path = Path(default_config_path).expanduser().resolve()
@@ -202,8 +226,8 @@ def compile_remote_config(confman):
     # ====Control Remote Config Layer====
 
     # Freeze Shepherd Control related config.
-    confman.freeze_value("shepherd", "control_server")
+    confman.freeze_value("shepherd", "control", "server")
-    confman.freeze_value("shepherd", "control_api_key")
+    confman.freeze_value("shepherd", "control", "intro_key")
 
     # Save current good local config
     confman.save_fallback()
@@ -255,7 +279,6 @@ def core_confspec():
                 "./shepherd-plugins")
     })
 
-    confspec.add_spec("control", control.control_confspec(), optional=True)
     return confspec
 
 
@@ -300,7 +323,6 @@ def load_config_layer_and_plugins(confman: ConfigManager, config_source):
 
     # List other bundle names to get plugins we need to load
     plugin_names = confman.get_bundle_names()
-    plugin_names.remove("shepherd")
 
     # Load plugins to get their config specifications
     plugin_interfaces = {name: plugin.load_plugin(name, plugin_dir) for name in plugin_names}

shepherd/agent/plugin.py

@@ -616,26 +616,41 @@ def load_plugin(plugin_name, plugin_dir=None):
                     F" than one PluginInterface instance.")
 
     _, interface = interface_list[0]
+
+    load_plugin_interface(plugin_name, interface, module)
+    return interface
+
+
+def load_plugin_interface(plugin_name, interface, module=None):
+    """
+    Load the plugin interface provided and add it to the plugin cache. If a module is provided or
+    the interface has a plugin class registered to it, scan them for plugin load markers and
+    perform the appropriate registrations on the interface.
+
+    Usually called by `load_plugin()`, but allows a PluginInterface to be loaded directly, rather
+    than searching for it.
+    """
+
     interface._plugin_name = plugin_name
 
     # Only looks for implicit confspec if one isn't registered. Uses a blank one if none are
     # supplied.
 
-    if interface._confspec is None:
+    if interface._confspec is None and module is not None:
-
         confspec_list = inspect.getmembers(module, is_instance_check(ConfigSpecification))
         if confspec_list:
             if len(confspec_list) > 1:
                 log.warning(F"Plugin {interface._plugin_name} has more"
                             F" than one root ConfigSpecification instance.")
             interface.register_confspec(confspec_list[0][1])
-        else:
+
+    if interface._confspec is None:
             interface._confspec = ConfigSpecification()
 
     interface._update_state.set_confspec(interface.confspec)
 
+    if module is not None:
     # Scan module for load markers left by decorators and pass them over to register methods
-
     for key, attr in module.__dict__.items():
         if hasattr(attr, "_shepherd_load_marker"):
             if isinstance(attr._shepherd_load_marker, FunctionMarker):
@@ -676,21 +691,22 @@ def load_plugin(plugin_name, plugin_dir=None):
 
     interface._loaded = True
 
+    # Add plugin interface to the cache
     _loaded_plugins[plugin_name] = interface
     return interface
 
 
-def init_plugins(plugin_configs, core_config, core_interface_functions):
+def init_plugins(plugin_configs, plugin_dir=None):
     """
-    Initialise plugins named as keys in plugin_configs. Plugins must already be loaded.
+    Loads and initialise plugins named as keys in plugin_configs.
     Returns dict of initialised plugin interfaces, and a dict of interface function namedtuples
     (one for each plugin)
     """
 
-    # Pick out plugins to init
+    # Pick out plugins to load (should already be loaded in cache)
     plugin_interfaces = {}
     for plugin_name in plugin_configs.keys():
-        plugin_interfaces[plugin_name] = _loaded_plugins[plugin_name]
+        plugin_interfaces[plugin_name] = load_plugin(plugin_name, plugin_dir)
 
     interface_functions = {}
     interface_functions_proxy = MappingProxyType(interface_functions)
@@ -731,13 +747,10 @@ def init_plugins(plugin_configs, core_config, core_interface_functions):
         # TODO We've run the object __init__, but not the init hook, that needs to be done
         interface._initialised = True
 
-    # Add core functions
-    interface_functions['shepherd'] = NamespaceProxy(core_interface_functions)
-
     # Wait until all plugins have run their init before filling in and giving access
     # to all the interface functions.
     for plugin_name, interface in plugin_interfaces.items():
         # Each plugin has a NamespaceProxy of it's interface functions for read-only attr access
         interface_functions[plugin_name] = NamespaceProxy(interface._functions)
 
-    return plugin_interfaces, interface_functions_proxy
+    return plugin_interfaces

tests/test_control.py

@@ -9,6 +9,8 @@ import responses
 import statesman
 from collections import namedtuple
 
+from configspec import ConfigSpecification
+
 from shepherd.agent import control
 from shepherd.agent import plugin
 
@@ -35,8 +37,16 @@ def control_config():
     return {'server': 'api.shepherd.test', 'intro_key': 'abcdefabcdefabcdef'}
 
 
-def test_config(control_config):
+@pytest.fixture
-    control.control_confspec().validate(control_config)
+def registered_interface():
+    interface = plugin.PluginInterface()
+    interface.register_confspec(ConfigSpecification())
+    control.register(interface)
+    return interface
+
+
+def test_config(control_config, registered_interface):
+    registered_interface.confspec.validate({'control': control_config})
 
 
 def test_url():

tests/test_core.py

@@ -92,3 +92,11 @@ def test_local_agent_plugin_start(local_agent, plugin_config):
     local_agent.start()
     assert local_agent.interface_functions["classtestplugin"].instance_method(
         3) == "instance method 3"
+
+
+def test_core_interface(local_agent, plugin_config):
+    local_agent.load(plugin_config)
+    local_agent.start()
+    plugin_interface = local_agent.plugin_interfaces["classtestplugin"]
+    assert plugin_interface.plugins["shepherd"].device_name() == "shepherd-test"
+    assert plugin_interface.plugins["shepherd"].root_dir() == str(plugin_config.parent)