New agent plugin structure

setup.py

@@ -32,7 +32,7 @@ setup(
     },
 
     entry_points={
-        'console_scripts': ['shepherd=shepherd.core:cli'],
+        'console_scripts': ['shepherd=shepherd.agent.core:cli'],
     },
     license='GPLv3+',
     description='Herd your mob of physically remote nodes',

shepherd/__init__.py

@@ -1 +1,5 @@
-from .agent.plugins import Plugin  # noqa
+from .agent.plugin import plugin  # noqa
+from .agent.plugin import PluginInterface # noqa
+from .agent.plugin import plugin_function # noqa
+from .agent.plugin import plugin_hook # noqa
+from .agent.plugin import plugin_attachment # noqa

shepherd/agent/core.py

@@ -66,15 +66,16 @@ def cli(ctx, default_config_path, local_operation, only_default_layer):
 
     confman = ConfigManager()
 
-    plugin_classes = compile_config_and_get_plugins(confman, default_config_path, layers_disabled)
+    compile_config(confman, default_config_path, layers_disabled)
+
     plugin_configs = confman.get_config_bundles()
-    core_config = confman.get_config_bundle("shepherd")
     del plugin_configs["shepherd"]
+    core_config = confman.get_config_bundle("shepherd")
 
     # control.init_control(core_config, plugin_configs)
 
     scheduler.init_scheduler(core_config)
-    plugin.init_plugins(plugin_classes, plugin_configs, core_config)
+    plugin.init_plugins(plugin_configs, core_config)
     # scheduler.restore_jobs()
 
     print(str(datetime.now()))
@@ -119,14 +120,14 @@ def template(ctx, plugin_name, include_all, config_path, plugin_dir):
     confspec = ConfigSpecification()
     if (not plugin_name) or (plugin_name == "shepherd"):
         plugin_name = "shepherd"
-        specify_core_config(confspec)
+        confspec = core_confspec()
     else:
         try:
-            plugin_class = plugin.find_plugins([plugin_name], plugin_dir)[plugin_name]
+            plugin_interface = plugin.load_plugin(plugin_name, plugin_dir)
         except plugin.PluginLoadError as e:
             log.error(e.args[0])
             sys.exit(1)
-        plugin_class.specify_config(confspec)
+        confspec = plugin_interface.confspec
 
     template_dict = confspec.get_template(include_all)
     template_toml = toml.dumps({plugin_name: template_dict})
@@ -169,22 +170,19 @@ def template(ctx, plugin_name, include_all, config_path, plugin_dir):
             f.write(template_toml)
 
 
-def compile_config_and_get_plugins(confman, default_config_path, layers_disabled):
+def compile_config(confman, default_config_path, layers_disabled):
     """
     Run through the process of assembling the various config layers, falling back to working
-    ones where necessary. Also gathers needed plugin classes in the process.
+    ones where necessary. As part of this, loads in the required plugins.
     """
 
-    # Create core confspec and populate it
+    confman.add_confspec("shepherd", core_confspec())
-    core_confspec = ConfigSpecification()
-    specify_core_config(core_confspec)
-    confman.add_confspec("shepherd", core_confspec)
 
     # ====Default Local Config Layer====
     # This must validate to continue.
     default_config_path = Path(default_config_path).expanduser()
     try:
-        plugin_classes = load_config_layer_and_plugins(confman, default_config_path)
+        load_config_layer_and_plugins(confman, default_config_path)
         log.info(F"Loaded default config layer from {default_config_path}")
     except Exception as e:
         if isinstance(e, InvalidConfigError):
@@ -209,7 +207,7 @@ def compile_config_and_get_plugins(confman, default_config_path, layers_disabled
     # If this fails, maintain default config but continue on to Control layer
     if "custom" not in layers_disabled:
         try:
-            plugin_classes = load_config_layer_and_plugins(confman, custom_config_path)
+            load_config_layer_and_plugins(confman, custom_config_path)
             log.info(F"Loaded custom config layer from {custom_config_path}")
         except Exception as e:
             if isinstance(e, InvalidConfigError):
@@ -239,7 +237,7 @@ def compile_config_and_get_plugins(confman, default_config_path, layers_disabled
         try:
             control_config = control.get_config(core_conf["root_dir"])
             try:
-                plugin_classes = load_config_layer_and_plugins(confman, control_config)
+                load_config_layer_and_plugins(confman, control_config)
                 log.info(F"Loaded cached Shepherd Control config layer")
             except Exception as e:
                 if isinstance(e, InvalidConfigError):
@@ -258,15 +256,13 @@ def compile_config_and_get_plugins(confman, default_config_path, layers_disabled
     log.debug("Compiled config: %s", confman.root_config)
     confman.dump_to_file(core_conf["generated_config_path"])
 
-    return plugin_classes
-
 
 # Relative pathnames here are all relative to "root_dir"
-def specify_core_config(confspec):
+def core_confspec():
     """
-    Defines the config specification by populating the ConfigSpecification passed in
+    Returns the core config specification
-    ``confspec`` - the same pattern plugins use
     """
+    confspec = ConfigSpecification()
     confspec.add_specs([
         ("name", StringSpec(helptext="Identifying name for this device")),
         ("hostname", StringSpec(default="", optional=True, helptext="If set, changes the system"
@@ -288,6 +284,8 @@ def specify_core_config(confspec):
     confspec.add_spec("control_server", StringSpec())
     confspec.add_spec("control_api_key", StringSpec())
 
+    return confspec
+
 
 def resolve_core_conf_paths(core_conf):
     """
@@ -303,7 +301,7 @@ def resolve_core_conf_paths(core_conf):
         Path(core_conf["generated_config_path"]).expanduser().resolve())
 
 
-def load_config_layer_and_plugins(confman, config_source):
+def load_config_layer_and_plugins(confman: ConfigManager, config_source):
     """
     Load a config layer, find the necessary plugin classes, then validate it.
     If this succeeds, the returned dict of plugin classes will directly match
@@ -321,13 +319,9 @@ def load_config_layer_and_plugins(confman, config_source):
     plugin_names.remove("shepherd")
 
     # Load plugins to get their config specifications
-    plugin_classes = plugin.find_plugins(plugin_names, plugin_dir)
+    plugin_interfaces = plugin.load_plugins(plugin_names, plugin_dir)
-    for plugin_name, plugin_class in plugin_classes.items():
+    for plugin_name, plugin_interface in plugin_interfaces.items():
-        new_conf_spec = ConfigSpecification()
+        confman.add_confspec(plugin_name, plugin_interface.confspec)
-        plugin_class.specify_config(new_conf_spec)
-        confman.add_confspec(plugin_name, new_conf_spec)
 
     # Validate all plugin configs
     confman.validate_bundles()
-
-    return plugin_classes

shepherd/agent/plugin.py

@@ -1,16 +1,9 @@
-#!/usr/bin/env python3
-
-import inspect
-from abc import ABC, abstractmethod
 import importlib
+import inspect
 import logging
-
-from types import SimpleNamespace
-from collections import namedtuple
 import sys
 import os
 
-import shepherd.scheduler
 log = logging.getLogger(__name__)
 
 
@@ -18,213 +11,37 @@ class PluginLoadError(Exception):
     pass
 
 
-class Hook():
+def plugin():
-    def __init__(self):
-        self.attached_functions = []
-
-    def attach(self, new_func):
-        if not callable(new_func):
-            raise TypeError("Argument to Hook.attach must be callable")
-        self.attached_functions.append(new_func)
-
-    def __call__(self, *args, **kwargs):
-        for func in self.attached_functions:
-            func(*args, **kwargs)
-
-
-class InterfaceFunction():
-    def __init__(self, func):
-        if not callable(func):
-            raise TypeError("Argument to InterfaceFunction must be callable")
-        self.func = func
-
-    def __call__(self, *args, **kwargs):
-        return self.func(*args, **kwargs)
-
-# TODO: Create tests to try and make a plugin subclass, and check exceptions to see if they
-# suggest the correct abstractmethods
-
-
-class Plugin(ABC):
-    @staticmethod
-    @abstractmethod
-    def specify_config(confspec):
     pass
 
-    @abstractmethod
-    def __init__(self, plugininterface, config):
-        pass
 
-    def run(self, hooks, plugins):
+def plugin_function():
     pass
 
 
-class SimplePlugin(Plugin):
+def plugin_hook():
-    @staticmethod
-    def specify_config(confspec):
     pass
 
-    def __init__(self, plugininterface, config):
-        super().__init__(plugininterface, config)
-        self.config = config
-        self.interface = plugininterface
-        self.plugins = plugininterface.other_plugins
-        self.hooks = plugininterface.hooks
-
-
-plugin_interfaces = {}  # dict of plugin interfaces
-
-# convenience dicts bundling together lists from interfaces
-plugin_functions = {}  # dict of plugins containing callable interface functions
-plugin_hooks = {}  # dict of plugins containing hook namespaces
-
-
-_defer = True
-_deferred_attachments = []
-_deferred_jobs = []
-
-
-def init_plugins(plugin_classes, plugin_configs, core_config):
-    # Startup pluginmanagers
-    global plugin_interfaces
-    global plugin_functions
-    global plugin_hooks
-
-    global _defer
-    global _deferred_attachments
-    global _deferred_jobs
-
-    for name, plugin_class in plugin_classes.items():
-        # Instanciate the plugin interface - this also instanciates
-        # the plugin
-        plugin_interfaces[name] = PluginInterface(
-            name, plugin_class, plugin_configs[name], core_config)
-        plugin_functions[name] = plugin_interfaces[name].functions
-        plugin_hooks[name] = plugin_interfaces[name].hooks
-
-    # interfaces and hooks should now be populated, attach hooks, schedule jobs
-    _defer = False
-    for attachment in _deferred_attachments:
-        _attach_hook(attachment)
-
-    for job_desc in _deferred_jobs:
-        _add_job(job_desc)
-
-    # Hand shared interface callables back out
-    for plugininterface in plugin_interfaces.values():
-        plugininterface.functions = plugin_functions
-
-
-def _add_job(job_desc):
-    global _deferred_jobs
-    global _defer
 
-    if not _defer:
+def plugin_attachment(hookname):
-        shepherd.scheduler.schedule_job(job_desc)
+    pass
-    else:
-        _deferred_jobs.append(job_desc)
-
-
-def _attach_hook(attachment):
-    global plugin_hooks
-    global _deferred_attachments
-    global _defer
-
-    if not _defer:
-        targetplugin_hooks = plugin_hooks.get(attachment.pluginname)
-        if targetplugin_hooks is not None:
-            targethook = getattr(targetplugin_hooks, attachment.hookname)
-            if targethook is not None:
-                targethook.attach(attachment.func)
-            else:
-                raise Exception("Could not find hook '" +
-                                attachment.hookname+"' in module '"+attachment.pluginname+"'")
-        else:
-            raise Exception(
-                "Cannot attach hook to non-existing module '"+attachment.pluginname+"'")
-    else:
-        _deferred_attachments.append(attachment)
-
-# Eventually, would like to be able to have client plugin simply:
-# self.shepherd.add_job(trigger, self.interface.myfunc)
-# self.shepherd.attach_hook(pluginnanme,hookname, callable)
-# self.shepherd.addinterface
-# self.shepherd.hooks.myhook()
-# self.shepherd.plugins.otherplugin.otherinterface()
-
-# self.shepherd.add_job()
-
-# Would be good to be able to use abstract methods to enable simpler plugin config
-# defs. A way to avoid instantiating the class would be to run it all as class methods,
-# enabling
-
-
-HookAttachment = namedtuple(
-    'HookAttachment', ['pluginname', 'hookname', 'func'])
 
 
 class PluginInterface():
-    '''
+    def __init__(self):
-    Class to handle the management of a single plugin.
+        self._confspec = None
-    All interaction to or from the plugin to other Shepherd components or
-    plugins should go through here.
-    '''
-
-    def __init__(self, pluginname, pluginclass, pluginconfig, coreconfig):
-        if not issubclass(pluginclass, Plugin):
-            raise TypeError(
-                "Argument must be subclass of shepherd.plugin.Plugin")
-
-        self.coreconfig = coreconfig
-
-        self.hooks = SimpleNamespace()  # My hooks
-        self.functions = SimpleNamespace()  # My callable interface functions
-
-        self._name = pluginname
-        self._plugin = pluginclass(self, pluginconfig)
-
-    def register_hook(self, name):
-        setattr(self.hooks, name, Hook())
-
-    def register_function(self, func):
-        setattr(self.functions, func.__name__, InterfaceFunction(func))
 
     @property
-    def other_plugins(self):
+    def confspec(self):
-        global plugin_functions
+        return self._confspec
-        return plugin_functions
-
-    def attach_hook(self, pluginname, hookname, func):
-        _attach_hook(HookAttachment(pluginname, hookname, func))
 
-    # Add a job to the scheduler. By default each will be identified by the interface
-    # callable name, and so adding another job with the same callable will oevrride the first.
-    # Use the optional job_name to differentiate jobs with an extra string
 
-    def add_job(self, func, hour, minute, second=0, job_name=""):
+def find_plugins(plugin_names, plugin_dir=None):
-        for function_name, function in self.functions.__dict__.items():
+    """
-            if func == function.func:
-                # jobstring should canonically describe this job, to be retrieved
-                # after reboot later. Of the form:
-                # shepherd:pluginname:functionname:jobname
-                jobstring = "shepherd:"+self._name+":"+function_name+":"+job_name
-                _add_job(shepherd.scheduler.JobDescription(jobstring, hour, minute, second))
-                break
-        else:
-            raise Exception(
-                "Could not add job. Callable must first be registered as "
-                "a plugin interface with PluginInterface.register_function()")
 
 
-"""
-An interface to a Shepherd module, accessible by other modules.
-All public methods in a module interface need to be threadsafe, as they will
-be called by other modules (which generally run in a seperate thread)
-"""
 
 
-def find_plugins(plugin_names, plugin_dir=None):
-    """
     Looks for the list of plugin names supplied and returns their classes.
     Will first try for plugin modules and packages locally located in ``shepherd.plugins``,
     then for modules and packages prefixed ``shepherd_`` located in the supplied ``plugin_dir``