diff --git a/setup.py b/setup.py index 4332161..f7f50e9 100644 --- a/setup.py +++ b/setup.py @@ -32,7 +32,7 @@ setup( }, entry_points={ - 'console_scripts': ['shepherd=shepherd.core:cli'], + 'console_scripts': ['shepherd=shepherd.agent.core:cli'], }, license='GPLv3+', description='Herd your mob of physically remote nodes', diff --git a/shepherd/__init__.py b/shepherd/__init__.py index c8b2664..1165b13 100644 --- a/shepherd/__init__.py +++ b/shepherd/__init__.py @@ -1 +1,5 @@ -from .agent.plugins import Plugin # noqa +from .agent.plugin import plugin # noqa +from .agent.plugin import PluginInterface # noqa +from .agent.plugin import plugin_function # noqa +from .agent.plugin import plugin_hook # noqa +from .agent.plugin import plugin_attachment # noqa diff --git a/shepherd/agent/__init__.py b/shepherd/agent/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/shepherd/agent/core.py b/shepherd/agent/core.py index 0acd3b7..73179e8 100644 --- a/shepherd/agent/core.py +++ b/shepherd/agent/core.py @@ -66,15 +66,16 @@ def cli(ctx, default_config_path, local_operation, only_default_layer): confman = ConfigManager() - plugin_classes = compile_config_and_get_plugins(confman, default_config_path, layers_disabled) + compile_config(confman, default_config_path, layers_disabled) + plugin_configs = confman.get_config_bundles() - core_config = confman.get_config_bundle("shepherd") del plugin_configs["shepherd"] + core_config = confman.get_config_bundle("shepherd") # control.init_control(core_config, plugin_configs) scheduler.init_scheduler(core_config) - plugin.init_plugins(plugin_classes, plugin_configs, core_config) + plugin.init_plugins(plugin_configs, core_config) # scheduler.restore_jobs() print(str(datetime.now())) @@ -119,14 +120,14 @@ def template(ctx, plugin_name, include_all, config_path, plugin_dir): confspec = ConfigSpecification() if (not plugin_name) or (plugin_name == "shepherd"): plugin_name = "shepherd" - specify_core_config(confspec) + confspec = core_confspec() else: try: - plugin_class = plugin.find_plugins([plugin_name], plugin_dir)[plugin_name] + plugin_interface = plugin.load_plugin(plugin_name, plugin_dir) except plugin.PluginLoadError as e: log.error(e.args[0]) sys.exit(1) - plugin_class.specify_config(confspec) + confspec = plugin_interface.confspec template_dict = confspec.get_template(include_all) template_toml = toml.dumps({plugin_name: template_dict}) @@ -169,22 +170,19 @@ def template(ctx, plugin_name, include_all, config_path, plugin_dir): f.write(template_toml) -def compile_config_and_get_plugins(confman, default_config_path, layers_disabled): +def compile_config(confman, default_config_path, layers_disabled): """ Run through the process of assembling the various config layers, falling back to working - ones where necessary. Also gathers needed plugin classes in the process. + ones where necessary. As part of this, loads in the required plugins. """ - # Create core confspec and populate it - core_confspec = ConfigSpecification() - specify_core_config(core_confspec) - confman.add_confspec("shepherd", core_confspec) + confman.add_confspec("shepherd", core_confspec()) # ====Default Local Config Layer==== # This must validate to continue. default_config_path = Path(default_config_path).expanduser() try: - plugin_classes = load_config_layer_and_plugins(confman, default_config_path) + load_config_layer_and_plugins(confman, default_config_path) log.info(F"Loaded default config layer from {default_config_path}") except Exception as e: if isinstance(e, InvalidConfigError): @@ -209,7 +207,7 @@ def compile_config_and_get_plugins(confman, default_config_path, layers_disabled # If this fails, maintain default config but continue on to Control layer if "custom" not in layers_disabled: try: - plugin_classes = load_config_layer_and_plugins(confman, custom_config_path) + load_config_layer_and_plugins(confman, custom_config_path) log.info(F"Loaded custom config layer from {custom_config_path}") except Exception as e: if isinstance(e, InvalidConfigError): @@ -239,7 +237,7 @@ def compile_config_and_get_plugins(confman, default_config_path, layers_disabled try: control_config = control.get_config(core_conf["root_dir"]) try: - plugin_classes = load_config_layer_and_plugins(confman, control_config) + load_config_layer_and_plugins(confman, control_config) log.info(F"Loaded cached Shepherd Control config layer") except Exception as e: if isinstance(e, InvalidConfigError): @@ -258,15 +256,13 @@ def compile_config_and_get_plugins(confman, default_config_path, layers_disabled log.debug("Compiled config: %s", confman.root_config) confman.dump_to_file(core_conf["generated_config_path"]) - return plugin_classes - # Relative pathnames here are all relative to "root_dir" -def specify_core_config(confspec): +def core_confspec(): """ - Defines the config specification by populating the ConfigSpecification passed in - ``confspec`` - the same pattern plugins use + Returns the core config specification """ + confspec = ConfigSpecification() confspec.add_specs([ ("name", StringSpec(helptext="Identifying name for this device")), ("hostname", StringSpec(default="", optional=True, helptext="If set, changes the system" @@ -288,6 +284,8 @@ def specify_core_config(confspec): confspec.add_spec("control_server", StringSpec()) confspec.add_spec("control_api_key", StringSpec()) + return confspec + def resolve_core_conf_paths(core_conf): """ @@ -303,7 +301,7 @@ def resolve_core_conf_paths(core_conf): Path(core_conf["generated_config_path"]).expanduser().resolve()) -def load_config_layer_and_plugins(confman, config_source): +def load_config_layer_and_plugins(confman: ConfigManager, config_source): """ Load a config layer, find the necessary plugin classes, then validate it. If this succeeds, the returned dict of plugin classes will directly match @@ -321,13 +319,9 @@ def load_config_layer_and_plugins(confman, config_source): plugin_names.remove("shepherd") # Load plugins to get their config specifications - plugin_classes = plugin.find_plugins(plugin_names, plugin_dir) - for plugin_name, plugin_class in plugin_classes.items(): - new_conf_spec = ConfigSpecification() - plugin_class.specify_config(new_conf_spec) - confman.add_confspec(plugin_name, new_conf_spec) + plugin_interfaces = plugin.load_plugins(plugin_names, plugin_dir) + for plugin_name, plugin_interface in plugin_interfaces.items(): + confman.add_confspec(plugin_name, plugin_interface.confspec) # Validate all plugin configs confman.validate_bundles() - - return plugin_classes diff --git a/shepherd/agent/plugin.py b/shepherd/agent/plugin.py index 627bc90..ed7988c 100644 --- a/shepherd/agent/plugin.py +++ b/shepherd/agent/plugin.py @@ -1,16 +1,9 @@ -#!/usr/bin/env python3 - -import inspect -from abc import ABC, abstractmethod import importlib +import inspect import logging - -from types import SimpleNamespace -from collections import namedtuple import sys import os -import shepherd.scheduler log = logging.getLogger(__name__) @@ -18,213 +11,37 @@ class PluginLoadError(Exception): pass -class Hook(): - def __init__(self): - self.attached_functions = [] - - def attach(self, new_func): - if not callable(new_func): - raise TypeError("Argument to Hook.attach must be callable") - self.attached_functions.append(new_func) - - def __call__(self, *args, **kwargs): - for func in self.attached_functions: - func(*args, **kwargs) - - -class InterfaceFunction(): - def __init__(self, func): - if not callable(func): - raise TypeError("Argument to InterfaceFunction must be callable") - self.func = func - - def __call__(self, *args, **kwargs): - return self.func(*args, **kwargs) - -# TODO: Create tests to try and make a plugin subclass, and check exceptions to see if they -# suggest the correct abstractmethods - - -class Plugin(ABC): - @staticmethod - @abstractmethod - def specify_config(confspec): - pass - - @abstractmethod - def __init__(self, plugininterface, config): - pass - - def run(self, hooks, plugins): - pass - - -class SimplePlugin(Plugin): - @staticmethod - def specify_config(confspec): - pass - - def __init__(self, plugininterface, config): - super().__init__(plugininterface, config) - self.config = config - self.interface = plugininterface - self.plugins = plugininterface.other_plugins - self.hooks = plugininterface.hooks - - -plugin_interfaces = {} # dict of plugin interfaces - -# convenience dicts bundling together lists from interfaces -plugin_functions = {} # dict of plugins containing callable interface functions -plugin_hooks = {} # dict of plugins containing hook namespaces - - -_defer = True -_deferred_attachments = [] -_deferred_jobs = [] - - -def init_plugins(plugin_classes, plugin_configs, core_config): - # Startup pluginmanagers - global plugin_interfaces - global plugin_functions - global plugin_hooks - - global _defer - global _deferred_attachments - global _deferred_jobs - - for name, plugin_class in plugin_classes.items(): - # Instanciate the plugin interface - this also instanciates - # the plugin - plugin_interfaces[name] = PluginInterface( - name, plugin_class, plugin_configs[name], core_config) - plugin_functions[name] = plugin_interfaces[name].functions - plugin_hooks[name] = plugin_interfaces[name].hooks - - # interfaces and hooks should now be populated, attach hooks, schedule jobs - _defer = False - for attachment in _deferred_attachments: - _attach_hook(attachment) - - for job_desc in _deferred_jobs: - _add_job(job_desc) - - # Hand shared interface callables back out - for plugininterface in plugin_interfaces.values(): - plugininterface.functions = plugin_functions - - -def _add_job(job_desc): - global _deferred_jobs - global _defer - - if not _defer: - shepherd.scheduler.schedule_job(job_desc) - else: - _deferred_jobs.append(job_desc) - - -def _attach_hook(attachment): - global plugin_hooks - global _deferred_attachments - global _defer +def plugin(): + pass - if not _defer: - targetplugin_hooks = plugin_hooks.get(attachment.pluginname) - if targetplugin_hooks is not None: - targethook = getattr(targetplugin_hooks, attachment.hookname) - if targethook is not None: - targethook.attach(attachment.func) - else: - raise Exception("Could not find hook '" + - attachment.hookname+"' in module '"+attachment.pluginname+"'") - else: - raise Exception( - "Cannot attach hook to non-existing module '"+attachment.pluginname+"'") - else: - _deferred_attachments.append(attachment) -# Eventually, would like to be able to have client plugin simply: -# self.shepherd.add_job(trigger, self.interface.myfunc) -# self.shepherd.attach_hook(pluginnanme,hookname, callable) -# self.shepherd.addinterface -# self.shepherd.hooks.myhook() -# self.shepherd.plugins.otherplugin.otherinterface() +def plugin_function(): + pass -# self.shepherd.add_job() -# Would be good to be able to use abstract methods to enable simpler plugin config -# defs. A way to avoid instantiating the class would be to run it all as class methods, -# enabling +def plugin_hook(): + pass -HookAttachment = namedtuple( - 'HookAttachment', ['pluginname', 'hookname', 'func']) +def plugin_attachment(hookname): + pass class PluginInterface(): - ''' - Class to handle the management of a single plugin. - All interaction to or from the plugin to other Shepherd components or - plugins should go through here. - ''' - - def __init__(self, pluginname, pluginclass, pluginconfig, coreconfig): - if not issubclass(pluginclass, Plugin): - raise TypeError( - "Argument must be subclass of shepherd.plugin.Plugin") - - self.coreconfig = coreconfig - - self.hooks = SimpleNamespace() # My hooks - self.functions = SimpleNamespace() # My callable interface functions - - self._name = pluginname - self._plugin = pluginclass(self, pluginconfig) - - def register_hook(self, name): - setattr(self.hooks, name, Hook()) - - def register_function(self, func): - setattr(self.functions, func.__name__, InterfaceFunction(func)) + def __init__(self): + self._confspec = None @property - def other_plugins(self): - global plugin_functions - return plugin_functions + def confspec(self): + return self._confspec - def attach_hook(self, pluginname, hookname, func): - _attach_hook(HookAttachment(pluginname, hookname, func)) - # Add a job to the scheduler. By default each will be identified by the interface - # callable name, and so adding another job with the same callable will oevrride the first. - # Use the optional job_name to differentiate jobs with an extra string - - def add_job(self, func, hour, minute, second=0, job_name=""): - for function_name, function in self.functions.__dict__.items(): - if func == function.func: - # jobstring should canonically describe this job, to be retrieved - # after reboot later. Of the form: - # shepherd:pluginname:functionname:jobname - jobstring = "shepherd:"+self._name+":"+function_name+":"+job_name - _add_job(shepherd.scheduler.JobDescription(jobstring, hour, minute, second)) - break - else: - raise Exception( - "Could not add job. Callable must first be registered as " - "a plugin interface with PluginInterface.register_function()") +def find_plugins(plugin_names, plugin_dir=None): + """ -""" -An interface to a Shepherd module, accessible by other modules. -All public methods in a module interface need to be threadsafe, as they will -be called by other modules (which generally run in a seperate thread) -""" -def find_plugins(plugin_names, plugin_dir=None): - """ Looks for the list of plugin names supplied and returns their classes. Will first try for plugin modules and packages locally located in ``shepherd.plugins``, then for modules and packages prefixed ``shepherd_`` located in the supplied ``plugin_dir``