From 74c8bbc5abe4fd2a98bab1b91773c912204b8bed Mon Sep 17 00:00:00 2001 From: Thomas Wilson Date: Mon, 6 Jan 2020 17:23:19 +0800 Subject: [PATCH] New plugin loading system --- shepherd/agent/core.py | 10 ++- shepherd/agent/plugin.py | 161 +++++++++++++++++++++++++-------------- 2 files changed, 111 insertions(+), 60 deletions(-) diff --git a/shepherd/agent/core.py b/shepherd/agent/core.py index 73179e8..f03bb1f 100644 --- a/shepherd/agent/core.py +++ b/shepherd/agent/core.py @@ -91,6 +91,10 @@ def cli(ctx, default_config_path, local_operation, only_default_layer): @cli.command() +@click.option('-d', '--plugin-dir', type=click.Path(), + help="Override the configured directory to search for plugin modules, in addition to" + " built in Shepherd plugins and the global import path." + " Supplying the option empty will use the current directory.") def test(): print("test!") @@ -107,8 +111,8 @@ def test(): @click.pass_context def template(ctx, plugin_name, include_all, config_path, plugin_dir): """ - Generate a template config TOML file for PLUGIN, or for the Shepherd core if - PLUGIN is not provided. + Generate a template config TOML file for PLUGIN_NAME, or for the Shepherd core if + PLUGIN_NAME is not provided. If config path is provided ("-c"), append to that file (if it exists) or write to a new file (if it doesn't yet exist). @@ -319,7 +323,7 @@ def load_config_layer_and_plugins(confman: ConfigManager, config_source): plugin_names.remove("shepherd") # Load plugins to get their config specifications - plugin_interfaces = plugin.load_plugins(plugin_names, plugin_dir) + plugin_interfaces = [plugin.load_plugin(name, plugin_dir) for name in plugin_names] for plugin_name, plugin_interface in plugin_interfaces.items(): confman.add_confspec(plugin_name, plugin_interface.confspec) diff --git a/shepherd/agent/plugin.py b/shepherd/agent/plugin.py index ed7988c..7b1e79a 100644 --- a/shepherd/agent/plugin.py +++ b/shepherd/agent/plugin.py @@ -1,8 +1,11 @@ import importlib +from pathlib import Path import inspect import logging import sys -import os +import pkgutil +import pkg_resources +from .. import base_plugins log = logging.getLogger(__name__) @@ -11,20 +14,24 @@ class PluginLoadError(Exception): pass -def plugin(): - pass +def plugin(cls): + return cls def plugin_function(): - pass + def wrapped(method): + return method + return wrapped -def plugin_hook(): - pass +def plugin_hook(method): + return method def plugin_attachment(hookname): - pass + def wrapped(method): + return method + return wrapped class PluginInterface(): @@ -36,64 +43,104 @@ class PluginInterface(): return self._confspec -def find_plugins(plugin_names, plugin_dir=None): +def discover_base_plugins(): + """ + Returns a list of base plugin names available to load. These are plugins included with + shepherd-agent, in 'base_plugins'. + """ + return [name for _, name, _ in pkgutil.iter_modules(base_plugins.__path__)] + + +def discover_custom_plugins(plugin_dir=None): """ + Returns a list of custom plugin names available to load. This includes all modules or packages + within the supplied custom plugin directory. + """ + if plugin_dir: + if Path(plugin_dir).is_dir(): + return [name for _, name, _ in pkgutil.iter_modules([plugin_dir])] + else: + log.warning(F"Custom plugin directory {plugin_dir} does not exist") + return [] + + +def discover_installed_plugins(): + """ + Returns a list of installed plugin names available to load. These are packages that have used + the 'shephed.plugin' entrypoint in their setup.py + """ + return [entrypoint.name for entrypoint in pkg_resources.iter_entry_points('shepherd.plugins')] + +loaded_plugins = {} +def load_plugin(plugin_name, plugin_dir=None): + """ + Finds a Shepherd plugin, loads it, and returns the resulting PluginInterface object. - Looks for the list of plugin names supplied and returns their classes. - Will first try for plugin modules and packages locally located in ``shepherd.plugins``, - then for modules and packages prefixed ``shepherd_`` located in the supplied ``plugin_dir`` - and lastly in the global import path. + Will check 3 sources, in order: + 1. Built-in plugin modules/subpackages within ''shepherd.base_plugins''. Plugin + module/package names match the plugin name. + 2. Modules/packages within the supplied ''plugin_dir'' path. Plugin module/package + names match the plugin name. + 3. Any installed packages supplying the ''shepherd.plugin'' entrypoint. Args: - plugin_names: List of plugin names to try and load - plugin_dir: optional search path + plugin_name: Name used to try and locate the plugin + plugin_dir: Optional directory path to be used for custom plugins - Returns: - Dict of plugin classes, with their names as keys + Returns: The PluginInterface for the loaded plugin """ - plugin_classes = {} - for plugin_name in plugin_names: - # First look for core plugins, then the plugin_dir, then in the general import path - # for custom ones prefixed with "shepherd_" + + if plugin_name in loaded_plugins: + return loaded_plugins[plugin_name] + + # Each of the 3 plugin sources have different import mechanisms. Discovery is broken out to + # allow them to be listed. Using a try/except block wouldn't be able to tell the difference + # between a plugin not being found or //it's// imports not loading correctly. + mod = None + if plugin_name in discover_base_plugins(): + mod = importlib.import_module(base_plugins.__name__+'.'+plugin_name) + log.info(F"Loading base plugin {plugin_name}") + + elif plugin_name in discover_custom_plugins(plugin_dir): + saved_syspath = sys.path try: - # mod = importlib.import_module("shepherd.plugins." + plugin_name) - mod = importlib.import_module('.'+plugin_name, "shepherd.plugins") - # TODO - ModuleNotFoundError is also triggered here if the plugin has a dependancy - # that can't be found - except ModuleNotFoundError: - try: - if plugin_dir: - if os.path.isdir(plugin_dir): - sys.path.append(plugin_dir) - mod = importlib.import_module("shepherd_" + plugin_name) - sys.path.remove(plugin_dir) - else: - raise PluginLoadError("plugin_dir is not a valid directory") - else: - mod = importlib.import_module("shepherd_" + plugin_name) - except ModuleNotFoundError: - raise PluginLoadError("Could not find plugin "+plugin_name) - - # Scan imported module for Plugin subclass - def is_module_plugin(member, module=mod): - return (inspect.isclass(member) and member.__module__ == - module.__name__ and issubclass(member, Plugin)) - - class_list = inspect.getmembers(mod, is_module_plugin) - - if class_list: - if len(class_list) > 1: - log.warning( - F"Plugin module {mod.__name__} has more than one shepherd.Plugin subclass.") - _, plugin_classes[plugin_name] = class_list[0] - log.info(F"Loading plugin {plugin_classes[plugin_name].__name__}" - " from module {mod.__name__}") - else: - raise PluginLoadError("Imported shepherd plugin modules must contain a" - " subclass of shepherd.plugin.Plugin, such as" - " shepherd.plugin.SimplePlugin") + sys.path = [str(plugin_dir)] + mod = importlib.import_module(plugin_name) + finally: + sys.path = saved_syspath + modpath = getattr(mod, "__path__", mod.__file__) + log.info(F"Loading custom plugin {plugin_name} from {modpath}") + + elif plugin_name in discover_installed_plugins(): + mod = pkg_resources.iter_entry_points('shepherd.plugins', plugin_name)[0].load() + log.info(F"Loading installed plugin {plugin_name} from {mod.__name__}") + + if not mod: + raise PluginLoadError("Could not find plugin "+plugin_name) + + # Scan imported module for PluginInterface instance + def is_plugininterface(member): + return isinstance(member, PluginInterface) - return plugin_classes + interface_list = inspect.getmembers(mod, is_plugininterface) + if not interface_list: + raise PluginLoadError("Imported shepherd plugins must contain an instance" + " of PluginInterface") + + if len(interface_list) > 1: + log.warning(F"Plugin module {mod.__name__} has more" + F" than one PluginInterface instance.") + + _, interface = interface_list[0] + + # TODO Populate plugin interface + + loaded_plugins[plugin_name] = interface + return interface + + +def init_plugins(plugin_configs, core_config): + pass