New plugin loading system

shepherd/agent/core.py

@@ -91,6 +91,10 @@ def cli(ctx, default_config_path, local_operation, only_default_layer):
 
 
 @cli.command()
+@click.option('-d', '--plugin-dir', type=click.Path(),
+              help="Override the configured directory to search for plugin modules, in addition to"
+              " built in Shepherd plugins and the global import path."
+              " Supplying the option empty will use the current directory.")
 def test():
     print("test!")
 
@@ -107,8 +111,8 @@ def test():
 @click.pass_context
 def template(ctx, plugin_name, include_all, config_path, plugin_dir):
     """
-    Generate a template config TOML file for PLUGIN, or for the Shepherd core if
+    Generate a template config TOML file for PLUGIN_NAME, or for the Shepherd core if
-    PLUGIN is not provided.
+    PLUGIN_NAME is not provided.
 
     If config path is provided ("-c"), append to that file (if it exists) or write to
     a new file (if it doesn't yet exist).
@@ -319,7 +323,7 @@ def load_config_layer_and_plugins(confman: ConfigManager, config_source):
     plugin_names.remove("shepherd")
 
     # Load plugins to get their config specifications
-    plugin_interfaces = plugin.load_plugins(plugin_names, plugin_dir)
+    plugin_interfaces = [plugin.load_plugin(name, plugin_dir) for name in plugin_names]
     for plugin_name, plugin_interface in plugin_interfaces.items():
         confman.add_confspec(plugin_name, plugin_interface.confspec)
 

shepherd/agent/plugin.py

@@ -1,8 +1,11 @@
 import importlib
+from pathlib import Path
 import inspect
 import logging
 import sys
-import os
+import pkgutil
+import pkg_resources
+from .. import base_plugins
 
 log = logging.getLogger(__name__)
 
@@ -11,20 +14,24 @@ class PluginLoadError(Exception):
     pass
 
 
-def plugin():
+def plugin(cls):
-    pass
+    return cls
 
 
 def plugin_function():
-    pass
+    def wrapped(method):
+        return method
+    return wrapped
 
 
-def plugin_hook():
+def plugin_hook(method):
-    pass
+    return method
 
 
 def plugin_attachment(hookname):
-    pass
+    def wrapped(method):
+        return method
+    return wrapped
 
 
 class PluginInterface():
@@ -36,64 +43,104 @@ class PluginInterface():
         return self._confspec
 
 
-def find_plugins(plugin_names, plugin_dir=None):
+def discover_base_plugins():
+    """
+    Returns a list of base plugin names available to load. These are plugins included with
+    shepherd-agent, in 'base_plugins'.
+    """
+    return [name for _, name, _ in pkgutil.iter_modules(base_plugins.__path__)]
+
+
+def discover_custom_plugins(plugin_dir=None):
     """
+    Returns a list of custom plugin names available to load. This includes all modules or packages
+    within the supplied custom plugin directory.
+    """
+    if plugin_dir:
+        if Path(plugin_dir).is_dir():
+            return [name for _, name, _ in pkgutil.iter_modules([plugin_dir])]
+        else:
+            log.warning(F"Custom plugin directory {plugin_dir} does not exist")
+    return []
+
+
+def discover_installed_plugins():
+    """
+    Returns a list of installed plugin names available to load. These are packages that have used
+    the 'shephed.plugin' entrypoint in their setup.py
+    """
+    return [entrypoint.name for entrypoint in pkg_resources.iter_entry_points('shepherd.plugins')]
+
 
+loaded_plugins = {}
 
 
+def load_plugin(plugin_name, plugin_dir=None):
+    """
+    Finds a Shepherd plugin, loads it, and returns the resulting PluginInterface object.
 
-    Looks for the list of plugin names supplied and returns their classes.
+    Will check 3 sources, in order:
-    Will first try for plugin modules and packages locally located in ``shepherd.plugins``,
+        1. Built-in plugin modules/subpackages within ''shepherd.base_plugins''. Plugin
-    then for modules and packages prefixed ``shepherd_`` located in the supplied ``plugin_dir``
+            module/package names match the plugin name.
-    and lastly in the global import path.
+        2. Modules/packages within the supplied ''plugin_dir'' path. Plugin module/package
+            names match the plugin name.
+        3. Any installed packages supplying the ''shepherd.plugin'' entrypoint.
 
     Args:
-        plugin_names: List of plugin names to try and load
+        plugin_name: Name used to try and locate the plugin
-        plugin_dir: optional search path
+        plugin_dir: Optional directory path to be used for custom plugins
 
-    Returns:
+    Returns: The PluginInterface for the loaded plugin
-        Dict of plugin classes, with their names as keys
     """
-    plugin_classes = {}
+
-    for plugin_name in plugin_names:
+    if plugin_name in loaded_plugins:
-        # First look for core plugins, then the plugin_dir, then in the general import path
+        return loaded_plugins[plugin_name]
-        # for custom ones prefixed with "shepherd_"
+
+    # Each of the 3 plugin sources have different import mechanisms. Discovery is broken out to
+    # allow them to be listed. Using a try/except block wouldn't be able to tell the difference
+    # between a plugin not being found or //it's// imports not loading correctly.
+    mod = None
+    if plugin_name in discover_base_plugins():
+        mod = importlib.import_module(base_plugins.__name__+'.'+plugin_name)
+        log.info(F"Loading base plugin {plugin_name}")
+
+    elif plugin_name in discover_custom_plugins(plugin_dir):
+        saved_syspath = sys.path
         try:
-            # mod = importlib.import_module("shepherd.plugins." + plugin_name)
+            sys.path = [str(plugin_dir)]
-            mod = importlib.import_module('.'+plugin_name, "shepherd.plugins")
+            mod = importlib.import_module(plugin_name)
-            # TODO - ModuleNotFoundError is also triggered here if the plugin has a dependancy
+        finally:
-            # that can't be found
+            sys.path = saved_syspath
-        except ModuleNotFoundError:
+        modpath = getattr(mod, "__path__", mod.__file__)
-            try:
+        log.info(F"Loading custom plugin {plugin_name} from {modpath}")
-                if plugin_dir:
+
-                    if os.path.isdir(plugin_dir):
+    elif plugin_name in discover_installed_plugins():
-                        sys.path.append(plugin_dir)
+        mod = pkg_resources.iter_entry_points('shepherd.plugins', plugin_name)[0].load()
-                        mod = importlib.import_module("shepherd_" + plugin_name)
+        log.info(F"Loading installed plugin {plugin_name} from {mod.__name__}")
-                        sys.path.remove(plugin_dir)
+
-                    else:
+    if not mod:
-                        raise PluginLoadError("plugin_dir is not a valid directory")
+        raise PluginLoadError("Could not find plugin "+plugin_name)
-                else:
+
-                    mod = importlib.import_module("shepherd_" + plugin_name)
+    # Scan imported module for PluginInterface instance
-            except ModuleNotFoundError:
+    def is_plugininterface(member):
-                raise PluginLoadError("Could not find plugin "+plugin_name)
+        return isinstance(member, PluginInterface)
-
-        # Scan imported module for Plugin subclass
-        def is_module_plugin(member, module=mod):
-            return (inspect.isclass(member) and member.__module__ ==
-                    module.__name__ and issubclass(member, Plugin))
-
-        class_list = inspect.getmembers(mod, is_module_plugin)
-
-        if class_list:
-            if len(class_list) > 1:
-                log.warning(
-                    F"Plugin module {mod.__name__} has more than one shepherd.Plugin subclass.")
-            _, plugin_classes[plugin_name] = class_list[0]
-            log.info(F"Loading plugin {plugin_classes[plugin_name].__name__}"
-                     " from module {mod.__name__}")
-        else:
-            raise PluginLoadError("Imported shepherd plugin modules must contain a"
-                                  " subclass of shepherd.plugin.Plugin, such as"
-                                  " shepherd.plugin.SimplePlugin")
 
-    return plugin_classes
+    interface_list = inspect.getmembers(mod, is_plugininterface)
+    if not interface_list:
+        raise PluginLoadError("Imported shepherd plugins must contain an instance"
+                              " of PluginInterface")
+
+    if len(interface_list) > 1:
+        log.warning(F"Plugin module {mod.__name__} has more"
+                    F" than one PluginInterface instance.")
+
+    _, interface = interface_list[0]
+
+    # TODO Populate plugin interface
+
+    loaded_plugins[plugin_name] = interface
+    return interface
+
+
+def init_plugins(plugin_configs, core_config):
+    pass