Restructure

.gitignore

@@ -0,0 +1,123 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/

LICENSE.txt

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2018 Thomas Wilson
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -0,0 +1,8 @@
+# Shepherd
+
+This is a project to try and standardise the management of physically remote
+nodes (used mainly for solar powered sensors and actuators on farms). 
+
+Shepherd provides a modular interface to define configuration parameters and
+schedule behaviour, allowing remote config changes while reducing the risk of breaking
+things (requiring a long trip out to the node to recover it).

setup.py

@@ -0,0 +1,20 @@
+from setuptools import setup
+
+setup(
+    name='shepherd',
+    version='0.2dev',
+    author='Thomas Wilson',
+    author_email='t.wilson@distreon.com.au',
+    packages=['shepherd', ],
+    install_requires=[
+          'toml',
+          'apscheduler',
+          'paramiko'
+      ],
+    entry_points={
+        'console_scripts': ['shepherd=shepherd.core:main'],
+    },
+    license='MIT license',
+    description='Herd your mob of physically remote nodes',
+    long_description=open('README.md').read(),
+)

shepherd.service

@@ -0,0 +1,13 @@
+# Should be copied to /lib/systemd/system/ then enabled with
+# sudo systemctl enable shepherd.service
+[Unit]
+Description=Monitoring and control service for remote nodes
+After=multi-user.target
+
+[Service]
+Type=simple
+Environment=PYTHONUNBUFFERED=true
+ExecStart=/usr/local/bin/shepherd /home/pi/shepherd.toml
+
+[Install]
+WantedBy=multi-user.target

shepherd/config.py

@@ -0,0 +1,223 @@
+import re
+import toml
+
+
+class InvalidConfigError(Exception):
+    pass
+
+# On start,
+# get conf_defs via imports in core
+# Load a conf file
+# Validate the Shepherd table in conf file, then load its values to get module list
+# Validate other loaded tables/modules, then load their values
+
+# How do modules wind up with their instance of ConfigValues? Return val instance
+# from validation function - as it needs to build the vals instance while validating anyway
+
+# Validate a conf file given a module or config_def list
+
+
+# idea is to create these similar to how arg parser works
+
+# how to store individual items in def? Dict of...
+# Tables need a dict, lists/arrays need a list of dicts, but what's in the dict?
+# if it's another instacnce of ConfigDef, then each Config Def needs to handle
+# one item, but cater for all the different types of items - and many of those
+# should be able to add a new item.
+
+# could have a separate class for the root, but lower tables really need to
+# perform exactly the same...
+
+# config def required interface:
+# Validate values.
+
+class _ConfigDefinition():
+    def __init__(self, default=None, optional=False):
+        self.default = default
+        self.optional = optional
+
+    def validate(self, value): # pylint: disable=W0613
+        raise TypeError("_ConfigDefinition.validate() is an abstract method")
+
+
+class BoolDef(_ConfigDefinition):
+    def __init__(self, default=None, optional=False): # pylint: disable=W0235
+        super().__init__(default, optional)
+
+    def validate(self, value):
+        if not isinstance(value, bool):
+            raise InvalidConfigError("Config value must be a boolean")
+
+
+class IntDef(_ConfigDefinition):
+    def __init__(self, default=None, minval=None, maxval=None,
+                 optional=False):
+        super().__init__(default, optional)
+        self.minval = minval
+        self.maxval = maxval
+
+    def validate(self, value):
+        if not isinstance(value, int):
+            raise InvalidConfigError("Config value must be an integer")
+        if self.minval is not None and value < self.minval:
+            raise InvalidConfigError("Config value must be >= " +
+                                     str(self.minval))
+        if self.maxval is not None and value > self.maxval:
+            raise InvalidConfigError("Config value must be <= " +
+                                     str(self.maxval))
+
+
+class StringDef(_ConfigDefinition):
+    def __init__(self, default=None, minlength=None, maxlength=None,
+                 optional=False):
+        super().__init__(default, optional)
+        self.minlength = minlength
+        self.maxlength = maxlength
+
+    def validate(self, value):
+        if not isinstance(value, str):
+            raise InvalidConfigError("Config value must be a string")
+        if self.minlength is not None and len(value) < self.minlength:
+            raise InvalidConfigError("Config string length must be >= " +
+                                     str(self.minlength))
+        if self.maxlength is not None and len(value) > self.maxlength:
+            raise InvalidConfigError("Config string length must be <= " +
+                                     str(self.maxlength))
+
+
+class TableDef(_ConfigDefinition):
+    def __init__(self, default=None, optional=False):
+        super().__init__(default, optional)
+        self.def_table = {}
+
+    def add_def(self, name, newdef):
+        if not isinstance(newdef, _ConfigDefinition):
+            raise TypeError("Config definiton must be an instance of a "
+                            "ConfigDefinition subclass")
+        if not isinstance(name, str):
+            raise TypeError("Config definition name must be a string")
+        self.def_table[name] = newdef
+        return newdef
+
+    def validate(self, value_table): # pylint: disable=W0221
+        def_set = set(self.def_table.keys())
+        value_set = set(value_table.keys())
+
+        for missing_key in def_set - value_set:
+            if not self.def_table[missing_key].optional:
+                raise InvalidConfigError("Table must contain key: " +
+                                         missing_key)
+            else:
+                value_table[missing_key] = self.def_table[missing_key].default
+
+        for extra_key in value_set - def_set:
+            raise InvalidConfigError("Table contains unknown key: " +
+                                     extra_key)
+
+        for key, value in value_table.items():
+            try:
+                self.def_table[key].validate(value)
+            except InvalidConfigError as e:
+                e.args = ("Key: " + key,) + e.args
+                raise
+
+
+class _ArrayDefMixin():
+    def validate(self, value_array):
+        if not isinstance(value_array, list):
+            raise InvalidConfigError("Config item must be an array")
+        for index, value in enumerate(value_array):
+            try:
+                super().validate(value)
+            except InvalidConfigError as e:
+                e.args = ("Array index: " + str(index),) + e.args
+                raise
+
+
+class BoolArrayDef(_ArrayDefMixin, BoolDef):
+    pass
+
+
+class IntArrayDef(_ArrayDefMixin, IntDef):
+    pass
+
+
+class StringArrayDef(_ArrayDefMixin, StringDef):
+    pass
+
+
+class TableArrayDef(_ArrayDefMixin, TableDef):
+    pass
+
+
+class ConfDefinition(TableDef):
+    pass
+
+
+class ConfigManager():
+    def __init__(self):
+        self.root_config = {}
+
+    def load(self, source):
+        if isinstance(source, dict):  # load from dict
+            self.root_config = source
+        elif isinstance(source, str):  # load from pathname
+            with open(source, 'r') as conf_file:
+                self.root_config = toml.load(conf_file)
+        else:  # load from file
+            self.root_config = toml.load(source)
+
+    def get_config(self, table_name, conf_def):
+        if not isinstance(conf_def, ConfDefinition):
+            raise TypeError("Supplied config definition must be an instance "
+                            "of ConfDefinition")
+        if table_name not in self.root_config:
+            raise InvalidConfigError("Config must contain table: " + table_name)
+        try:
+            conf_def.validate(self.root_config[table_name])
+        except InvalidConfigError as e:
+            e.args = ("Module: " + table_name,) + e.args
+            raise
+        return self.root_config[table_name]
+
+    def get_configs(self, conf_defs):
+        config_values = {}
+        for name, conf_def in conf_defs.items():
+            config_values[name] = self.get_config(name, conf_def)
+        return config_values
+
+    def get_module_configs(self, modules):
+        config_values = {}
+        for name, module in modules.items():
+            config_values[name] = self.get_config(name, module.conf_def)
+        return config_values
+
+    def dump_toml(self):
+        return toml.dumps(self.root_config)
+
+    def dump_to_file(self, filepath, message=None):
+        with open(filepath, 'w+') as f:
+            content = self.dump_toml()
+            if message is not None:
+                content = content.rstrip() + gen_comment(message)
+            f.write(content)
+
+
+def strip_toml_message(string):
+    print("stripping...")
+    return re.sub("(?m)^#\\ shepherd_message:[^\\n]*$\\n?(?:^#[^\\n]+$\\n?)*",
+                  '', string)
+
+
+def update_toml_message(filepath, message):
+    with open(filepath, 'r+') as f:
+        content = f.read()
+        content = strip_toml_message(content).rstrip()
+        content += gen_comment(message)
+        f.seek(0)
+        f.write(content)
+        f.truncate()
+
+
+def gen_comment(string):
+    return '\n# shepherd_message: ' + '\n# '.join(string.replace('#', '').splitlines()) + '\n'

shepherd/core.py

@@ -0,0 +1,138 @@
+#!/usr/bin/env python3
+
+# depends on:
+# python 3.4 (included in Raspbian Jessie)
+# APScheduler
+
+
+import sys
+import argparse
+import os
+import toml
+
+
+import shepherd.config
+import shepherd.module
+
+from apscheduler.schedulers.blocking import BlockingScheduler
+from datetime import datetime
+
+from types import SimpleNamespace
+
+
+# Future implementations of checking config differences should be done on
+# the hash of the nested conf dict, so comments shouldn't affect this.
+
+# save old config to somewhere in the shepherd root dir - probably need to
+# implement a TOML writer in the config module.
+
+# later on, there's going to be an issue with a new config being applied
+# remotely, then the system restarting, and an old edit in /boot being
+# applied over the top...
+# Fix this by saving the working config to /boot when new config applied
+# remotely.
+
+def load_config(config_path):
+    confman = shepherd.config.ConfigManager()
+    confman.load(os.path.expanduser(config_path))
+    core_conf = confman.get_config("shepherd", core_confdef())
+
+    # Check for an edit_conf file, and try to load it
+    try:
+        edit_confman = shepherd.config.ConfigManager()
+        edit_confman.load(os.path.expanduser(core_conf["conf_edit_path"]))
+        core_edit_conf = edit_confman.get_config("shepherd", core_confdef())
+
+        mod_classes = shepherd.module.find_modules(core_edit_conf["modules"])
+        mod_configs = edit_confman.get_module_configs(mod_classes)
+
+    except FileNotFoundError:
+        conf_edit_message = None
+    except shepherd.config.InvalidConfigError as e:
+        conf_edit_message = "Invalid config.\n " + str(e.args)
+    except toml.TomlDecodeError as e:
+        conf_edit_message = "TOML syntax error.\n" + str(e)
+    except Exception:
+        conf_edit_message = "Error processing new config"
+    else:
+        conf_edit_message = ("Successfully applied this config at:" +
+                             str(datetime.now()))
+        confman = edit_confman
+        core_conf = core_edit_conf
+
+    if conf_edit_message is not None:
+        shepherd.config.update_toml_message(
+            os.path.expanduser(core_conf["conf_edit_path"]), conf_edit_message)
+
+    # if editconf failed, load current config
+    if confman is not edit_confman:
+        mod_classes = shepherd.module.find_modules(core_conf["modules"])
+        mod_configs = confman.get_module_configs(mod_classes)
+
+    # If no editconf file was found, write out the current config as a template
+    if conf_edit_message is None:
+        confman.dump_to_file(os.path.expanduser(core_conf["conf_edit_path"]),
+                             "Config generated at:" + str(datetime.now()))
+
+    return (core_conf, mod_classes, mod_configs)
+
+
+def core_confdef():
+    confdef = shepherd.config.ConfDefinition()
+    confdef.add_def("id", shepherd.config.StringDef())
+    confdef.add_def("modules", shepherd.config.StringArrayDef())
+    confdef.add_def("root_dir", shepherd.config.StringDef())
+    confdef.add_def("conf_edit_path", shepherd.config.StringDef())
+    return confdef
+
+
+class ShepherdInterface(shepherd.module.Interface):
+    def __init__(self, scheduler, config):
+        super().__init__(None)
+        self.id = config["id"]
+        self.root_dir = config["root_dir"]
+        self.scheduler = scheduler
+
+
+def main():
+    argparser = argparse.ArgumentParser(description="Keep track of a mob "
+                                                    "of roaming Pis")
+    argparser.add_argument("configfile", nargs='?', metavar="configfile",
+                           help="Path to configfile", default="shepherd.toml")
+
+    args = argparser.parse_args()
+
+    confman = shepherd.config.ConfigManager()
+    confman.load(os.path.expanduser(args.configfile))
+    core_conf = confman.get_config("shepherd", core_confdef())
+    breakpoint()
+
+    (core_conf, mod_classes, mod_configs) = load_config(args.configfile)
+
+    scheduler = BlockingScheduler()
+    core_interface = ShepherdInterface(scheduler, core_conf)
+
+    # get validated config values for modules, then instantiate them
+    modules = {}
+    mod_interfaces = {}
+    for name, mod_class in mod_classes.items():
+        modules[name] = mod_class(mod_configs[name], core_interface)
+        mod_interfaces[name] = modules[name].interface
+
+    # run post init after all modules are loaded to allow them to hook in to
+    # each other
+    mod_interfaces = SimpleNamespace(**mod_interfaces)
+    for module in modules.values():
+        module.init_other_interfaces(mod_interfaces)
+
+    print(str(datetime.now()))
+    print('Press Ctrl+{0} to exit'.format('Break' if os.name == 'nt' else 'C'))
+
+    try:
+        scheduler.start()
+    except (KeyboardInterrupt, SystemExit):
+        pass
+
+
+if __name__ == "__main__":
+    main()

shepherd/module.py

@@ -0,0 +1,84 @@
+#!/usr/bin/env python3
+
+from contextlib import suppress
+import importlib
+
+
+class Hook():
+    def __init__(self):
+        self.attached_functions = []
+
+    def attach(self, new_func):
+        if not callable(new_func):
+            raise TypeError("Argument to Hook.attach must be callable")
+        self.attached_functions.append(new_func)
+
+    def __call__(self, *args, **kwargs):
+        for func in self.attached_functions:
+            func(*args, **kwargs)
+
+
+class Module():
+    def __init__(self, config, core_interface):
+        self.config = config
+        self.shepherd = core_interface
+        #self.shepherd.scheduler
+
+        self.interface = Interface(self)
+        self.modules = {}
+        # dummy interface in case module doesn't want one
+
+    def init_other_interfaces(self, interfaces):
+        if not isinstance(self.interface, Interface):
+            raise TypeError("shepherd.module.Module interface attribute must "
+                            "be subclass of type shepherd.module.Interface")
+        self.modules = interfaces
+
+
+# Look at providing a run() function or similar, which is a thread started
+# post_modules_setup
+class SimpleModule(Module):
+    def __init__(self, config, core_interface):
+        super().__init__(config, core_interface)
+        self.setup()
+
+    def init_other_interfaces(self, interfaces):
+        super().init_other_interfaces(interfaces)
+        self.setup_other_modules()
+
+    # Users override this, self.shepherd and self.config are available now.
+    # User still needs to define self.interface if it is used.
+    def setup(self):
+        pass
+
+    def setup_other_modules(self):
+        pass
+
+
+"""
+An interface to a Shepherd module, accessible by other modules.
+All public methods in a module interface need to be threadsafe, as they will
+be called by other modules (which generally run in a seperate thread)
+"""
+class Interface():
+    def __init__(self, module):
+        self._module = module
+
+
+def find_modules(module_names):
+    module_classes = {}
+    for module_name in module_names:
+        mod = importlib.import_module("shepherd.modules." + module_name)
+        attrs = [getattr(mod, name) for name in dir(mod)]
+
+        for attr in attrs:
+            with suppress(TypeError):
+                if issubclass(attr, Module):
+                    module_classes[module_name] = attr
+                    break
+        else:
+            raise Exception("Imported shepherd modules must contain a "
+                            "subclass of shepherd.module.Module, such as"
+                            "shepherd.module.SimpleModule")
+
+    return module_classes

shepherd/modules/aphidtrap.py

@@ -0,0 +1,77 @@
+#!/usr/bin/env python3
+
+import shepherd.config
+import shepherd.module
+
+import sys
+import os
+import time
+import argparse
+
+from gpiozero import OutputDevice, Device
+from gpiozero.pins.pigpio import PiGPIOFactory
+
+from shepherd.modules.betterservo import BetterServo
+
+Device.pin_factory = PiGPIOFactory()
+
+
+APHIDTRAP_LED_PIN = 5 #Out2
+
+
+class AphidtrapConfDef(shepherd.config.ConfDefinition):
+    def __init__(self):
+        super().__init__()
+
+
+class AphidtrapModule(shepherd.module.SimpleModule):
+    conf_def = AphidtrapConfDef()
+
+    def setup(self):
+
+        print("Aphidtrap config:")
+        print(self.config)
+
+        self.led_power = OutputDevice(APHIDTRAP_LED_PIN,
+                                   active_high=True,
+                                   initial_value=False)
+
+    def setup_other_modules(self):
+        self.modules.picam.hook_pre_cam.attach(self.led_on)
+        self.modules.picam.hook_post_cam.attach(self.led_off)
+
+    def led_on(self):
+        self.led_power.on()
+
+    def led_off(self):
+        self.led_power.off()
+
+
+
+def main(argv):
+    argparser = argparse.ArgumentParser(
+      description='Module for aphidtrap control functions. Run for testing')
+    argparser.add_argument("configfile", nargs='?', metavar="configfile",
+                           help="Path to configfile", default="conf.toml")
+
+
+    args = argparser.parse_args()
+    confman = shepherd.config.ConfigManager()
+
+    srcdict = {"aphidtrap": {}}
+
+    if os.path.isfile(args.configfile):
+        confman.load(args.configfile)
+    else:
+        confman.load(srcdict)
+
+    aphidtrap_mod = AphidtrapModule(confman.get_config("aphid", AphidtrapConfDef()),
+                                  shepherd.module.Interface(None))
+                                  
+    aphidtrap_mod.led_on()
+    time.sleep(2)
+    aphidtrap_mod.led_off()
+
+
+if __name__ == "__main__":
+    main(sys.argv[1:])

shepherd/modules/betterservo.py

@@ -0,0 +1,147 @@
+from gpiozero import PWMOutputDevice, SourceMixin, CompositeDevice
+
+
+class BetterServo(SourceMixin, CompositeDevice):
+    """
+    Copy of GPIOZero servo, but with control over pulse width and active_high
+    """
+    def __init__(
+            self, pin=None, initial_value=0.0,
+            min_pulse_width=1/1000, max_pulse_width=2/1000,
+            frame_width=20/1000, pin_factory=None, active_high=True):
+        if min_pulse_width >= max_pulse_width:
+            raise ValueError('min_pulse_width must be less than max_pulse_width')
+        if max_pulse_width >= frame_width:
+            raise ValueError('max_pulse_width must be less than frame_width')
+        self._frame_width = frame_width
+        self._min_dc = min_pulse_width / frame_width
+        self._dc_range = (max_pulse_width - min_pulse_width) / frame_width
+        self._min_value = -1
+        self._value_range = 2
+        super(BetterServo, self).__init__(
+            pwm_device=PWMOutputDevice(
+                pin, frequency=int(1 / frame_width), pin_factory=pin_factory,
+                active_high=False
+            ),
+            pin_factory=pin_factory
+        )
+        self.pwm_device.active_high=active_high
+        try:
+            self.value = initial_value
+        except:
+            self.close()
+            raise
+
+    @property
+    def frame_width(self):
+        """
+        The time between control pulses, measured in seconds.
+        """
+        return self._frame_width
+
+    @property
+    def min_pulse_width(self):
+        """
+        The control pulse width corresponding to the servo's minimum position,
+        measured in seconds.
+        """
+        return self._min_dc * self.frame_width
+
+    @property
+    def max_pulse_width(self):
+        """
+        The control pulse width corresponding to the servo's maximum position,
+        measured in seconds.
+        """
+        return (self._dc_range * self.frame_width) + self.min_pulse_width
+
+    @property
+    def pulse_width(self):
+        """
+        Returns the current pulse width controlling the servo.
+        """
+        if self.pwm_device.frequency is None:
+            return None
+        else:
+            return self.pwm_device.state * self.frame_width
+
+    @pulse_width.setter
+    def pulse_width(self, value):
+        if value is None:
+            self.pwm_device.frequency = None
+        elif self.min_pulse_width <= value <= self.max_pulse_width:
+            self.pwm_device.frequency = int(1 / self.frame_width)
+            self.pwm_device.value = (value / self.frame_width)
+        else:
+            raise OutputDeviceBadValue("Servo pulse_width must be between min and max supplied during construction, or None")
+
+    def min(self):
+        """
+        Set the servo to its minimum position.
+        """
+        self.value = -1
+
+    def mid(self):
+        """
+        Set the servo to its mid-point position.
+        """
+        self.value = 0
+
+    def max(self):
+        """
+        Set the servo to its maximum position.
+        """
+        self.value = 1
+
+    def detach(self):
+        """
+        Temporarily disable control of the servo. This is equivalent to
+        setting :attr:`value` to ``None``.
+        """
+        self.value = None
+
+    def _get_value(self):
+        if self.pwm_device.frequency is None:
+            return None
+        else:
+            return (
+                ((self.pwm_device.state - self._min_dc) / self._dc_range) *
+                self._value_range + self._min_value)
+
+    @property
+    def value(self):
+        """
+        Represents the position of the servo as a value between -1 (the minimum
+        position) and +1 (the maximum position). This can also be the special
+        value ``None`` indicating that the servo is currently "uncontrolled",
+        i.e. that no control signal is being sent. Typically this means the
+        servo's position remains unchanged, but that it can be moved by hand.
+        """
+        result = self._get_value()
+        if result is None:
+            return result
+        else:
+            # NOTE: This round() only exists to ensure we don't confuse people
+            # by returning 2.220446049250313e-16 as the default initial value
+            # instead of 0. The reason _get_value and _set_value are split
+            # out is for descendents that require the un-rounded values for
+            # accuracy
+            return round(result, 14)
+
+    @value.setter
+    def value(self, value):
+        if value is None:
+            self.pwm_device.frequency = None
+        elif -1 <= value <= 1:
+            self.pwm_device.frequency = int(1 / self.frame_width)
+            self.pwm_device.value = (
+                self._min_dc + self._dc_range *
+                ((value - self._min_value) / self._value_range)
+                )
+        else:
+            raise OutputDeviceBadValue(
+                "Servo value must be between -1 and 1, or None")
+
+    @property
+    def is_active(self):
+        return self.value is not None

shepherd/modules/mothtrap.py

@@ -0,0 +1,113 @@
+#!/usr/bin/env python3
+
+import shepherd.config
+import shepherd.module
+
+import sys
+import os
+import time
+import argparse
+
+from gpiozero import OutputDevice, Device
+from gpiozero.pins.pigpio import PiGPIOFactory
+
+from shepherd.modules.betterservo import BetterServo
+
+Device.pin_factory = PiGPIOFactory()
+
+
+MOTHTRAP_LED_PIN = 6
+MOTHTRAP_SERVO_PIN = 10
+MOTHTRAP_SERVO_POWER_PIN = 9
+
+
+class MothtrapConfDef(shepherd.config.ConfDefinition):
+    def __init__(self):
+        super().__init__()
+        self.add_def('servo_open_pulse', shepherd.config.IntDef(default=1200))
+        self.add_def('servo_closed_pulse', shepherd.config.IntDef(default=1800))
+        self.add_def('servo_open_time', shepherd.config.IntDef(default=5))
+
+
+class MothtrapModule(shepherd.module.SimpleModule):
+    conf_def = MothtrapConfDef()
+
+    def setup(self):
+
+        print("Mothtrap config:")
+        print(self.config)
+
+        servo_max = self.config["servo_open_pulse"] / 1000000
+        servo_min = self.config["servo_closed_pulse"] / 1000000
+
+        if servo_min > servo_max:
+            servo_min, servo_max = servo_max, servo_min
+
+        self.door_servo = BetterServo(MOTHTRAP_SERVO_PIN, initial_value=None,
+                                 active_high=False,
+                                 min_pulse_width=servo_min,
+                                 max_pulse_width=servo_max)
+
+        self.door_servo_power = OutputDevice(MOTHTRAP_SERVO_POWER_PIN,
+                                   active_high=True,
+                                   initial_value=False)
+
+        self.led_power = OutputDevice(MOTHTRAP_LED_PIN,
+                                   active_high=True,
+                                   initial_value=False)
+
+    def setup_other_modules(self):
+        self.modules.picam.hook_pre_cam.attach(self.led_on)
+        self.modules.picam.hook_post_cam.attach(self.led_off)
+        self.modules.picam.hook_post_cam.attach(self.run_servo)
+
+    def led_on(self):
+        self.led_power.on()
+
+    def led_off(self):
+        self.led_power.off()
+
+    def run_servo(self):
+        self.door_servo_power.on()
+        time.sleep(0.5)
+        self.door_servo.pulse_width = self.config["servo_open_pulse"] / 1000000
+        time.sleep(self.config["servo_open_time"])
+        self.door_servo.pulse_width = self.config["servo_closed_pulse"] / 1000000
+        time.sleep(self.config["servo_open_time"])
+        self.door_servo.detach()
+        self.door_servo_power.off()
+
+
+def main(argv):
+    argparser = argparse.ArgumentParser(
+      description='Module for mothtrap control functions. Run for testing')
+    argparser.add_argument("configfile", nargs='?', metavar="configfile",
+                           help="Path to configfile", default="conf.toml")
+    argparser.add_argument("test_function", nargs='?', choices=['servo'],
+                           help="test function to perform", default="servo")
+    argparser.add_argument("-o", help="servo open position, in us", type=int, default=1200, dest="servo_open_pulse")
+    argparser.add_argument("-c", help="servo closed position, in us", type=int, default=1800, dest="servo_closed_pulse")
+    argparser.add_argument("-w", help="wait time, in seconds", type=int, default=5, dest="servo_open_time")
+
+    args = argparser.parse_args()
+    confman = shepherd.config.ConfigManager()
+
+    srcdict = {"mothtrap": {"servo_open_pulse": args.servo_open_pulse,
+                            "servo_closed_pulse":args.servo_closed_pulse,
+                            "servo_open_time":args.servo_open_time}}
+
+    if os.path.isfile(args.configfile):
+        confman.load(args.configfile)
+    else:
+        confman.load(srcdict)
+
+    mothtrap_mod = MothtrapModule(confman.get_config("mothtrap", MothtrapConfDef()),
+                                  shepherd.module.Interface(None))
+                                  
+    mothtrap_mod.led_on()
+    mothtrap_mod.run_servo()
+    mothtrap_mod.led_off()
+
+
+if __name__ == "__main__":
+    main(sys.argv[1:])

shepherd/modules/picam.py

@@ -0,0 +1,134 @@
+import shepherd.config
+import shepherd.module
+
+import io
+import os
+from datetime import datetime
+import time
+from picamera import PiCamera
+from PIL import Image, ImageDraw, ImageFont
+
+
+overlayfont = "/usr/share/fonts/truetype/dejavu/DejaVuSansMono.ttf"
+
+
+class PiCamConfDef(shepherd.config.ConfDefinition):
+    def __init__(self):
+        super().__init__()
+        self.add_def('upload_images', shepherd.config.BoolDef(default=False, optional=True))
+        self.add_def('upload_bucket', shepherd.config.StringDef(default="", optional=True))
+        self.add_def('save_directory', shepherd.config.StringDef(default="", optional=False))
+        self.add_def('append_text', shepherd.config.StringDef(default="", optional=True))
+        self.add_def('append_id', shepherd.config.BoolDef(default=True, optional=True))
+
+        array = self.add_def('trigger', shepherd.config.TableArrayDef())
+        array.add_def('hour', shepherd.config.StringDef())
+        array.add_def('minute', shepherd.config.StringDef())
+        array.add_def('second', shepherd.config.StringDef(default="0", optional=True))
+
+# on server side, we want to be able to list commands that a module responds to
+# without actually instantiating the module class. Add command templates into
+# the conf_def, than attach to them in the interface? Was worried about having
+# "two sources of truth", but you already need to match the conf_def to the
+# name where you access the value in the module. Could have add_command, which
+# you then add standard conf_def subclasses to, to reuse validation and server
+# form generation logic...
+
+
+class PiCamInterface(shepherd.module.Interface):
+    def __init__(self, module):
+        super().__init__(module)
+
+        self.hook_pre_cam = shepherd.module.Hook()
+        self.hook_post_cam = shepherd.module.Hook()
+
+        # self.add_command("trigger", self.module.camera_job)
+
+# other module can then call, in init_interfaces, if self.modules.picam is not None:
+# self.modules.picam.hooks.attach("pre_cam",self.myfunc)
+# self.modules.picam.pre_cam.attach(self.my_func)
+
+# self.modules.picam.trigger()
+
+
+class PiCamModule(shepherd.module.SimpleModule):
+    conf_def = PiCamConfDef()
+
+    def setup(self):
+        self.interface = PiCamInterface(self)
+        # do some camera init stuff
+
+        print("Camera config:")
+        print(self.config)
+
+        if self.config["save_directory"] is "":
+            self.save_directory = os.path.join(self.shepherd.root_dir,
+                                               "camera")
+        else:
+            self.save_directory = self.config["save_directory"]
+            
+        if not os.path.exists(self.save_directory):
+            os.makedirs(self.save_directory)
+
+        #global cam_led
+        #cam_led = LED(CAMERA_LED_PIN, active_high=False, initial_value=False)
+
+        for trigger in self.config["trigger"]:
+            self.shepherd.scheduler.add_job(self.camera_job, 'cron',
+                                            hour=trigger["hour"],
+                                            minute=trigger["minute"],
+                                            second=trigger["second"])
+
+    def setup_other_modules(self):
+        pass
+
+    def camera_job(self):
+        self.interface.hook_pre_cam()
+
+        print("Running camera...")
+        stream = io.BytesIO()
+        with PiCamera() as picam:
+            picam.resolution = (3280, 2464)
+            picam.start_preview()
+            time.sleep(2)
+            picam.capture(stream, format='jpeg')
+        # "Rewind" the stream to the beginning so we can read its content
+        image_time = datetime.now()
+
+        stream.seek(0)
+        newimage = Image.open(stream)
+        try:
+            fnt = ImageFont.truetype(overlayfont, 50)
+        except IOError:
+            fnt = ImageFont.load_default()
+
+        draw = ImageDraw.Draw(newimage)
+
+        image_text = image_time.strftime("%Y-%m-%d %H:%M:%S")
+        if self.config["append_id"]:
+            image_text = image_text + " " + self.shepherd.id
+        image_text = image_text + self.config["append_text"]
+
+        draw.text((50, newimage.height-100), image_text, font=fnt,
+                  fill=(255, 255, 255, 200))
+
+        image_filename = image_time.strftime("%Y-%m-%d %H-%M-%S")
+        if self.config["append_id"]:
+            image_filename = image_filename + " " + self.shepherd.id
+
+        image_filename = image_filename + self.config["append_text"] + ".jpg"
+        image_filename = os.path.join(self.save_directory, image_filename)
+        newimage.save(image_filename+".writing", "JPEG")
+        os.rename(image_filename+".writing", image_filename)
+
+        if self.config["upload_images"]:
+            self.modules.uploader.move_to_bucket(image_filename,
+                                                 self.config["upload_bucket"])
+        self.interface.hook_post_cam()
+
+
+
+if __name__ == "__main__":
+    pass
+    # print("main")
+    # main(sys.argv[1:])

shepherd/modules/uploader.py

@@ -0,0 +1,210 @@
+import shutil
+import os
+import threading
+import paramiko
+import shepherd.config
+import shepherd.module
+# configdef = shepherd.config.definition()
+
+# Can either import shepherd.config here, and call a function to build a config_def
+# or can leave a config_def entry point.
+# probably go with entry point, to stay consistent with the module
+
+
+class UploaderConfDef(shepherd.config.ConfDefinition):
+    def __init__(self):
+        super().__init__()
+        dests = self.add_def('destination', shepherd.config.TableArrayDef())
+        dests.add_def('name', shepherd.config.StringDef())
+        dests.add_def('protocol', shepherd.config.StringDef())
+        dests.add_def('address', shepherd.config.StringDef(optional=True))
+        dests.add_def('port', shepherd.config.IntDef(optional=True))
+        dests.add_def('path', shepherd.config.StringDef(optional=True))
+        dests.add_def('username', shepherd.config.StringDef(optional=True))
+        dests.add_def('password', shepherd.config.StringDef(optional=True))
+        dests.add_def('keyfile', shepherd.config.StringDef(default="", optional=True))
+        dests.add_def('add_id_to_path', shepherd.config.BoolDef(default=True, optional=True))
+
+        buckets = self.add_def('bucket', shepherd.config.TableArrayDef())
+        buckets.add_def('name', shepherd.config.StringDef())
+        buckets.add_def('open_link_on_new', shepherd.config.BoolDef())
+        buckets.add_def('opportunistic', shepherd.config.BoolDef(default=True, optional=True))
+        buckets.add_def('keep_copy', shepherd.config.BoolDef())
+        buckets.add_def('destination', shepherd.config.StringDef())
+
+
+# on server side, we want to be able to list commands that a module responds to
+# without actually instantiating the module class. Add command templates into
+# the conf_def, than attach to them in the interface? Was worried about having
+# "two sources of truth", but you already need to match the conf_def to the
+# name where you access the value in the module. Could have add_command, which
+# you then add standard conf_def subclasses to, to reuse validation and server
+# form generation logic...
+
+
+class UploaderInterface(shepherd.module.Interface):
+    def __init__(self, module):
+        super().__init__(module)
+        # self.add_command("trigger", self.module.camera_job)
+    def _get_bucket_path(self, bucket_name):
+        return self._module.buckets[bucket_name].path
+
+    def move_to_bucket(self, filepath, bucket_name):
+        dest_path = os.path.join(self._get_bucket_path(bucket_name),
+                                 os.path.basename(filepath))
+        temp_dest_path = dest_path + ".writing"
+        shutil.move(filepath, temp_dest_path)
+        os.rename(temp_dest_path, dest_path)
+
+        # notify bucket to check for new files
+        self._module.buckets[bucket_name].newfile_event.set()
+
+
+class Destination():
+    def __init__(self, config, core_interface):
+        self.config = config
+        self.shepherd = core_interface
+        self.sendlist_condition = threading.Condition()
+        self.send_list = []
+
+        self.thread = threading.Thread(target=self._send_files)
+        self.thread.start()
+
+    # Override this in subclasses, implementing the actual upload process.
+    # Return true on success, false on failure.
+    def upload(self, filepath, suffix):
+        print ("Dummy uploading "+filepath)
+        return True
+
+    def add_files_to_send(self, file_path_list):
+        self.sendlist_condition.acquire()
+        for file_path in file_path_list:
+            if file_path not in self.send_list:
+                self.send_list.append(file_path)
+                self.sendlist_condition.notify()
+
+        self.sendlist_condition.release()
+        
+    def _file_available(self):
+        return len(self.send_list) > 0
+
+    def _send_files(self):
+        while True:
+            self.sendlist_condition.acquire()
+            # this drops through immediately if there is something to send, otherwise waits
+            self.sendlist_condition.wait_for(self._file_available)
+            file_to_send = self.send_list.pop(0)
+            os.rename(file_to_send, file_to_send+".uploading")
+            self.sendlist_condition.release()
+
+            # Rename uploaded file to end with ".uploaded" on success, or back
+            # to original path on failure.
+            try:
+                self.upload(file_to_send, ".uploading")
+                os.rename(file_to_send+".uploading", file_to_send+".uploaded")
+            except:
+                os.rename(file_to_send+".uploading", file_to_send)
+                self.send_list.append(file_to_send)
+
+
+class SFTPDestination(Destination):
+    def upload(self, filepath, suffix):
+        with paramiko.Transport((self.config["address"],
+                                        self.config["port"])) as transport:
+            transport.connect(username=self.config["username"],
+                              password=self.config["password"])
+            with paramiko.SFTPClient.from_transport(transport) as sftp:
+                print("Uploading "+filepath+" to "+self.config["address"]+" via SFTP")
+                if self.config["add_id_to_path"]:
+                    destdir = os.path.join(self.config["path"],
+                                           self.shepherd.id)
+                else:
+                    destdir = self.config["path"]
+
+                try:
+                    sftp.listdir(destdir)
+                except IOError:
+                    print("Creating remot dir:" + destdir)
+                    sftp.mkdir(destdir)
+
+                print("Target dir:"+destdir)
+                sftp.put(filepath+suffix,
+                         os.path.join(destdir, os.path.basename(filepath)))
+
+
+class Bucket():
+    def __init__(self, name, open_link_on_new, opportunistic, keep_copy,
+                 destination, core_interface, path=None, old_path=None):
+        self.newfile_event = threading.Event()
+        self.newfile_event.set()
+
+        self.destination = destination
+
+        self.shepherd = core_interface
+        self.path = path
+        if self.path is None:
+            self.path = os.path.join(self.shepherd.root_dir, name)
+        if not os.path.exists(self.path):
+            os.makedirs(self.path)
+
+        if keep_copy:
+            self.old_path = old_path
+            if self.old_path is None:
+                self.old_path = os.path.join(self.shepherd.root_dir, name + "_old")
+            if not os.path.exists(self.old_path):
+                os.makedirs(self.old_path)
+
+        self.thread = threading.Thread(target=self._check_files)
+        self.thread.start()
+
+    def _check_files(self):
+
+        while True:
+            self.newfile_event.wait(timeout=10)
+            self.newfile_event.clear()
+            bucket_files = []
+            for item in os.listdir(self.path):
+                item_path = os.path.join(self.path, item)
+                if (os.path.isfile(item_path) and
+                   (not item.endswith(".writing")) and
+                   (not item.endswith(".uploading")) and
+                   (not item.endswith(".uploaded"))):
+                    bucket_files.append(item_path)
+
+            if bucket_files:
+                self.destination.add_files_to_send(bucket_files)
+
+
+class UploaderModule(shepherd.module.Module):
+    conf_def = UploaderConfDef()
+
+    def __init__(self, config, core_interface):
+        super().__init__(config, core_interface)
+        
+        print("Uploader config:")
+        print(self.config)
+        
+        self.interface = UploaderInterface(self)
+        
+        self.destinations = {}
+        self.buckets = {}
+
+        for dest_conf in self.config["destination"]:
+            if dest_conf["protocol"] == "sftp":
+                self.destinations[dest_conf["name"]] = SFTPDestination(dest_conf, core_interface)
+            else:
+                self.destinations[dest_conf["name"]] = Destination(dest_conf, core_interface)
+
+        for bucketconf in self.config["bucket"]:
+            bucketconf["destination"] = self.destinations[bucketconf["destination"]]
+            self.buckets[bucketconf["name"]] = Bucket(
+                **bucketconf, core_interface=self.shepherd)
+
+    def init_other_modules(self, interfaces): # pylint: disable=W0235
+        super().init_other_modules(interfaces)
+
+
+if __name__ == "__main__":
+    pass
+    #print("main")
+    #main(sys.argv[1:])