From 680b7ad3a35ca85db1dd5332884fed58e1b0261f Mon Sep 17 00:00:00 2001 From: novirium Date: Wed, 11 Sep 2019 07:56:07 +0800 Subject: [PATCH] Add control comms --- shepherd/config.py | 4 +-- shepherd/control.py | 56 ++++++++++++++++++++++++++++----- shepherd/core.py | 9 +++++- shepherd/plugins/scout/scout.py | 14 +++++++++ test.toml | 4 +++ 5 files changed, 77 insertions(+), 10 deletions(-) diff --git a/shepherd/config.py b/shepherd/config.py index 241d8ed..ab7f4fd 100644 --- a/shepherd/config.py +++ b/shepherd/config.py @@ -81,7 +81,7 @@ class IntDef(_ConfigDefinition): @freezedryable class StringDef(_ConfigDefinition): - def __init__(self, default=None, minlength=None, maxlength=None, + def __init__(self, default="", minlength=None, maxlength=None, optional=False, helptext=""): super().__init__(default, optional, helptext) self.minlength = minlength @@ -89,7 +89,7 @@ class StringDef(_ConfigDefinition): def validate(self, value): if not isinstance(value, str): - raise InvalidConfigError("Config value must be a string") + raise InvalidConfigError(F"Config value must be a string and is {value}") if self.minlength is not None and len(value) < self.minlength: raise InvalidConfigError("Config string length must be >= " + str(self.minlength)) diff --git a/shepherd/control.py b/shepherd/control.py index cd1695d..2324fca 100644 --- a/shepherd/control.py +++ b/shepherd/control.py @@ -4,7 +4,9 @@ import subprocess import requests import threading import json +from urllib.parse import urlparse, urlunparse, urljoin +import shepherd.plugin # Check for shepherd.new file in edit conf dir. If there, # or if no shepherd.id file can be found, generate a new one. # For now, also attempt to delete /var/lib/zerotier-one/identity.public and identity.secret @@ -14,14 +16,18 @@ import json #Start new thread, and push ID and core config to api.shepherd.distreon.net/client/update client_id = None +control_url = None +api_key = None -def _update_job(core_config): - payload = {"client_id":client_id, "core_config":core_config} - json_string = json.dumps(payload) +def _update_job(core_config, plugin_config): + payload = {"client_id":client_id, "core_config":core_config,"plugin_config":plugin_config} + #json_string = json.dumps(payload) try: - r = requests.post('http://api.shepherd.distreon.net/client/update', data=json_string) + # Using the json arg rather than json.dumps ourselves automatically sets the Content-Type + # header to application/json, which Flask expects to work correctly + r = requests.post(control_url, json=payload, auth=(client_id, api_key)) except requests.exceptions.ConnectionError: - pass + raise def generate_new_zerotier_id(): print("Removing old Zerotier id files") @@ -41,14 +47,34 @@ def generate_new_id(root_dir): f.write(client_id) generate_new_zerotier_id() -def init_control(core_config): +def init_control(core_config, plugin_config): global client_id + global control_url + global api_key + root_dir = os.path.expanduser(core_config["root_dir"]) editconf_dir = os.path.dirname(os.path.expanduser(core_config["conf_edit_path"])) + #Some weirdness with URL parsing means that by default most urls (like www.google.com) + # get treated as relative + # https://stackoverflow.com/questions/53816559/python-3-netloc-value-in-urllib-parse-is-empty-if-url-doesnt-have + + control_url = core_config["control_server"] + if "//" not in control_url: + control_url = "//"+control_url + control_url = urlunparse(urlparse(control_url)._replace(scheme="https")) + control_url = urljoin(control_url, "/client/update") + print(F"Control url is: {control_url}") + + api_key = core_config["api_key"] + if os.path.isfile(os.path.join(editconf_dir, "shepherd.new")): generate_new_id(root_dir) os.remove(os.path.join(editconf_dir, "shepherd.new")) + print(F"Config hostname: {core_config['hostname']}") + if not (core_config["hostname"] == ""): + print("Attempting to change hostname") + subprocess.run(["raspi-config", "nonint", "do_hostname", core_config["hostname"]]) elif not os.path.isfile(os.path.join(root_dir, "shepherd.id")): generate_new_id(root_dir) else: @@ -57,6 +83,22 @@ def init_control(core_config): print(F"Client ID is: {client_id}") - control_thread = threading.Thread(target=_update_job, args=(core_config,)) + control_thread = threading.Thread(target=_update_job, args=(core_config,plugin_config)) control_thread.start() + +def _post_logs_job(): + logs = shepherd.plugin.plugin_functions["scout"].get_logs() + measurements = shepherd.plugin.plugin_functions["scout"].get_measurements() + + + payload = {"client_id":client_id, "logs":logs, "measurements":measurements} + + try: + r = requests.post(control_url, json=payload, auth=(client_id, api_key)) + except requests.exceptions.ConnectionError: + pass + +def post_logs(): + post_logs_thread = threading.Thread(target=_post_logs_job, args=()) + post_logs_thread.start() \ No newline at end of file diff --git a/shepherd/core.py b/shepherd/core.py index baca17c..62e893d 100644 --- a/shepherd/core.py +++ b/shepherd/core.py @@ -30,11 +30,15 @@ import shepherd.control def define_core_config(confdef): confdef.add_def("id", shepherd.config.StringDef()) + confdef.add_def("hostname", shepherd.config.StringDef(default="", optional=True)) confdef.add_def("plugins", shepherd.config.StringArrayDef()) confdef.add_def("plugin_dir", shepherd.config.StringDef()) confdef.add_def("root_dir", shepherd.config.StringDef()) confdef.add_def("conf_edit_path", shepherd.config.StringDef()) + confdef.add_def("control_server", shepherd.config.StringDef()) + confdef.add_def("api_key", shepherd.config.StringDef()) + def load_config(config_path,load_editconf): # Load config from default location confman = shepherd.config.ConfigManager() @@ -109,11 +113,14 @@ def main(): (core_conf, plugin_classes, plugin_configs) = load_config(args.configfile, not args.noedit) if args.test is None: - shepherd.control.init_control(core_conf) + shepherd.control.init_control(core_conf, plugin_configs) shepherd.scheduler.init_scheduler(core_conf) shepherd.plugin.init_plugins(plugin_classes, plugin_configs, core_conf) + + if args.test is None: + shepherd.control.post_logs() shepherd.scheduler.restore_jobs() diff --git a/shepherd/plugins/scout/scout.py b/shepherd/plugins/scout/scout.py index 2a977bf..56d6c52 100644 --- a/shepherd/plugins/scout/scout.py +++ b/shepherd/plugins/scout/scout.py @@ -79,6 +79,8 @@ class ScoutPlugin(shepherd.plugin.Plugin): self.interface.register_function(self.set_out1) self.interface.register_function(self.set_out2) self.interface.register_function(self.test_logs) + self.interface.register_function(self.get_logs) + self.interface.register_function(self.get_measurements) self.interface.register_function(self.test) @@ -136,6 +138,18 @@ class ScoutPlugin(shepherd.plugin.Plugin): return cmd.response.arguments[0] return None + def get_logs(self): + rqst = self.msg_handler.send_request(MsgName.LOG) + if rqst.wait_for_response(): + return rqst.response.multipart_args + return None + + def get_measurements(self): + rqst = self.msg_handler.send_request(MsgName.MEASUREMENT) + if rqst.wait_for_response(): + return rqst.response.multipart_args + return None + def test_logs(self): rqst = self.msg_handler.send_request(MsgName.LOG) if rqst.wait_for_response(): diff --git a/test.toml b/test.toml index 3f00f83..5c187ab 100644 --- a/test.toml +++ b/test.toml @@ -4,6 +4,10 @@ root_dir = "~/shepherd/" conf_edit_path = "~/shepherd.toml" id = "testnode" + hostname = "shepherd-test" + control_server = "api.shepherd.distreon.net" + #control_server = "127.0.0.1:5000" + api_key = "v2EgvYzx79c8fCP4P7jlWxTZ3pc" [scout] boardver = "3" serialport = "/dev/ttyUSB0"