diff --git a/shepherd/agent/core.py b/shepherd/agent/core.py
index c027e34..077d94a 100644
--- a/shepherd/agent/core.py
+++ b/shepherd/agent/core.py
@@ -8,6 +8,8 @@ import sys
 from pathlib import Path
 import glob
 from datetime import datetime
+from types import SimpleNamespace
+from pprint import pprint
 import logging
 import chromalog
 import pkg_resources
@@ -50,14 +52,19 @@ def cli(ctx, default_config_path, local_operation, only_default_layer):
         default_config_path = sorted(glob.glob("./shepherd*.toml"))[:1]
         if default_config_path:
             default_config_path = default_config_path[0]
-            log.info(F"No default config file provided, using {default_config_path}")
+            log.info(F"No default config file provided, found {default_config_path}")
+            with open(default_config_path, 'r+') as f:
+                content = f.read()
+                if "Compiled Shepherd config" in content:
+                    log.warn("Default config file looks like it is full compiled config file"
+                             " generated by Shepherd and picked up due to accidental name match")
         else:
             log.error("No default config file provided, and no 'shepherd*.toml' could be"
                       " found in the current directory")
             sys.exit(1)
 
     layers_disabled = []
-    if local_operation:
+    if local_operation or (ctx.invoked_subcommand == "test"):
         layers_disabled.append("control")
         log.info("Running in local only mode")
 
@@ -184,24 +191,25 @@ def compile_config(confman, default_config_path, layers_disabled):
 
     # ====Default Local Config Layer====
     # This must validate to continue.
-    default_config_path = Path(default_config_path).expanduser()
+    default_config_path = Path(default_config_path).expanduser().resolve()
     try:
         load_config_layer_and_plugins(confman, default_config_path)
         log.info(F"Loaded default config layer from {default_config_path}")
     except Exception as e:
         if isinstance(e, InvalidConfigError):
-            log.error(F"Failed to load default config from {default_config_path}. {e.args[0]}")
+            log.error(
+                F"Failed to load default config from {default_config_path}. {chr(10).join(e.args)}")
         else:
             log.error(F"Failed to load default config from {default_config_path}", exc_info=True)
         sys.exit(1)
 
     # Resolve and freeze local install paths that shouldn't be changed from default config
     core_conf = confman.get_config_bundle("shepherd")
-    resolve_core_conf_paths(core_conf)
+    resolve_core_conf_paths(core_conf, default_config_path.parent)
     confman.freeze_value("shepherd", "root_dir")
     confman.freeze_value("shepherd", "plugin_dir")
     confman.freeze_value("shepherd", "custom_config_path")
-    confman.freeze_value("shepherd", "generated_config_path")
+    confman.freeze_value("shepherd", "compiled_config_path")
 
     # Pull out custom config path and save current good config
     custom_config_path = core_conf["custom_config_path"]
@@ -232,7 +240,7 @@ def compile_config(confman, default_config_path, layers_disabled):
 
     # Freeze Shepherd Control related config.
     core_conf = confman.get_config_bundle("shepherd")
-    resolve_core_conf_paths(core_conf)
+    resolve_core_conf_paths(core_conf, default_config_path.parent)
     confman.freeze_value("shepherd", "control_server")
     confman.freeze_value("shepherd", "control_api_key")
 
@@ -262,7 +270,10 @@ def compile_config(confman, default_config_path, layers_disabled):
         log.info("Shepherd Control config layer disabled")
 
     log.debug("Compiled config: %s", confman.root_config)
-    confman.dump_to_file(core_conf["generated_config_path"])
+    if core_conf["compiled_config_path"]:
+        message = F"Compiled Shepherd config at {datetime.now()}"
+        confman.dump_to_file(core_conf["compiled_config_path"], message=message)
+        log.info(F"Saved compiled config to {core_conf['compiled_config_path']}")
 
 
 # Relative pathnames here are all relative to "root_dir"
@@ -283,7 +294,7 @@ def core_confspec():
                                 " working files.")),
         ("custom_config_path", StringSpec(optional=True, helptext="Path to custom config"
                                           " layer TOML file.")),
-        ("generated_config_path", StringSpec(default="shepherd-generated.toml", optional=True,
+        ("compiled_config_path", StringSpec(default="compiled-config.toml", optional=True,
                                              helptext="Path to custom file Shepherd will generate"
                                              " to show compiled config that was used and any"
                                              " errors in validation."))
@@ -295,7 +306,7 @@ def core_confspec():
     return confspec
 
 
-def resolve_core_conf_paths(core_conf):
+def resolve_core_conf_paths(core_conf, relative_dir):
     """
     Set the cwd to ``root_dir`` and resolve other core config paths relative to that.
     ``root_dir`` itself will resolve relative to ``relative_dir``, intended to be the config
@@ -316,8 +327,9 @@ def resolve_core_conf_paths(core_conf):
     if core_conf["custom_config_path"]:
     core_conf["custom_config_path"] = str(
         Path(core_conf["custom_config_path"]).expanduser().resolve())
-    core_conf["generated_config_path"] = str(
-        Path(core_conf["generated_config_path"]).expanduser().resolve())
+    if core_conf["compiled_config_path"]:
+        core_conf["compiled_config_path"] = str(
+            Path(core_conf["compiled_config_path"]).expanduser().resolve())
 
 
 def load_config_layer_and_plugins(confman: ConfigManager, config_source):