Free up -c for config and make config audits work on fresh clones

Utils/AuditConfig.py

@@ -3,7 +3,9 @@
 import re
 import argparse
 import os
+import subprocess
 import sys
+import tempfile
 
 
 # Map FileNotFoundError to IOError in Python 2 as it does not exist
@@ -63,6 +65,30 @@ def extractConfigOptions(file_path):
     return options
 
 
+def readTemplateFromGit():
+    """ Read the pristine .config tracked in the repository using git.
+
+    .configTemplate is normally created by RMS_Update.sh, so it does not exist on a fresh clone or on
+    a station that never updated. The repository's own .config is the same content, so it can be used
+    as the template instead.
+
+    Returns:
+        [str or None] Contents of the repository .config, or None if git is unavailable.
+    """
+
+    rms_root_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+
+    try:
+        with open(os.devnull, 'w') as devnull:
+            content = subprocess.check_output(['git', 'show', 'HEAD:.config'], cwd=rms_root_dir,
+                stderr=devnull)
+
+        return content.decode('utf-8', 'replace')
+
+    except Exception:
+        return None
+
+
 def parseConfigFile(config_path):
     """Parse the .config file, excluding specific words
 
@@ -143,32 +169,64 @@ def compareConfigs(config_path, template_path, configreader_path, dev_report=Fal
         dev_report = True
         print("Error loading .config file: {}".format(e))
 
+    # The temporary template extracted from git (if used) is removed in the finally block below
+    template_tmp_path = None
     try:
-        validatePath(template_path, ".configTemplate")
-        template_file_options = parseConfigFile(template_path)
-        found_template = True
-    except (ValueError, FileNotFoundError) as e:
-        dev_report = True
-        print("Error loading .configTemplate file: {}".format(e))
-
-    try:
-        validatePath(configreader_path, "ConfigReader.py")
-        configreader_file_options = extractConfigOptions(configreader_path)
-        found_configreader = True
-    except (ValueError, FileNotFoundError) as e:
-        dev_report = True
-        print("Error loading ConfigReader.py file: {}".format(e))
-
-    # Find missing and extra options
-    missing_in_config_wrt_template = template_file_options - config_file_options if found_template and found_config else set()
-    missing_in_config_wrt_cr = configreader_file_options - config_file_options if found_configreader and found_config else set()
-    missing_in_template_wrt_cr = configreader_file_options - template_file_options if found_configreader and found_template else set()
-    extra_in_config = config_file_options - configreader_file_options if found_config and found_configreader else set()
-    extra_in_template = template_file_options - configreader_file_options if found_template and found_configreader else set()
-
-    # Find commented out options (only if respective files are found)
-    commented_options_in_config = checkCommentedOptions(config_path, missing_in_config_wrt_cr) if found_config else set()
-    commented_options_in_template = checkCommentedOptions(template_path, missing_in_template_wrt_cr) if found_template else set()
+        try:
+            validatePath(template_path, ".configTemplate")
+            template_file_options = parseConfigFile(template_path)
+            found_template = True
+        except (ValueError, FileNotFoundError) as e:
+
+            # Fall back to the pristine .config tracked in git, so fresh clones which don't have a
+            # .configTemplate yet still get a complete comparison
+            template_content = readTemplateFromGit()
+
+            if template_content is not None:
+
+                tmp_file = tempfile.NamedTemporaryFile(mode='w', suffix='.configTemplate', delete=False)
+                tmp_file.write(template_content)
+                tmp_file.close()
+
+                template_tmp_path = tmp_file.name
+                template_path = template_tmp_path
+
+                template_file_options = parseConfigFile(template_path)
+                found_template = True
+
+                print("Note: .configTemplate not found, using the repository default .config (via git) "
+                      "as the template.")
+
+            else:
+                dev_report = True
+                print("Error loading .configTemplate file: {}".format(e))
+
+        try:
+            validatePath(configreader_path, "ConfigReader.py")
+            configreader_file_options = extractConfigOptions(configreader_path)
+            found_configreader = True
+        except (ValueError, FileNotFoundError) as e:
+            dev_report = True
+            print("Error loading ConfigReader.py file: {}".format(e))
+
+        # Find missing and extra options
+        missing_in_config_wrt_template = template_file_options - config_file_options if found_template and found_config else set()
+        missing_in_config_wrt_cr = configreader_file_options - config_file_options if found_configreader and found_config else set()
+        missing_in_template_wrt_cr = configreader_file_options - template_file_options if found_configreader and found_template else set()
+        extra_in_config = config_file_options - configreader_file_options if found_config and found_configreader else set()
+        extra_in_template = template_file_options - configreader_file_options if found_template and found_configreader else set()
+
+        # Find commented out options (only if respective files are found)
+        commented_options_in_config = checkCommentedOptions(config_path, missing_in_config_wrt_cr) if found_config else set()
+        commented_options_in_template = checkCommentedOptions(template_path, missing_in_template_wrt_cr) if found_template else set()
+
+    finally:
+        # Clean up the temporary template extracted from git
+        if template_tmp_path is not None:
+            try:
+                os.remove(template_tmp_path)
+            except OSError:
+                pass
 
     # Remove commented out options from missing
     missing_in_config_wrt_template -= commented_options_in_config
@@ -274,13 +332,18 @@ def compareConfigs(config_path, template_path, configreader_path, dev_report=Fal
     # Init the command line arguments parser
     arg_parser = argparse.ArgumentParser(description="Audit .config and optionally .configTemplate files.")
 
-    arg_parser.add_argument("config_path", nargs='?', default="./.config", help="Path to the .config file")
+    # Resolve defaults relative to the RMS repository root so the script works from any directory
+    rms_root_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+
+    arg_parser.add_argument("config_path", nargs='?', default=os.path.join(rms_root_dir, ".config"),
+                        help="Path to the .config file (default: the one in the RMS repository root)")
 
-    arg_parser.add_argument("--template", default="./.configTemplate",
-                        help="Path to .configTemplate (default: ./.configTemplate)")
+    arg_parser.add_argument("--template", default=os.path.join(rms_root_dir, ".configTemplate"),
+                        help="Path to .configTemplate (default: the one in the RMS repository root; "
+                             "if missing, the repository default .config is used via git)")
 
-    arg_parser.add_argument("--configreader", default="./RMS/ConfigReader.py",
-                        help="Path to ConfigReader.py (default: ./RMS/ConfigReader.py)")
+    arg_parser.add_argument("--configreader", default=os.path.join(rms_root_dir, "RMS", "ConfigReader.py"),
+                        help="Path to ConfigReader.py (default: the one in the RMS repository root)")
 
     arg_parser.add_argument('-d', '--dev', action="store_true", help="""Audit template file. """)
 

Utils/FOVSkyMap.py

@@ -479,7 +479,7 @@ def plotFOVSkyMap(platepars, configs, out_dir, north_up=False, show_pointing=Fal
     arg_parser.add_argument('-i', '--show_ip', dest='show_ip', default=False, action="store_true",
                             help="Show ip address of the camera.")
 
-    arg_parser.add_argument('-c', '--show_coordinates', dest='show_coordinates', default=False, action="store_true",
+    arg_parser.add_argument('--show_coordinates', dest='show_coordinates', default=False, action="store_true",
                             help="Show coordinates of the camera.")
 
     arg_parser.add_argument('-s', '--show_sun', dest='show_sun', default=False, action="store_true",

Utils/FluxAuto.py

@@ -960,7 +960,7 @@ def fluxAutoRun(config, data_path, ref_dt, days_prev=2, days_next=1, all_prev_ye
     arg_parser.add_argument('-o', '--outdir', metavar='OUTPUT_DIRECTORY', type=str,
         help="Path to a directory where the plots will be saved. If not given, the data directory will be used.")
 
-    arg_parser.add_argument('-c', '--csvdir', metavar='CSV_DIRECTORY', type=str,
+    arg_parser.add_argument('--csvdir', metavar='CSV_DIRECTORY', type=str,
         help="Path to a directory where the CSV files will be saved. If not given, the output directory will be used.")
 
     arg_parser.add_argument('-i', '--indexdir', metavar='INDEX_DIRECTORY', type=str,

Utils/MigrateConfig.py

@@ -23,12 +23,13 @@
 import os
 import re
 import argparse
+import tempfile
 from datetime import datetime
 import shutil
 import platform
 from io import StringIO
 
-from Utils.AuditConfig import extractConfigOptions
+from Utils.AuditConfig import extractConfigOptions, readTemplateFromGit
 from RMS.Misc import getRmsRootDir
 
 # Get ConfigReader.py path dynamically
@@ -378,56 +379,85 @@ def getSystemInfo():
 
     if args.template:
         template_config_file = args.template
-
-    print("\nTemplate: {}".format(template_config_file))
 
-    # assume default input
-    original_config_files = [os.path.join(rms_root_dir, ".config")]
+    # If the default template doesn't exist yet (it's only created by RMS_Update.sh), fall back to
+    # the pristine .config tracked in git. A template given explicitly with -t is never substituted -
+    # if that path is wrong, updateConfig() fails with an error as before.
+    template_tmp_path = None
+    if (not args.template) and (not os.path.exists(template_config_file)):
 
-    # if multi-cam find and assume those
-    stations_dir = os.path.expanduser("~/source/Stations")
-    if os.path.isdir(stations_dir):
-        for d in os.listdir(stations_dir):
-            f = os.path.join(stations_dir, d, ".config")
-            if os.path.isfile(f):                 # skip broken/missing configs
-                original_config_files.append(f)
-    print("Multi-cam count: {}".format(len(original_config_files) - 1))
+        template_content = readTemplateFromGit()
 
-    # if specified assume
-    if args.input:
-        original_config_files = [args.input]
+        if template_content is not None:
 
-    # process each input config
-    for orig_config in original_config_files:
-        log_buffer = StringIO()
-        out_file_name, station_id = updateConfig(orig_config, template_config_file, args, backup=args.update)
+            tmp_file = tempfile.NamedTemporaryFile(mode='w', suffix='.configTemplate', delete=False)
+            tmp_file.write(template_content)
+            tmp_file.close()
 
-        if args.update:
-            try:
-                shutil.copy(out_file_name, orig_config)
-                os.remove(out_file_name)
-                print("Updated\n")
+            template_tmp_path = tmp_file.name
+            template_config_file = template_tmp_path
 
-                log_file = os.path.join(os.path.dirname(orig_config), "{}_MigrateConfig.log".format(station_id))
-                with open(log_file, "a") as log:
-                    log.write("\n=== Migration Log: {} ===\n\n".format(datetime.now().strftime("%Y-%m-%d %H:%M:%S")))
-                    log.write(log_buffer.getvalue())
-                    log.write("\nMigration applied successfully.\n")
+            print("\nNote: .configTemplate not found, using the repository default .config (via git) "
+                  "as the template.")
 
-                log_buffer.close()
+    print("\nTemplate: {}".format(template_config_file))
 
-                print("Updated\n")
+    try:
+        # assume default input
+        original_config_files = [os.path.join(rms_root_dir, ".config")]
+
+        # if multi-cam find and assume those
+        stations_dir = os.path.expanduser("~/source/Stations")
+        if os.path.isdir(stations_dir):
+            for d in os.listdir(stations_dir):
+                f = os.path.join(stations_dir, d, ".config")
+                if os.path.isfile(f):                 # skip broken/missing configs
+                    original_config_files.append(f)
+        print("Multi-cam count: {}".format(len(original_config_files) - 1))
+
+        # if specified assume
+        if args.input:
+            original_config_files = [args.input]
+
+        # process each input config
+        for orig_config in original_config_files:
+            log_buffer = StringIO()
+            out_file_name, station_id = updateConfig(orig_config, template_config_file, args, backup=args.update)
+
+            if args.update:
+                try:
+                    shutil.copy(out_file_name, orig_config)
+                    os.remove(out_file_name)
+                    print("Updated\n")
+
+                    log_file = os.path.join(os.path.dirname(orig_config), "{}_MigrateConfig.log".format(station_id))
+                    with open(log_file, "a") as log:
+                        log.write("\n=== Migration Log: {} ===\n\n".format(datetime.now().strftime("%Y-%m-%d %H:%M:%S")))
+                        log.write(log_buffer.getvalue())
+                        log.write("\nMigration applied successfully.\n")
+
+                    log_buffer.close()
+
+                    print("Updated\n")
+
+                except Exception as e:
+                    print("ERROR: Update failed: {}".format(e))
+                    sys.exit(1)
+            else:
+                print("\nSaved new config to: {}".format(out_file_name))
 
-            except Exception as e:
-                print("ERROR: Update failed: {}".format(e))
-                sys.exit(1)
-        else:
-            print("\nSaved new config to: {}".format(out_file_name))
+        if not args.update:
+            print(
+                "\nAfter saving a copy of the existing .config, copy/rename new version(s) into production."
+            )
+            print("    e.g. cp ConfigNew_XXxxxx .config \n")
 
-    if not args.update:
-        print(
-            "\nAfter saving a copy of the existing .config, copy/rename new version(s) into production."
-        )
-        print("    e.g. cp ConfigNew_XXxxxx .config \n")
+    finally:
+        # Clean up the temporary template extracted from git
+        if template_tmp_path is not None:
+            try:
+                os.remove(template_tmp_path)
+            except OSError:
+                pass
 
     print("Done.\n")

Utils/RetroactiveFixup.py

@@ -34,7 +34,7 @@
     arg_parser.add_argument('date', metavar='DATE', type=str, \
         help="Reference date in the YYYYMMDD format.")
 
-    arg_parser.add_argument('-c', '--copyconfig', action="store_true", \
+    arg_parser.add_argument('--copyconfig', action="store_true", \
         help="""Copy the config from the data dir closest to the given date to older data directories. """)
 
     # Parse the command line arguments