diff --git a/Utils/AuditConfig.py b/Utils/AuditConfig.py index 20c0ed722..7bc39a3b8 100644 --- a/Utils/AuditConfig.py +++ b/Utils/AuditConfig.py @@ -3,7 +3,9 @@ import re import argparse import os +import subprocess import sys +import tempfile # Map FileNotFoundError to IOError in Python 2 as it does not exist @@ -63,6 +65,30 @@ def extractConfigOptions(file_path): return options +def readTemplateFromGit(): + """ Read the pristine .config tracked in the repository using git. + + .configTemplate is normally created by RMS_Update.sh, so it does not exist on a fresh clone or on + a station that never updated. The repository's own .config is the same content, so it can be used + as the template instead. + + Returns: + [str or None] Contents of the repository .config, or None if git is unavailable. + """ + + rms_root_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + + try: + with open(os.devnull, 'w') as devnull: + content = subprocess.check_output(['git', 'show', 'HEAD:.config'], cwd=rms_root_dir, + stderr=devnull) + + return content.decode('utf-8', 'replace') + + except Exception: + return None + + def parseConfigFile(config_path): """Parse the .config file, excluding specific words @@ -143,32 +169,64 @@ def compareConfigs(config_path, template_path, configreader_path, dev_report=Fal dev_report = True print("Error loading .config file: {}".format(e)) + # The temporary template extracted from git (if used) is removed in the finally block below + template_tmp_path = None try: - validatePath(template_path, ".configTemplate") - template_file_options = parseConfigFile(template_path) - found_template = True - except (ValueError, FileNotFoundError) as e: - dev_report = True - print("Error loading .configTemplate file: {}".format(e)) - - try: - validatePath(configreader_path, "ConfigReader.py") - configreader_file_options = extractConfigOptions(configreader_path) - found_configreader = True - except (ValueError, FileNotFoundError) as e: - dev_report = True - print("Error loading ConfigReader.py file: {}".format(e)) - - # Find missing and extra options - missing_in_config_wrt_template = template_file_options - config_file_options if found_template and found_config else set() - missing_in_config_wrt_cr = configreader_file_options - config_file_options if found_configreader and found_config else set() - missing_in_template_wrt_cr = configreader_file_options - template_file_options if found_configreader and found_template else set() - extra_in_config = config_file_options - configreader_file_options if found_config and found_configreader else set() - extra_in_template = template_file_options - configreader_file_options if found_template and found_configreader else set() - - # Find commented out options (only if respective files are found) - commented_options_in_config = checkCommentedOptions(config_path, missing_in_config_wrt_cr) if found_config else set() - commented_options_in_template = checkCommentedOptions(template_path, missing_in_template_wrt_cr) if found_template else set() + try: + validatePath(template_path, ".configTemplate") + template_file_options = parseConfigFile(template_path) + found_template = True + except (ValueError, FileNotFoundError) as e: + + # Fall back to the pristine .config tracked in git, so fresh clones which don't have a + # .configTemplate yet still get a complete comparison + template_content = readTemplateFromGit() + + if template_content is not None: + + tmp_file = tempfile.NamedTemporaryFile(mode='w', suffix='.configTemplate', delete=False) + tmp_file.write(template_content) + tmp_file.close() + + template_tmp_path = tmp_file.name + template_path = template_tmp_path + + template_file_options = parseConfigFile(template_path) + found_template = True + + print("Note: .configTemplate not found, using the repository default .config (via git) " + "as the template.") + + else: + dev_report = True + print("Error loading .configTemplate file: {}".format(e)) + + try: + validatePath(configreader_path, "ConfigReader.py") + configreader_file_options = extractConfigOptions(configreader_path) + found_configreader = True + except (ValueError, FileNotFoundError) as e: + dev_report = True + print("Error loading ConfigReader.py file: {}".format(e)) + + # Find missing and extra options + missing_in_config_wrt_template = template_file_options - config_file_options if found_template and found_config else set() + missing_in_config_wrt_cr = configreader_file_options - config_file_options if found_configreader and found_config else set() + missing_in_template_wrt_cr = configreader_file_options - template_file_options if found_configreader and found_template else set() + extra_in_config = config_file_options - configreader_file_options if found_config and found_configreader else set() + extra_in_template = template_file_options - configreader_file_options if found_template and found_configreader else set() + + # Find commented out options (only if respective files are found) + commented_options_in_config = checkCommentedOptions(config_path, missing_in_config_wrt_cr) if found_config else set() + commented_options_in_template = checkCommentedOptions(template_path, missing_in_template_wrt_cr) if found_template else set() + + finally: + # Clean up the temporary template extracted from git + if template_tmp_path is not None: + try: + os.remove(template_tmp_path) + except OSError: + pass # Remove commented out options from missing missing_in_config_wrt_template -= commented_options_in_config @@ -274,13 +332,18 @@ def compareConfigs(config_path, template_path, configreader_path, dev_report=Fal # Init the command line arguments parser arg_parser = argparse.ArgumentParser(description="Audit .config and optionally .configTemplate files.") - arg_parser.add_argument("config_path", nargs='?', default="./.config", help="Path to the .config file") + # Resolve defaults relative to the RMS repository root so the script works from any directory + rms_root_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + + arg_parser.add_argument("config_path", nargs='?', default=os.path.join(rms_root_dir, ".config"), + help="Path to the .config file (default: the one in the RMS repository root)") - arg_parser.add_argument("--template", default="./.configTemplate", - help="Path to .configTemplate (default: ./.configTemplate)") + arg_parser.add_argument("--template", default=os.path.join(rms_root_dir, ".configTemplate"), + help="Path to .configTemplate (default: the one in the RMS repository root; " + "if missing, the repository default .config is used via git)") - arg_parser.add_argument("--configreader", default="./RMS/ConfigReader.py", - help="Path to ConfigReader.py (default: ./RMS/ConfigReader.py)") + arg_parser.add_argument("--configreader", default=os.path.join(rms_root_dir, "RMS", "ConfigReader.py"), + help="Path to ConfigReader.py (default: the one in the RMS repository root)") arg_parser.add_argument('-d', '--dev', action="store_true", help="""Audit template file. """) diff --git a/Utils/FOVSkyMap.py b/Utils/FOVSkyMap.py index 2fedd871a..026793b5b 100644 --- a/Utils/FOVSkyMap.py +++ b/Utils/FOVSkyMap.py @@ -479,7 +479,7 @@ def plotFOVSkyMap(platepars, configs, out_dir, north_up=False, show_pointing=Fal arg_parser.add_argument('-i', '--show_ip', dest='show_ip', default=False, action="store_true", help="Show ip address of the camera.") - arg_parser.add_argument('-c', '--show_coordinates', dest='show_coordinates', default=False, action="store_true", + arg_parser.add_argument('--show_coordinates', dest='show_coordinates', default=False, action="store_true", help="Show coordinates of the camera.") arg_parser.add_argument('-s', '--show_sun', dest='show_sun', default=False, action="store_true", diff --git a/Utils/FluxAuto.py b/Utils/FluxAuto.py index d20ba91f7..58beb5241 100644 --- a/Utils/FluxAuto.py +++ b/Utils/FluxAuto.py @@ -960,7 +960,7 @@ def fluxAutoRun(config, data_path, ref_dt, days_prev=2, days_next=1, all_prev_ye arg_parser.add_argument('-o', '--outdir', metavar='OUTPUT_DIRECTORY', type=str, help="Path to a directory where the plots will be saved. If not given, the data directory will be used.") - arg_parser.add_argument('-c', '--csvdir', metavar='CSV_DIRECTORY', type=str, + arg_parser.add_argument('--csvdir', metavar='CSV_DIRECTORY', type=str, help="Path to a directory where the CSV files will be saved. If not given, the output directory will be used.") arg_parser.add_argument('-i', '--indexdir', metavar='INDEX_DIRECTORY', type=str, diff --git a/Utils/MigrateConfig.py b/Utils/MigrateConfig.py index 2128ad739..cfde23fb8 100644 --- a/Utils/MigrateConfig.py +++ b/Utils/MigrateConfig.py @@ -23,12 +23,13 @@ import os import re import argparse +import tempfile from datetime import datetime import shutil import platform from io import StringIO -from Utils.AuditConfig import extractConfigOptions +from Utils.AuditConfig import extractConfigOptions, readTemplateFromGit from RMS.Misc import getRmsRootDir # Get ConfigReader.py path dynamically @@ -378,56 +379,85 @@ def getSystemInfo(): if args.template: template_config_file = args.template - - print("\nTemplate: {}".format(template_config_file)) - # assume default input - original_config_files = [os.path.join(rms_root_dir, ".config")] + # If the default template doesn't exist yet (it's only created by RMS_Update.sh), fall back to + # the pristine .config tracked in git. A template given explicitly with -t is never substituted - + # if that path is wrong, updateConfig() fails with an error as before. + template_tmp_path = None + if (not args.template) and (not os.path.exists(template_config_file)): - # if multi-cam find and assume those - stations_dir = os.path.expanduser("~/source/Stations") - if os.path.isdir(stations_dir): - for d in os.listdir(stations_dir): - f = os.path.join(stations_dir, d, ".config") - if os.path.isfile(f): # skip broken/missing configs - original_config_files.append(f) - print("Multi-cam count: {}".format(len(original_config_files) - 1)) + template_content = readTemplateFromGit() - # if specified assume - if args.input: - original_config_files = [args.input] + if template_content is not None: - # process each input config - for orig_config in original_config_files: - log_buffer = StringIO() - out_file_name, station_id = updateConfig(orig_config, template_config_file, args, backup=args.update) + tmp_file = tempfile.NamedTemporaryFile(mode='w', suffix='.configTemplate', delete=False) + tmp_file.write(template_content) + tmp_file.close() - if args.update: - try: - shutil.copy(out_file_name, orig_config) - os.remove(out_file_name) - print("Updated\n") + template_tmp_path = tmp_file.name + template_config_file = template_tmp_path - log_file = os.path.join(os.path.dirname(orig_config), "{}_MigrateConfig.log".format(station_id)) - with open(log_file, "a") as log: - log.write("\n=== Migration Log: {} ===\n\n".format(datetime.now().strftime("%Y-%m-%d %H:%M:%S"))) - log.write(log_buffer.getvalue()) - log.write("\nMigration applied successfully.\n") + print("\nNote: .configTemplate not found, using the repository default .config (via git) " + "as the template.") - log_buffer.close() + print("\nTemplate: {}".format(template_config_file)) - print("Updated\n") + try: + # assume default input + original_config_files = [os.path.join(rms_root_dir, ".config")] + + # if multi-cam find and assume those + stations_dir = os.path.expanduser("~/source/Stations") + if os.path.isdir(stations_dir): + for d in os.listdir(stations_dir): + f = os.path.join(stations_dir, d, ".config") + if os.path.isfile(f): # skip broken/missing configs + original_config_files.append(f) + print("Multi-cam count: {}".format(len(original_config_files) - 1)) + + # if specified assume + if args.input: + original_config_files = [args.input] + + # process each input config + for orig_config in original_config_files: + log_buffer = StringIO() + out_file_name, station_id = updateConfig(orig_config, template_config_file, args, backup=args.update) + + if args.update: + try: + shutil.copy(out_file_name, orig_config) + os.remove(out_file_name) + print("Updated\n") + + log_file = os.path.join(os.path.dirname(orig_config), "{}_MigrateConfig.log".format(station_id)) + with open(log_file, "a") as log: + log.write("\n=== Migration Log: {} ===\n\n".format(datetime.now().strftime("%Y-%m-%d %H:%M:%S"))) + log.write(log_buffer.getvalue()) + log.write("\nMigration applied successfully.\n") + + log_buffer.close() + + print("Updated\n") + + except Exception as e: + print("ERROR: Update failed: {}".format(e)) + sys.exit(1) + else: + print("\nSaved new config to: {}".format(out_file_name)) - except Exception as e: - print("ERROR: Update failed: {}".format(e)) - sys.exit(1) - else: - print("\nSaved new config to: {}".format(out_file_name)) + if not args.update: + print( + "\nAfter saving a copy of the existing .config, copy/rename new version(s) into production." + ) + print(" e.g. cp ConfigNew_XXxxxx .config \n") - if not args.update: - print( - "\nAfter saving a copy of the existing .config, copy/rename new version(s) into production." - ) - print(" e.g. cp ConfigNew_XXxxxx .config \n") + finally: + # Clean up the temporary template extracted from git + if template_tmp_path is not None: + try: + os.remove(template_tmp_path) + except OSError: + pass print("Done.\n") diff --git a/Utils/RetroactiveFixup.py b/Utils/RetroactiveFixup.py index 126ac0519..1b92fce34 100644 --- a/Utils/RetroactiveFixup.py +++ b/Utils/RetroactiveFixup.py @@ -34,7 +34,7 @@ arg_parser.add_argument('date', metavar='DATE', type=str, \ help="Reference date in the YYYYMMDD format.") - arg_parser.add_argument('-c', '--copyconfig', action="store_true", \ + arg_parser.add_argument('--copyconfig', action="store_true", \ help="""Copy the config from the data dir closest to the given date to older data directories. """) # Parse the command line arguments