From 39170590a9d5f7ea6e8df8f592c48935325a821e Mon Sep 17 00:00:00 2001 From: Sammy Joseph <101523651+Sammyjoseph999@users.noreply.github.com> Date: Fri, 19 Jun 2026 01:35:56 +0300 Subject: [PATCH] feat: --models flag to fetch/save all NEX-GDDP models in one command (#130) (#134) The fetch_data CLIs accepted only a single --model; passing more than one errored. Add a --models option (comma-separated list, or 'all' for every available model) to source_data.py, transform_data.py, and preprocess_data.py. Each requested model is fetched in turn and written to its own file (output stem + _); 'all' expands to the 16 CMIP6 models. --model still works for the single-model case; --models overrides it. Unknown model names are rejected with the available list. resolve_models()/_suffix_path() live in source_data.py and are reused by transform_data.py; preprocess_data.py carries an equivalent resolver. --- .../preprocess_data/preprocess_data.py | 66 ++++++++++++--- .../fetch_data/source_data/source_data.py | 82 ++++++++++++++----- .../transform_data/transform_data.py | 49 ++++++----- 3 files changed, 146 insertions(+), 51 deletions(-) diff --git a/climate_tookit/fetch_data/preprocess_data/preprocess_data.py b/climate_tookit/fetch_data/preprocess_data/preprocess_data.py index d9d6fb0..54a17d8 100644 --- a/climate_tookit/fetch_data/preprocess_data/preprocess_data.py +++ b/climate_tookit/fetch_data/preprocess_data/preprocess_data.py @@ -17,6 +17,31 @@ from transform_data import transform_data from sources.utils.models import ClimateVariable, ClimateDataset +from sources.nex_gddp import AVAILABLE_MODELS as NEX_GDDP_MODELS + + +def resolve_models(model, models): + """Resolve --model/--models into a list. 'all' = every NEX-GDDP model; + returns [None] when no model was requested (non-NEX-GDDP sources).""" + if models: + spec = models.strip() + if spec.lower() == 'all': + return list(NEX_GDDP_MODELS) + names = [m.strip() for m in spec.split(',') if m.strip()] + unknown = [m for m in names if m not in NEX_GDDP_MODELS] + if unknown: + raise ValueError( + f"Unknown model(s): {', '.join(unknown)}. " + f"Available: {', '.join(NEX_GDDP_MODELS)}" + ) + return names + return [model] + + +def _suffix_path(path, suffix): + """Insert '_' before the file extension.""" + stem, ext = os.path.splitext(path) + return f"{stem}_{suffix}{ext}" def clean_climate_data(df: pd.DataFrame) -> pd.DataFrame: @@ -174,6 +199,10 @@ def save_output(data, output_path, fmt): parser.add_argument("--start", type=str) parser.add_argument("--end", type=str) parser.add_argument("--model", type=str) + parser.add_argument("--models", type=str, default=None, + help="NEX-GDDP only. Comma-separated models, or 'all'. " + "Saves one file per model (output stem + _). " + "Overrides --model.") parser.add_argument("--scenario", type=str) parser.add_argument("-o", "--output", default=None) parser.add_argument( @@ -188,20 +217,31 @@ def save_output(data, output_path, fmt): date_from = date.fromisoformat(args.start) if args.start else None date_to = date.fromisoformat(args.end) if args.end else None - df = preprocess_data( - source=args.source, - location_coord=location_coord, - date_from=date_from, - date_to=date_to, - model=args.model, - scenario=args.scenario - ) + try: + model_list = resolve_models(args.model, args.models) + except ValueError as exc: + print(f"Error: {exc}") + sys.exit(1) + multi = len(model_list) > 1 + + for model in model_list: + if multi: + print(f"\n=== NEX-GDDP model: {model} ===") + df = preprocess_data( + source=args.source, + location_coord=location_coord, + date_from=date_from, + date_to=date_to, + model=model, + scenario=args.scenario + ) - if args.format == "print" or not args.output: - print(df) - else: - save_output(df, args.output, args.format) - print(f"Saved to {args.output}") + if args.format == "print" or not args.output: + print(df) + else: + out_path = _suffix_path(args.output, model) if multi else args.output + save_output(df, out_path, args.format) + print(f"Saved to {out_path}") # python climate_tookit/fetch_data/preprocess_data/preprocess_data.py --source era_5 --lon 36.8 --lat -1.3 --start 2020-01-01 --end 2020-03-05 diff --git a/climate_tookit/fetch_data/source_data/source_data.py b/climate_tookit/fetch_data/source_data/source_data.py index c97500a..a3a1f1f 100644 --- a/climate_tookit/fetch_data/source_data/source_data.py +++ b/climate_tookit/fetch_data/source_data/source_data.py @@ -15,6 +15,7 @@ from sources.tamsat import DownloadTAMSAT from sources.nasa_power import DownloadData as DownloadNASA from sources.nex_gddp import DownloadData as DownloadNEXGDDP +from sources.nex_gddp import AVAILABLE_MODELS as NEX_GDDP_MODELS from sources.soil_grid import DownloadData as DownloadSoilGrid from sources.utils.models import ClimateDataset, ClimateVariable, SoilVariable, Location from sources.utils.settings import Settings @@ -109,6 +110,31 @@ def save_output(data, output_path, fmt): else: raise ValueError(fmt) +def resolve_models(model, models): + """Resolve the requested model list from --model/--models. + + Returns a list of model names (or [None] when no model was requested, e.g. + for non-NEX-GDDP sources). 'all' expands to every available NEX-GDDP model. + """ + if models: + spec = models.strip() + if spec.lower() == 'all': + return list(NEX_GDDP_MODELS) + names = [m.strip() for m in spec.split(',') if m.strip()] + unknown = [m for m in names if m not in NEX_GDDP_MODELS] + if unknown: + raise ValueError( + f"Unknown model(s): {', '.join(unknown)}. " + f"Available: {', '.join(NEX_GDDP_MODELS)}" + ) + return names + return [model] + +def _suffix_path(path, suffix): + """Insert '_' before the file extension (foo.csv -> foo_GFDL.csv).""" + stem, ext = os.path.splitext(path) + return f"{stem}_{suffix}{ext}" + def main(): parser = argparse.ArgumentParser(description='Download climate data') parser.add_argument('--lon', type=float, required=True) @@ -117,7 +143,13 @@ def main(): parser.add_argument('--variables', required=True) parser.add_argument('--from', dest='date_from', required=True) parser.add_argument('--to', dest='date_to', required=True) - parser.add_argument('--model', default=None) + parser.add_argument('--model', default=None, + help='Single NEX-GDDP model (e.g. GFDL-ESM4).') + parser.add_argument('--models', default=None, + help="NEX-GDDP only. Comma-separated models, or 'all' " + "for every available model. Fetches each model and " + "saves one file per model (output stem + _). " + "Overrides --model.") parser.add_argument('--scenario', default=None) parser.add_argument('--output', '-o', default=None) parser.add_argument( @@ -157,24 +189,36 @@ def main(): settings = Settings.load() - source_data = SourceData( - location_coord=(args.lat, args.lon), - variables=variables, - source=source, - date_from_utc=date_from, - date_to_utc=date_to, - settings=settings, - model=args.model, - scenario=args.scenario - ) - - climate_data = source_data.download() - - if args.format == "print" or not args.output: - print(climate_data.to_string()) - else: - save_output(climate_data, args.output, args.format) - print(f"Saved to {args.output}") + try: + model_list = resolve_models(args.model, args.models) + except ValueError as exc: + print(f"Error: {exc}") + return 1 + multi = len(model_list) > 1 + + for model in model_list: + if multi: + print(f"\n=== NEX-GDDP model: {model} ===") + source_data = SourceData( + location_coord=(args.lat, args.lon), + variables=variables, + source=source, + date_from_utc=date_from, + date_to_utc=date_to, + settings=settings, + model=model, + scenario=args.scenario + ) + + climate_data = source_data.download() + + if args.format == "print" or not args.output: + print(climate_data.to_string()) + else: + # One file per model when multiple are requested. + out_path = _suffix_path(args.output, model) if multi else args.output + save_output(climate_data, out_path, args.format) + print(f"Saved to {out_path}") return 0 diff --git a/climate_tookit/fetch_data/transform_data/transform_data.py b/climate_tookit/fetch_data/transform_data/transform_data.py index 9b8396a..5bf052b 100644 --- a/climate_tookit/fetch_data/transform_data/transform_data.py +++ b/climate_tookit/fetch_data/transform_data/transform_data.py @@ -5,7 +5,7 @@ sys.path.append(os.path.join(os.path.dirname(__file__), "..", "source_data")) -from source_data import SourceData +from source_data import SourceData, resolve_models, _suffix_path from sources.utils.models import ClimateVariable, ClimateDataset, SoilVariable from sources.utils.settings import Settings @@ -171,6 +171,10 @@ def save_output(data, output_path, fmt): parser.add_argument("--start", type=str) parser.add_argument("--end", type=str) parser.add_argument("--model", type=str) + parser.add_argument("--models", type=str, default=None, + help="NEX-GDDP only. Comma-separated models, or 'all'. " + "Saves one file per model (output stem + _). " + "Overrides --model.") parser.add_argument("--scenario", type=str) parser.add_argument("-o", "--output", default=None) parser.add_argument("--format", choices=["csv", "json", "print"], default="print") @@ -186,13 +190,20 @@ def save_output(data, output_path, fmt): date_from = date.fromisoformat(args.start) if args.start else None date_to = date.fromisoformat(args.end) if args.end else None + try: + model_list = resolve_models(args.model, args.models) + except ValueError as exc: + print(f"Error: {exc}") + sys.exit(1) + multi = len(model_list) > 1 + errors = validate_inputs( args.source, args.lat, args.lon, date_from, date_to, - args.model, + model_list[0], args.scenario, ) @@ -202,24 +213,24 @@ def save_output(data, output_path, fmt): print(f" - {err}") sys.exit(1) - location_coord = (args.lat, args.lon) - date_from = date.fromisoformat(args.start) if args.start else None - date_to = date.fromisoformat(args.end) if args.end else None - - df = transform_data( - source=args.source, - location_coord=location_coord, - date_from=date_from, - date_to=date_to, - model=args.model, - scenario=args.scenario, - ) + for model in model_list: + if multi: + print(f"\n=== NEX-GDDP model: {model} ===") + df = transform_data( + source=args.source, + location_coord=location_coord, + date_from=date_from, + date_to=date_to, + model=model, + scenario=args.scenario, + ) - if args.format == "print" or not args.output: - print(df) - else: - save_output(df, args.output, args.format) - print(f"Saved to {args.output}") + if args.format == "print" or not args.output: + print(df) + else: + out_path = _suffix_path(args.output, model) if multi else args.output + save_output(df, out_path, args.format) + print(f"Saved to {out_path}") # python climate_tookit/fetch_data/transform_data/transform_data.py --source era_5 --lon 36.817223 --lat -1.286389 --start 2020-01-01 --end 2020-03-05