From 1f561b9a08736696a00f70764040481444e66484 Mon Sep 17 00:00:00 2001 From: maritsandstad Date: Fri, 2 Oct 2026 13:41:54 +0200 Subject: [PATCH 1/9] First commit to update tables and grids to official, not clean --- cmip7-cmor-tables | 2 +- data/cesm_regrid_maps.yaml | 17 +++++++++++ data/cmor_dataset_noresm.json | 2 +- data/cmor_dataset_noresm3-lm.json | 51 +++++++++++++++++++++++++++++++ data/cmor_dataset_noresm3-mm.json | 51 +++++++++++++++++++++++++++++++ data/noresm_regrid_maps.yaml | 16 ++++++++++ scripts/cmor_driver.py | 49 ++++++++++++++++++++++------- scripts/convert_csv_to_yaml.py | 2 +- scripts/validate_cmor_output.py | 2 +- src/cmip7_prep/cmor_writer.py | 10 +++--- src/cmip7_prep/regrid_maps.py | 21 +++++++++++++ 11 files changed, 204 insertions(+), 19 deletions(-) create mode 100644 data/cmor_dataset_noresm3-lm.json create mode 100644 data/cmor_dataset_noresm3-mm.json diff --git a/cmip7-cmor-tables b/cmip7-cmor-tables index cf46e3c0..7d1d15a7 160000 --- a/cmip7-cmor-tables +++ b/cmip7-cmor-tables @@ -1 +1 @@ -Subproject commit cf46e3c000f0c8d798efd786ae593872f7bf49fe +Subproject commit 7d1d15a716772ad627c2dabf3d595948c2075a7c diff --git a/data/cesm_regrid_maps.yaml b/data/cesm_regrid_maps.yaml index 56cf9273..bbd67bad 100644 --- a/data/cesm_regrid_maps.yaml +++ b/data/cesm_regrid_maps.yaml @@ -20,3 +20,20 @@ resolutions: regular: conservative: cpl/gridmaps/tx2_3v2/map_t232_TO_1x1d_aave.251023.nc bilinear: cpl/gridmaps/tx2_3v2/map_t232_TO_1x1d_blin.251023.nc + + +grid_names_per_realm: + ne16: + atmos: g123 + aerosol: g123 + atmosChem: g123 + land: g123 + seaIce: g202 + landIce: g194 + ne30: + atmos: g106 + aerosol: g106 + atmosChem: g106 + land: g106 + seaIce: g106 + landIce: g106 \ No newline at end of file diff --git a/data/cmor_dataset_noresm.json b/data/cmor_dataset_noresm.json index a73fcaff..ba465dac 100644 --- a/data/cmor_dataset_noresm.json +++ b/data/cmor_dataset_noresm.json @@ -11,7 +11,7 @@ "horizontal_label": "hxy", "vertical_label": "l", "temporal_label": "tavg", - "source_id": "NorESM3", + "source_id": "NorESM3-LM", "source": "NorESM3 (2025): atmosphere: CAM; atmosChem: OsloChemSimp; ocean: BLOM; oceanBgchem: HAMOCC; sea ice: CICE6; land: CLM-FATES; aerosol: OsloAero; coupler: CMEPS.", "source_type": "AOGCM", "nominal_resolution": "100 km", diff --git a/data/cmor_dataset_noresm3-lm.json b/data/cmor_dataset_noresm3-lm.json new file mode 100644 index 00000000..d7b58993 --- /dev/null +++ b/data/cmor_dataset_noresm3-lm.json @@ -0,0 +1,51 @@ +{ + "project_id": "CMIP7", + "mip_era": "CMIP7", + "activity_id": "CMIP", + "drs_specs": "MIP-DRS7", + "institution_id": "NCC", + "institution": "NorESM Climate modeling Consortium consisting of CICERO (Center for International Climate and Environmental Research, Oslo 0349), MET-Norway (Norwegian Meteorological Institute, Oslo 0313), NERSC (Nansen Environmental and Remote Sensing Center, Bergen 5006), NILU (Norwegian Institute for Air Research, Kjeller 2027), UiB (University of Bergen, Bergen 5007), UiO (University of Oslo, Oslo 0313) and UNI (Uni Research, Bergen 5008), Norway. Mailing address: NCC, c/o MET-Norway, Henrik Mohns plass 1, Oslo 0313, Norway", + "contact": "Please send any requests or bug reports to noresm-ncc@met.no.", + "product": "model-output", + "_cmip7_option": true, + "horizontal_label": "hxy", + "vertical_label": "l", + "temporal_label": "tavg", + "source_id": "NorESM3-LM", + "source": "NorESM3 (2025): atmosphere: CAM; atmosChem: OsloChemSimp; ocean: BLOM; oceanBgchem: HAMOCC; sea ice: CICE6; land: CLM-FATES; aerosol: OsloAero; coupler: CMEPS.", + "source_type": "AOGCM", + "nominal_resolution": "100 km", + "region": "glb", + "branding_suffix": "tavg-l-hxy-u", + "experiment_id": "piControl", + "experiment": "Pre-industrial control simulation with prescribed carbon dioxide concentrations (for prescribed carbon dioxide emissions, see `esm-piControl`). Used to characterise natural variability and unforced behaviour.", + "branch_method": "no parent", + "branch_time_in_parent": 0.0, + "branch_time_in_child": 0.0, + "parent_time_units": "days since 1850-01-01-00000", + "parent_variant_label": "r1i1p1f1", + "parent_mip_era": "CMIP7", + "parent_source_id": "NorESM3-LM", + "parent_activity_id": "CMIP", + "parent_experiment_id": "piControl-spinup", + "sub_experiment_id": "none", + "sub_experiment": "none", + "archive_id": "WCRP", + "area_label" : "u", + "license_id": "CC-BY-4.0", + "realization_index": "r1", + "initialization_index": "i1", + "physics_index": "p1", + "forcing_index": "f1", + "member_id": "r1i1p1f1", + "calendar": "noleap", + "frequency": "mon", + "temporal_subset": "1850", + "grid": "data mapped to a 2x2 degree.", + "grid_label": "g123", + "title": "NorESM3 piControl on 2x2 degree grid (regridded from ne16np3 se grid)", + "references": "See NorESM3 documentation and CMIP7 references for NCAR/CGD.", + "outpath": ".", + "output_path_template": "", + "output_file_template": "" +} diff --git a/data/cmor_dataset_noresm3-mm.json b/data/cmor_dataset_noresm3-mm.json new file mode 100644 index 00000000..77c38150 --- /dev/null +++ b/data/cmor_dataset_noresm3-mm.json @@ -0,0 +1,51 @@ +{ + "project_id": "CMIP7", + "mip_era": "CMIP7", + "activity_id": "CMIP", + "drs_specs": "MIP-DRS7", + "institution_id": "NCC", + "institution": "NorESM Climate modeling Consortium consisting of CICERO (Center for International Climate and Environmental Research, Oslo 0349), MET-Norway (Norwegian Meteorological Institute, Oslo 0313), NERSC (Nansen Environmental and Remote Sensing Center, Bergen 5006), NILU (Norwegian Institute for Air Research, Kjeller 2027), UiB (University of Bergen, Bergen 5007), UiO (University of Oslo, Oslo 0313) and UNI (Uni Research, Bergen 5008), Norway. Mailing address: NCC, c/o MET-Norway, Henrik Mohns plass 1, Oslo 0313, Norway", + "contact": "Please send any requests or bug reports to noresm-ncc@met.no.", + "product": "model-output", + "_cmip7_option": true, + "horizontal_label": "hxy", + "vertical_label": "l", + "temporal_label": "tavg", + "source_id": "NorESM3-MM", + "source": "NorESM3 (2025): atmosphere: CAM; atmosChem: OsloChemSimp; ocean: BLOM; oceanBgchem: HAMOCC; sea ice: CICE6; land: CLM-FATES; aerosol: OsloAero; coupler: CMEPS.", + "source_type": "AOGCM", + "nominal_resolution": "100 km", + "region": "glb", + "branding_suffix": "tavg-l-hxy-u", + "experiment_id": "piControl", + "experiment": "Pre-industrial control simulation with prescribed carbon dioxide concentrations (for prescribed carbon dioxide emissions, see `esm-piControl`). Used to characterise natural variability and unforced behaviour.", + "branch_method": "no parent", + "branch_time_in_parent": 0.0, + "branch_time_in_child": 0.0, + "parent_time_units": "days since 1850-01-01-00000", + "parent_variant_label": "r1i1p1f1", + "parent_mip_era": "CMIP7", + "parent_source_id": "NorESM3", + "parent_activity_id": "CMIP", + "parent_experiment_id": "piControl-spinup", + "sub_experiment_id": "none", + "sub_experiment": "none", + "archive_id": "WCRP", + "area_label" : "u", + "license_id": "CC-BY-4.0", + "realization_index": "r1", + "initialization_index": "i1", + "physics_index": "p1", + "forcing_index": "f1", + "member_id": "r1i1p1f1", + "calendar": "noleap", + "frequency": "mon", + "temporal_subset": "1850", + "grid": "data mapped to a 2x2 degree.", + "grid_label": "g106", + "title": "NorESM3 piControl on 1x1 degree grid (regridded from ne30np3 se grid)", + "references": "See NorESM3 documentation and CMIP7 references for NCAR/CGD.", + "outpath": ".", + "output_path_template": "", + "output_file_template": "" +} diff --git a/data/noresm_regrid_maps.yaml b/data/noresm_regrid_maps.yaml index 47066533..47a2572c 100644 --- a/data/noresm_regrid_maps.yaml +++ b/data/noresm_regrid_maps.yaml @@ -30,3 +30,19 @@ resolutions: regular: conservative: map_tnx1v4_to_1x1_aave_c260531.nc bilinear: map_tnx1v4_to_1x1_blin_c260531.nc + +grid_names_per_realm: + ne16: + atmos: g123 + aerosol: g123 + atmosChem: g123 + land: g123 + seaIce: g202 + landIce: g194 + ne30: + atmos: g106 + aerosol: g106 + atmosChem: g106 + land: g106 + seaIce: g106 + landIce: g106 diff --git a/scripts/cmor_driver.py b/scripts/cmor_driver.py index ae734c53..4642251a 100755 --- a/scripts/cmor_driver.py +++ b/scripts/cmor_driver.py @@ -8,12 +8,16 @@ Preserves all comments and error handling from both atm_monthly.py and lnd_monthly.py. """ - from __future__ import annotations import argparse from concurrent.futures import as_completed +import inspect + +print(inspect.getframeinfo(inspect.getouterframes(inspect.currentframe())[0][0])[0]) + + import os from pathlib import Path import logging @@ -27,6 +31,14 @@ import json import numpy as np import xarray as xr + +# Load all possible cmip vars for this realm and this experiment +# The data_request_api is a CMIP7-specific Python package that is +# separate from CMOR itself but closely related to it. +# It produce lists of variables requested for each CMIP7 experiment +from data_request_api.query import data_request as dr +from data_request_api.content import dump_transformation as dt + from cmor import set_cur_dataset_attribute from cmip7_prep.cmor_utils import ( @@ -50,9 +62,14 @@ from cmip7_prep.cmor_writer import CmorSession from cmip7_prep.mom6_static import ocean_fx_fields from cmip7_prep.variable_selection import assemble_yaml_defined_cmip_vars +from cmip7_prep.regrid_maps import get_grid_names + from dask import delayed + + + # The only logging configuration in the package: library modules attach # handlers to their own loggers but leave the root config to the entry point. logging.basicConfig( @@ -628,8 +645,17 @@ def process_one_var( # Initialize CMOR class metadata_json = None if model == "noresm": - metadata_json = packaged_dataset_json("cmor_dataset_noresm.json") - + metadata_json = packaged_dataset_json( + "cmor_dataset_noresm.json", + ) + if resolution == "ne16": + metadata_json = packaged_dataset_json( + "cmor_dataset_noresm3-lm.json", + ) + elif resolution == "ne30": + metadata_json = packaged_dataset_json( + "cmor_dataset_noresm3-mm.json", + ) with CmorSession( tables_root=tables_root, log_dir=log_dir, @@ -648,6 +674,13 @@ def process_one_var( set_cur_dataset_attribute("forcing_index", forcing_index) region = write_cfg.get("region", "glb") set_cur_dataset_attribute("region", region) + try: + grid_name = get_grid_names(model, resolution, realm) + + except ValueError as e: + logger.error(f"Error setting grid attribute: {e}") + grid_name = grid + set_cur_dataset_attribute("grid", grid_name) # Updating with correct experiment info from CMIP7 tables experiment_info = get_experiment_info_from_tables( experiment, tables_root @@ -673,12 +706,13 @@ def process_one_var( "standard_name": write_cfg.get("standard_name", None), "levels": write_cfg.get("levels", None), "branded_variable_name": cmip7name, + "grid_label": grid_name, }, )() - # Now use CMOR utility to write out netcdf variable cm.write_variable(ds_cmor_write, cmip_var, vdef) + # A high-water mark for the process, so this is the largest any # variable has needed so far, not this one alone. A jump from the # previous line means this variable exceeded all before it. @@ -758,13 +792,6 @@ def main(): mapping = Mapping.from_packaged_default(filename=yaml_filename) mapping.default_freq = frequency - # Load all possible cmip vars for this realm and this experiment - # The data_request_api is a CMIP7-specific Python package that is - # separate from CMOR itself but closely related to it. - # It produce lists of variables requested for each CMIP7 experiment - from data_request_api.query import data_request as dr - from data_request_api.content import dump_transformation as dt - logger.info("Loading data request content %s", realm) content_dic = dt.get_transformed_content() logger.info("Content dictionary obtained") diff --git a/scripts/convert_csv_to_yaml.py b/scripts/convert_csv_to_yaml.py index d8655a61..5a149917 100755 --- a/scripts/convert_csv_to_yaml.py +++ b/scripts/convert_csv_to_yaml.py @@ -82,7 +82,7 @@ def load_intensive_vars() -> set[str]: "normalize_dim_names": True, "dataset_overrides": { "institution_id": "NCC", - "source_id": "NorESM3", + "source_id": "NorESM3-LM", "nominal_resolution": "200 km", "yaml_coax_dummy": [0, 1, 2], }, diff --git a/scripts/validate_cmor_output.py b/scripts/validate_cmor_output.py index bfb03e3c..5e06de48 100644 --- a/scripts/validate_cmor_output.py +++ b/scripts/validate_cmor_output.py @@ -40,7 +40,7 @@ logger = logging.getLogger("cmip7_prep.validate_cmor_output") MODEL_NAMING_MAPS = { - "noresm": ["NCC", "NorESM3"], + "noresm": ["NCC", "NorESM3-LM"], "cesm": ["NCAR", "CESM3"], } diff --git a/src/cmip7_prep/cmor_writer.py b/src/cmip7_prep/cmor_writer.py index 549f1cc2..1775e9b7 100644 --- a/src/cmip7_prep/cmor_writer.py +++ b/src/cmip7_prep/cmor_writer.py @@ -1206,12 +1206,12 @@ def _write_fx_2d(self, ds: xr.Dataset, name: str, units: str) -> None: logger.debug("FX variable %s define lon_id %s", name, lon_id) logger.debug("Writing fx variable %s on curvilinear grid", name) cmor.set_cur_dataset_attribute("grid", "curvilinear") - cmor.set_cur_dataset_attribute("grid_label", "gn") + #cmor.set_cur_dataset_attribute("grid_label", "gn") if name == "deptho": name = "deptho_ti-u-hxy-sea" else: cmor.set_cur_dataset_attribute("grid", "1x1 degree") - cmor.set_cur_dataset_attribute("grid_label", "gr") + #cmor.set_cur_dataset_attribute("grid_label", "gr") if name in ("areacella_ti-u-hxy-u", "sftlf_ti-u-hxy-u"): self.load_table(self.tables_root, "land") elif name in ("sftof_ti-u-hxy-u", "deptho", "areacello"): @@ -1439,11 +1439,13 @@ def write_variable( ) logger.debug("Now define time dimension and write data") if "lat" in data.dims and "lon" in data.dims: + # These should be pulled fromt he grid in the tables cmor.set_cur_dataset_attribute("grid", "1x1 degree") - cmor.set_cur_dataset_attribute("grid_label", "gr") + cmor.set_cur_dataset_attribute("grid_label", vdef.get("grid_label", "gr")) else: + # These should be pulled fromt he grid in the tables cmor.set_cur_dataset_attribute("grid", "curvilinear") - cmor.set_cur_dataset_attribute("grid_label", "gn") + cmor.set_cur_dataset_attribute("grid_label", vdef.get("grid_label", "gn")) # ---- Prepare time info for this write (local, not cached) ---- time_da = ds.coords.get("time") if time_da is None: diff --git a/src/cmip7_prep/regrid_maps.py b/src/cmip7_prep/regrid_maps.py index 5c4520a3..5b00b7d5 100644 --- a/src/cmip7_prep/regrid_maps.py +++ b/src/cmip7_prep/regrid_maps.py @@ -77,3 +77,24 @@ def get_map_paths(model: str, resolution: str) -> dict[str, Path]: "every resolution needs one, since fx fields are always conservative" ) return {method: root / name for method, name in entry.items()} + +def get_grid_names(model: str, resolution: str, realm: str) -> dict[str, str]: + """Return the grid names for one model and resolution. + + The result has keys like 'atmos', 'aerosol', 'land', 'seaIce', 'landIce'. + """ + table = _load_cached(model) + grid_names = table.get("grid_names_per_realm") or {} + if resolution not in grid_names: + raise ValueError( + f"No grid names defined for model={model}, resolution={resolution}; " + f"available: {sorted(grid_names)}" + ) + realm_grids = grid_names.get(resolution) or {} + if realm not in realm_grids: + raise ValueError( + f"No grid names defined for model={model}, resolution={resolution}, realm={realm}; " + f"available: {sorted(realm_grids)}" + ) + + return grid_names[resolution][realm] or {} \ No newline at end of file From e828b920cbfba2eba252357735dc16e28e483f08 Mon Sep 17 00:00:00 2001 From: maritsandstad Date: Sat, 3 Oct 2026 14:39:58 +0200 Subject: [PATCH 2/9] Refactor cleanup for the cmorsession avoiding json-files and pulling info from controlled vocabulary --- data/cesm_regrid_maps.yaml | 14 ++- data/cmor_dataset.json | 50 --------- data/cmor_dataset_noresm.json | 51 --------- data/cmor_dataset_noresm3-lm.json | 51 --------- data/cmor_dataset_noresm3-mm.json | 51 --------- data/noresm_regrid_maps.yaml | 18 ++++ scripts/cmor_driver.py | 53 +++------ src/cmip7_prep/cmor_utils.py | 171 ++++++++++++++++++++++++++++++ src/cmip7_prep/cmor_writer.py | 54 +++++----- src/cmip7_prep/regrid_maps.py | 19 +++- tests/test_cmor_writer.py | 63 +++++------ 11 files changed, 294 insertions(+), 301 deletions(-) delete mode 100644 data/cmor_dataset.json delete mode 100644 data/cmor_dataset_noresm.json delete mode 100644 data/cmor_dataset_noresm3-lm.json delete mode 100644 data/cmor_dataset_noresm3-mm.json diff --git a/data/cesm_regrid_maps.yaml b/data/cesm_regrid_maps.yaml index bbd67bad..a363d0fc 100644 --- a/data/cesm_regrid_maps.yaml +++ b/data/cesm_regrid_maps.yaml @@ -36,4 +36,16 @@ grid_names_per_realm: atmosChem: g106 land: g106 seaIce: g106 - landIce: g106 \ No newline at end of file + landIce: g106 + +# Static dataset (global attribute) metadata for the CMOR dataset JSON builder; +# see noresm_regrid_maps.yaml for the schema. CESM writes CESM3 at every +# resolution for now (placeholder). +dataset: + base_source_id: CESM3 + source_type: AOGCM + calendar: noleap + source_ids: {} + default_source_id: CESM3 + nominal_resolution: {} + default_nominal_resolution: 100 km \ No newline at end of file diff --git a/data/cmor_dataset.json b/data/cmor_dataset.json deleted file mode 100644 index 449b1ccd..00000000 --- a/data/cmor_dataset.json +++ /dev/null @@ -1,50 +0,0 @@ -{ - "project_id": "CMIP7", - "mip_era": "CMIP7", - "activity_id": "CMIP", - "drs_specs": "MIP-DRS7", - "institution_id": "NCAR", - "institution": "NSF National Center for Atmospheric Research, Climate and Global Dynamics Laboratory, 1850 Table Mesa Drive, Boulder, CO 80305, USA", - "contact": "cesm-cmip@ucar.edu", - "product": "model-output", - "_cmip7_option": true, - "horizontal_label": "hxy", - "vertical_label": "l", - "temporal_label": "tavg", - "source_id": "CESM3", - "source": "CESM3 (2025): atmosphere: CAM; ocean: MOM; sea ice: CICE6; land: CTSM; aerosol: MAM4; coupler: CMEPS.", - "source_type": "AOGCM", - "nominal_resolution": "100 km", - "branding_suffix": "tavg-l-hxy-u", - "experiment_id": "piControl", - "experiment": "Pre-industrial control simulation with prescribed carbon dioxide concentrations (for prescribed carbon dioxide emissions, see `esm-piControl`). Used to characterise natural variability and unforced behaviour.", - "branch_method": "no parent", - "branch_time_in_parent": 0.0, - "branch_time_in_child": 0.0, - "parent_time_units": "days since 1850-01-01-00000", - "parent_variant_label": "r1i1p1f1", - "parent_mip_era": "CMIP7", - "parent_source_id": "CESM3", - "parent_activity_id": "CMIP", - "parent_experiment_id": "piControl-spinup", - "sub_experiment_id": "none", - "sub_experiment": "none", - "archive_id": "WCRP", - "area_label" : "u", - "license_id": "CC-BY-4-0", - "realization_index": "r1", - "initialization_index": "i1", - "physics_index": "p1", - "forcing_index": "f1", - "member_id": "r1i1p1f1", - "calendar": "noleap", - "frequency": "mon", - "temporal_subset": "1850", - "grid": "data mapped to a 1x1 degree.", - "grid_label": "gr", - "title": "CESM3 piControl on 1x1 degree grid (regridded from ne30np3 se grid)", - "references": "See CESM3 documentation and CMIP7 references for NCAR/CGD.", - "outpath": ".", - "output_path_template": "", - "output_file_template": "" -} diff --git a/data/cmor_dataset_noresm.json b/data/cmor_dataset_noresm.json deleted file mode 100644 index ba465dac..00000000 --- a/data/cmor_dataset_noresm.json +++ /dev/null @@ -1,51 +0,0 @@ -{ - "project_id": "CMIP7", - "mip_era": "CMIP7", - "activity_id": "CMIP", - "drs_specs": "MIP-DRS7", - "institution_id": "NCC", - "institution": "NorESM Climate modeling Consortium consisting of CICERO (Center for International Climate and Environmental Research, Oslo 0349), MET-Norway (Norwegian Meteorological Institute, Oslo 0313), NERSC (Nansen Environmental and Remote Sensing Center, Bergen 5006), NILU (Norwegian Institute for Air Research, Kjeller 2027), UiB (University of Bergen, Bergen 5007), UiO (University of Oslo, Oslo 0313) and UNI (Uni Research, Bergen 5008), Norway. Mailing address: NCC, c/o MET-Norway, Henrik Mohns plass 1, Oslo 0313, Norway", - "contact": "Please send any requests or bug reports to noresm-ncc@met.no.", - "product": "model-output", - "_cmip7_option": true, - "horizontal_label": "hxy", - "vertical_label": "l", - "temporal_label": "tavg", - "source_id": "NorESM3-LM", - "source": "NorESM3 (2025): atmosphere: CAM; atmosChem: OsloChemSimp; ocean: BLOM; oceanBgchem: HAMOCC; sea ice: CICE6; land: CLM-FATES; aerosol: OsloAero; coupler: CMEPS.", - "source_type": "AOGCM", - "nominal_resolution": "100 km", - "region": "glb", - "branding_suffix": "tavg-l-hxy-u", - "experiment_id": "piControl", - "experiment": "Pre-industrial control simulation with prescribed carbon dioxide concentrations (for prescribed carbon dioxide emissions, see `esm-piControl`). Used to characterise natural variability and unforced behaviour.", - "branch_method": "no parent", - "branch_time_in_parent": 0.0, - "branch_time_in_child": 0.0, - "parent_time_units": "days since 1850-01-01-00000", - "parent_variant_label": "r1i1p1f1", - "parent_mip_era": "CMIP7", - "parent_source_id": "NorESM3", - "parent_activity_id": "CMIP", - "parent_experiment_id": "piControl-spinup", - "sub_experiment_id": "none", - "sub_experiment": "none", - "archive_id": "WCRP", - "area_label" : "u", - "license_id": "CC-BY-4.0", - "realization_index": "r1", - "initialization_index": "i1", - "physics_index": "p1", - "forcing_index": "f1", - "member_id": "r1i1p1f1", - "calendar": "noleap", - "frequency": "mon", - "temporal_subset": "1850", - "grid": "data mapped to a 2x2 degree.", - "grid_label": "gr", - "title": "NorESM3 piControl on 2x2 degree grid (regridded from ne16np3 se grid)", - "references": "See NorESM3 documentation and CMIP7 references for NCAR/CGD.", - "outpath": ".", - "output_path_template": "", - "output_file_template": "" -} diff --git a/data/cmor_dataset_noresm3-lm.json b/data/cmor_dataset_noresm3-lm.json deleted file mode 100644 index d7b58993..00000000 --- a/data/cmor_dataset_noresm3-lm.json +++ /dev/null @@ -1,51 +0,0 @@ -{ - "project_id": "CMIP7", - "mip_era": "CMIP7", - "activity_id": "CMIP", - "drs_specs": "MIP-DRS7", - "institution_id": "NCC", - "institution": "NorESM Climate modeling Consortium consisting of CICERO (Center for International Climate and Environmental Research, Oslo 0349), MET-Norway (Norwegian Meteorological Institute, Oslo 0313), NERSC (Nansen Environmental and Remote Sensing Center, Bergen 5006), NILU (Norwegian Institute for Air Research, Kjeller 2027), UiB (University of Bergen, Bergen 5007), UiO (University of Oslo, Oslo 0313) and UNI (Uni Research, Bergen 5008), Norway. Mailing address: NCC, c/o MET-Norway, Henrik Mohns plass 1, Oslo 0313, Norway", - "contact": "Please send any requests or bug reports to noresm-ncc@met.no.", - "product": "model-output", - "_cmip7_option": true, - "horizontal_label": "hxy", - "vertical_label": "l", - "temporal_label": "tavg", - "source_id": "NorESM3-LM", - "source": "NorESM3 (2025): atmosphere: CAM; atmosChem: OsloChemSimp; ocean: BLOM; oceanBgchem: HAMOCC; sea ice: CICE6; land: CLM-FATES; aerosol: OsloAero; coupler: CMEPS.", - "source_type": "AOGCM", - "nominal_resolution": "100 km", - "region": "glb", - "branding_suffix": "tavg-l-hxy-u", - "experiment_id": "piControl", - "experiment": "Pre-industrial control simulation with prescribed carbon dioxide concentrations (for prescribed carbon dioxide emissions, see `esm-piControl`). Used to characterise natural variability and unforced behaviour.", - "branch_method": "no parent", - "branch_time_in_parent": 0.0, - "branch_time_in_child": 0.0, - "parent_time_units": "days since 1850-01-01-00000", - "parent_variant_label": "r1i1p1f1", - "parent_mip_era": "CMIP7", - "parent_source_id": "NorESM3-LM", - "parent_activity_id": "CMIP", - "parent_experiment_id": "piControl-spinup", - "sub_experiment_id": "none", - "sub_experiment": "none", - "archive_id": "WCRP", - "area_label" : "u", - "license_id": "CC-BY-4.0", - "realization_index": "r1", - "initialization_index": "i1", - "physics_index": "p1", - "forcing_index": "f1", - "member_id": "r1i1p1f1", - "calendar": "noleap", - "frequency": "mon", - "temporal_subset": "1850", - "grid": "data mapped to a 2x2 degree.", - "grid_label": "g123", - "title": "NorESM3 piControl on 2x2 degree grid (regridded from ne16np3 se grid)", - "references": "See NorESM3 documentation and CMIP7 references for NCAR/CGD.", - "outpath": ".", - "output_path_template": "", - "output_file_template": "" -} diff --git a/data/cmor_dataset_noresm3-mm.json b/data/cmor_dataset_noresm3-mm.json deleted file mode 100644 index 77c38150..00000000 --- a/data/cmor_dataset_noresm3-mm.json +++ /dev/null @@ -1,51 +0,0 @@ -{ - "project_id": "CMIP7", - "mip_era": "CMIP7", - "activity_id": "CMIP", - "drs_specs": "MIP-DRS7", - "institution_id": "NCC", - "institution": "NorESM Climate modeling Consortium consisting of CICERO (Center for International Climate and Environmental Research, Oslo 0349), MET-Norway (Norwegian Meteorological Institute, Oslo 0313), NERSC (Nansen Environmental and Remote Sensing Center, Bergen 5006), NILU (Norwegian Institute for Air Research, Kjeller 2027), UiB (University of Bergen, Bergen 5007), UiO (University of Oslo, Oslo 0313) and UNI (Uni Research, Bergen 5008), Norway. Mailing address: NCC, c/o MET-Norway, Henrik Mohns plass 1, Oslo 0313, Norway", - "contact": "Please send any requests or bug reports to noresm-ncc@met.no.", - "product": "model-output", - "_cmip7_option": true, - "horizontal_label": "hxy", - "vertical_label": "l", - "temporal_label": "tavg", - "source_id": "NorESM3-MM", - "source": "NorESM3 (2025): atmosphere: CAM; atmosChem: OsloChemSimp; ocean: BLOM; oceanBgchem: HAMOCC; sea ice: CICE6; land: CLM-FATES; aerosol: OsloAero; coupler: CMEPS.", - "source_type": "AOGCM", - "nominal_resolution": "100 km", - "region": "glb", - "branding_suffix": "tavg-l-hxy-u", - "experiment_id": "piControl", - "experiment": "Pre-industrial control simulation with prescribed carbon dioxide concentrations (for prescribed carbon dioxide emissions, see `esm-piControl`). Used to characterise natural variability and unforced behaviour.", - "branch_method": "no parent", - "branch_time_in_parent": 0.0, - "branch_time_in_child": 0.0, - "parent_time_units": "days since 1850-01-01-00000", - "parent_variant_label": "r1i1p1f1", - "parent_mip_era": "CMIP7", - "parent_source_id": "NorESM3", - "parent_activity_id": "CMIP", - "parent_experiment_id": "piControl-spinup", - "sub_experiment_id": "none", - "sub_experiment": "none", - "archive_id": "WCRP", - "area_label" : "u", - "license_id": "CC-BY-4.0", - "realization_index": "r1", - "initialization_index": "i1", - "physics_index": "p1", - "forcing_index": "f1", - "member_id": "r1i1p1f1", - "calendar": "noleap", - "frequency": "mon", - "temporal_subset": "1850", - "grid": "data mapped to a 2x2 degree.", - "grid_label": "g106", - "title": "NorESM3 piControl on 1x1 degree grid (regridded from ne30np3 se grid)", - "references": "See NorESM3 documentation and CMIP7 references for NCAR/CGD.", - "outpath": ".", - "output_path_template": "", - "output_file_template": "" -} diff --git a/data/noresm_regrid_maps.yaml b/data/noresm_regrid_maps.yaml index 47a2572c..1240ba1f 100644 --- a/data/noresm_regrid_maps.yaml +++ b/data/noresm_regrid_maps.yaml @@ -46,3 +46,21 @@ grid_names_per_realm: land: g106 seaIce: g106 landIce: g106 + +# Static dataset (global attribute) metadata for the CMOR dataset JSON builder. +# base_source_id is the key looked up in cmor-cvs.json for institution_id and +# source text; it must be a CV source_id entry. source_ids/nominal_resolution +# are the branded, per-resolution values actually written to the output; the +# default_* keys catch any resolution not listed (e.g. non-CMIP test runs). +dataset: + base_source_id: NorESM3 + source_type: AOGCM + calendar: noleap + source_ids: + ne16: NorESM3-LM + ne30: NorESM3-MM + default_source_id: NorESM3 + nominal_resolution: + ne16: 250 km + ne30: 100 km + default_nominal_resolution: 100 km diff --git a/scripts/cmor_driver.py b/scripts/cmor_driver.py index 4642251a..71c5b0a7 100755 --- a/scripts/cmor_driver.py +++ b/scripts/cmor_driver.py @@ -45,7 +45,7 @@ load_positive_overrides, bounds_from_centers_1d, roll_for_monotonic_with_bounds, - packaged_dataset_json, + build_dataset_cfg, ) from cmip7_prep.include_patterns import ( get_include_patterns, @@ -501,9 +501,6 @@ def process_one_var( """Compute+write one CMIP variable. Returns a list of (varname, 'ok' or error message) tuples.""" varname = cmip_var.branded_variable_name.name - realization_index, initialization_index, physics_index, forcing_index = ( - parse_realization_initialization_physics_forcing(ripf_index) - ) # At this point you have a cmip_var (metadata from database query for the target variable) # queried a cmor database from the cloud logger.debug(f"Starting processing for variable: {varname}") @@ -641,54 +638,36 @@ def process_one_var( try: log_dir = outdir / "logs" - # TODO: add NorESM institution_id below - # Initialize CMOR class - metadata_json = None - if model == "noresm": - metadata_json = packaged_dataset_json( - "cmor_dataset_noresm.json", - ) - if resolution == "ne16": - metadata_json = packaged_dataset_json( - "cmor_dataset_noresm3-lm.json", - ) - elif resolution == "ne30": - metadata_json = packaged_dataset_json( - "cmor_dataset_noresm3-mm.json", - ) + # Global/dataset attributes assembled from the CV plus the + # per-model dataset config; replaces the packaged cmor_dataset + # JSON files. frequency, variant indices and experiment + # metadata are already baked in, so only the per-variable + # region/grid are set below. + dataset_cfg = build_dataset_cfg( + model=model, + resolution=resolution, + experiment=experiment, + frequency=frequency, + ripf=ripf_index, + tables_root=tables_root, + ) with CmorSession( tables_root=tables_root, log_dir=log_dir, log_name=f"cmor_{datetime.now(UTC).strftime('%Y%m%dT%H%M%SZ')}_{varname}.log", - dataset_json=metadata_json, - dataset_attrs={"institution_id": "NCC", "GLOBAL_IS_CMIP7": True}, + dataset_attrs=dataset_cfg, outdir=outdir, ice_sheet=ice_sheet, ) as cm: - set_cur_dataset_attribute("frequency", frequency) - set_cur_dataset_attribute("realization_index", realization_index) - set_cur_dataset_attribute( - "initialization_index", initialization_index - ) - set_cur_dataset_attribute("physics_index", physics_index) - set_cur_dataset_attribute("forcing_index", forcing_index) region = write_cfg.get("region", "glb") set_cur_dataset_attribute("region", region) try: grid_name = get_grid_names(model, resolution, realm) - + except ValueError as e: logger.error(f"Error setting grid attribute: {e}") grid_name = grid set_cur_dataset_attribute("grid", grid_name) - # Updating with correct experiment info from CMIP7 tables - experiment_info = get_experiment_info_from_tables( - experiment, tables_root - ) - for key, value in experiment_info.items(): - if isinstance(value, list): - value = value[0] - set_cur_dataset_attribute(key, value) logger.info( f"Writing CMOR variable {cmip7name.name} with frequency {frequency}" diff --git a/src/cmip7_prep/cmor_utils.py b/src/cmip7_prep/cmor_utils.py index dd99c03c..b4a9a18e 100644 --- a/src/cmip7_prep/cmor_utils.py +++ b/src/cmip7_prep/cmor_utils.py @@ -2,6 +2,7 @@ from functools import lru_cache from pathlib import Path +import json import warnings import re import datetime as dt @@ -49,6 +50,176 @@ def packaged_dataset_json(filename: str = "cmor_dataset.json"): return as_file(res) +# --------------------------------------------------------------------- +# CMOR dataset JSON builder +# --------------------------------------------------------------------- +# institution_id per model, matching the CV source_id entries. Used as a +# fallback only, when the CV lookup by base_source_id yields nothing. +_INSTITUTION_ID_BY_MODEL = {"noresm": "NCC", "cesm": "NCAR"} + +# DRS path/filename templates; structural, so they live here rather than in the +# CV or the per-model config. +_OUTPUT_PATH_TEMPLATE = ( + "" + "" +) +_OUTPUT_FILE_TEMPLATE = ( + "" + "" +) + + +@lru_cache(maxsize=None) +def _load_controlled_vocabulary(tables_root: str) -> dict: + """Read and cache the CV block of cmor-cvs.json for a tables root.""" + path = Path(tables_root) / "tables-cvs" / "cmor-cvs.json" + with open(path, encoding="utf-8") as handle: + return json.load(handle).get("CV", {}) + + +def _first(value, default=None): + """Return value[0] for a list/tuple, the value itself otherwise, else default.""" + if isinstance(value, (list, tuple)): + return value[0] if value else default + return value if value is not None else default + + +def _resolve_by_resolution(mapping, resolution: str, default): + """Return mapping[resolution] when mapping is a dict with that key, else default.""" + if isinstance(mapping, dict): + return mapping.get(resolution, default) + return default + + +def _format_license(license_cv: dict, license_id: str, institution_id: str) -> str: + """Expand the CV license_template for one license_id and institution.""" + template = license_cv.get("license_template", "") + entry = (license_cv.get("license_id") or {}).get(license_id, {}) + return ( + template.replace("", license_id) + .replace("", institution_id or "") + .replace("", entry.get("license_type", "")) + .replace("", entry.get("license_url", "")) + ) + + +def _parse_ripf(ripf: str) -> tuple[str, str, str, str]: + """Split 'rXiYpZfW' into its ('rX', 'iY', 'pZ', 'fW') CMIP7 index strings.""" + match = re.fullmatch(r"r(\d+)i(\d+)p(\d+)f(\d+)", ripf.strip()) + if not match: + raise ValueError( + f"Invalid realization-initialization-physics-forcing string {ripf!r}; " + "expected rXiYpZfW (e.g. r1i1p1f1)" + ) + realization, initialization, physics, forcing = match.groups() + return f"r{realization}", f"i{initialization}", f"p{physics}", f"f{forcing}" + + +def build_dataset_cfg( + *, + model: str, + resolution: str, + experiment: str, + frequency: str, + ripf: str, + tables_root, + license_id: str = "CC-BY-4.0", +) -> dict: + """Assemble the CMOR dataset-attribute dict for one run. + + Institutional, source, license and experiment metadata are read from the + controlled vocabulary (cmor-cvs.json); the branded source_id and nominal + resolution come from the per-model ``dataset`` block in + ``data/_regrid_maps.yaml``. Replaces the packaged cmor_dataset*.json. + """ + from .regrid_maps import load_regrid_maps # local import avoids import cycle + + cv = _load_controlled_vocabulary(str(tables_root)) + meta = load_regrid_maps(model).get("dataset") or {} + base_source_id = meta.get("base_source_id") + + # Branded source_id/nominal_resolution written to the output, by resolution. + source_id = _resolve_by_resolution( + meta.get("source_ids"), resolution, meta.get("default_source_id", base_source_id) + ) + nominal_resolution = _resolve_by_resolution( + meta.get("nominal_resolution"), + resolution, + meta.get("default_nominal_resolution"), + ) + + # institution/source derived from the CV via the base (CV-valid) source_id. + source_entry = (cv.get("source_id") or {}).get(base_source_id, {}) + institution_id = _first( + source_entry.get("institution_id"), _INSTITUTION_ID_BY_MODEL.get(model) + ) + institution = (cv.get("institution_id") or {}).get(institution_id, "") + source = source_entry.get("source", base_source_id) + license_str = _format_license(cv.get("license") or {}, license_id, institution_id) + + # experiment metadata (activity_id, parentage) straight from the CV. + experiment_entry = (cv.get("experiment_id") or {}).get(experiment, {}) + activity_id = _first(experiment_entry.get("activity_id"), "CMIP") + parent_experiment_id = _first( + experiment_entry.get("parent_experiment_id"), "no parent" + ) + parent_activity_id = _first(experiment_entry.get("parent_activity_id"), activity_id) + + realization_index, initialization_index, physics_index, forcing_index = _parse_ripf( + ripf + ) + mip_era = cv.get("mip_era", "CMIP7") + + return { + "mip_era": mip_era, + "activity_id": activity_id, + "drs_specs": cv.get("drs_specs", "MIP-DRS7"), + "data_specs_version": cv.get("data_specs_version"), + "Conventions": _first(cv.get("Conventions"), "CF-1.12"), + "product": "model-output", + "_cmip7_option": True, + "institution_id": institution_id, + "institution": institution, + "source_id": source_id, + "source": source, + "source_type": meta.get("source_type", "AOGCM"), + "nominal_resolution": nominal_resolution, + "license_id": license_id, + "license": license_str, + "calendar": meta.get("calendar", "noleap"), + "frequency": frequency, + "experiment_id": experiment, + "experiment": experiment_entry.get("experiment", experiment), + "sub_experiment_id": "none", + "sub_experiment": "none", + "realization_index": realization_index, + "initialization_index": initialization_index, + "physics_index": physics_index, + "forcing_index": forcing_index, + "variant_label": ripf, + "member_id": ripf, + "branch_method": "no parent", + "branch_time_in_parent": 0.0, + "branch_time_in_child": 0.0, + "parent_mip_era": mip_era, + "parent_activity_id": parent_activity_id, + "parent_experiment_id": parent_experiment_id, + "parent_source_id": base_source_id, + "parent_variant_label": ripf, + "parent_time_units": "days since 1850-01-01", + "horizontal_label": "hxy", + "vertical_label": "l", + "temporal_label": "tavg", + "area_label": "u", + "branding_suffix": "tavg-l-hxy-u", + "grid_label": "gr", + "region": "glb", + "tracking_prefix": cv.get("tracking_prefix", "hdl:21.14107"), + "output_path_template": _OUTPUT_PATH_TEMPLATE, + "output_file_template": _OUTPUT_FILE_TEMPLATE, + } + + def filled_for_cmor( da: xr.DataArray, fill: float | None = None ) -> tuple[xr.DataArray, float]: diff --git a/src/cmip7_prep/cmor_writer.py b/src/cmip7_prep/cmor_writer.py index 1775e9b7..20509b1d 100644 --- a/src/cmip7_prep/cmor_writer.py +++ b/src/cmip7_prep/cmor_writer.py @@ -2,10 +2,10 @@ """Thin CMOR wrapper used by cmip7_prep. This module centralizes CMOR session setup and writing so that the rest of the -pipeline can stay xarray-first. It supports either a dataset JSON file (preferred) -or directly injected global attributes, and creates axes based on the coordinates -present in the provided dataset. It also supports a packaged default -`cmor_dataset.json` living under `cmip7_prep/data/`. +pipeline can stay xarray-first. It takes the global/dataset attributes as an +in-memory dict (built by ``cmor_utils.build_dataset_cfg``), or, as a fallback, a +dataset JSON path, and creates axes based on the coordinates present in the +provided dataset. """ from pathlib import Path @@ -23,7 +23,6 @@ import numpy as np import xarray as xr from .cmor_utils import ( - packaged_dataset_json, get_cmor_attr, set_cmor_attr, encode_time_to_num, @@ -78,17 +77,19 @@ class CmorSession( tables_root : str or Path Directory containing CMOR tables and tables-csv directories with JSONs (e.g., CMIP6_*.json or CMIP7_*.json). - dataset_json : str or Path, optional - Path to a cmor_dataset.json with the experiment/source metadata. dataset_attrs : dict, optional - Alternative to `dataset_json`, allowing direct attribute injection. + Global/dataset attributes to write, typically produced by + ``cmor_utils.build_dataset_cfg``. Preferred over ``dataset_json``. + dataset_json : str or Path, optional + Path to a cmor_dataset.json with the experiment/source metadata; a + fallback for callers that still carry a file. """ def __init__( self, *, tables_root: Path | str, - dataset_attrs: dict[str, str] | None = None, + dataset_attrs: dict[str, Any] | None = None, dataset_json: Optional[DatasetJsonLike] = None, tracking_prefix: str | None = None, # NEW: one log per run (session) @@ -166,22 +167,27 @@ def __enter__(self) -> "CmorSession": getattr(cmor, name)(str(self._log_path)) break - # Resolve dataset_json to a real filesystem path - dj = self.dataset_json - if dj is None: - # packaged file → returns a context manager - cm = packaged_dataset_json("cmor_dataset.json") - self._dataset_json_cm = cm - p = cm.__enter__() # ← ENTER the CM, get a Path - elif isinstance(dj, (str, Path)): - p = Path(dj) + # Resolve the dataset attributes: the in-memory dict is preferred; a + # dataset_json path remains as a fallback for callers that still have a + # file. Either way CMOR is fed a temp JSON, since cmor.dataset_json + # must be called to load the CV before any variable is defined. + if self.dataset_attrs: + cfg = dict(self.dataset_attrs) + elif self.dataset_json is not None: + dj = self.dataset_json + if isinstance(dj, (str, Path)): + p = Path(dj) + else: + # caller passed a context manager directly + self._dataset_json_cm = dj + p = dj.__enter__() # ← ENTER the CM, get a Path + logger.debug("CMOR JSON dataset: %s", p) + with open(p, encoding="utf-8") as f: + cfg = json.load(f) else: - # caller passed a context manager directly - self._dataset_json_cm = dj - p = dj.__enter__() # ← ENTER the CM, get a Path - logger.debug("CMOR JSON dataset: %s", p) - with open(p, encoding="utf-8") as f: - cfg = json.load(f) + raise ValueError( + "CmorSession requires either dataset_attrs or dataset_json" + ) cfg["outpath"] = str(self._outdir) tmp = tempfile.NamedTemporaryFile("w", suffix=".json", delete=False) json.dump(cfg, tmp) diff --git a/src/cmip7_prep/regrid_maps.py b/src/cmip7_prep/regrid_maps.py index 5b00b7d5..fd25f549 100644 --- a/src/cmip7_prep/regrid_maps.py +++ b/src/cmip7_prep/regrid_maps.py @@ -97,4 +97,21 @@ def get_grid_names(model: str, resolution: str, realm: str) -> dict[str, str]: f"available: {sorted(realm_grids)}" ) - return grid_names[resolution][realm] or {} \ No newline at end of file + return grid_names[resolution][realm] or {} + +def get_grid_desc(model: str, resolution: str, realm: str) -> str: + """Return the grid description for one model and resolution.""" + table = _load_cached(model) + grid_descs = table.get("grid_descs_per_realm") or {} + if resolution not in grid_descs: + raise ValueError( + f"No grid descriptions defined for model={model}, resolution={resolution}; " + f"available: {sorted(grid_descs)}" + ) + realm_descs = grid_descs.get(resolution) or {} + if realm not in realm_descs: + raise ValueError( + f"No grid descriptions defined for model={model}, resolution={resolution}, realm={realm}; " + f"available: {sorted(realm_descs)}" + ) + return grid_descs[resolution][realm] or "" \ No newline at end of file diff --git a/tests/test_cmor_writer.py b/tests/test_cmor_writer.py index 3371100f..2833cf0c 100644 --- a/tests/test_cmor_writer.py +++ b/tests/test_cmor_writer.py @@ -6,6 +6,7 @@ import xarray as xr from cmip7_prep.cmor_writer import CmorSession +from cmip7_prep.cmor_utils import build_dataset_cfg def test_cmor_session_basic(tmp_path): @@ -56,22 +57,19 @@ class BrandedName: branded_variable_name = BrandedName() - dataset_json_path = Path(__file__).parent.parent / "data" / "cmor_dataset.json" + dataset_cfg = build_dataset_cfg( + model="cesm", + resolution="ne30", + experiment="piControl", + frequency="mon", + ripf="r1i1p1f1", + tables_root=tables_root, + ) log_dir = tmp_path log_name = "cmor_test.log" with CmorSession( tables_root=tables_root, - dataset_json=dataset_json_path, - dataset_attrs={ - "institution_id": "NCAR", - "GLOBAL_IS_CMIP7": True, - "branded_variable": { - "variable_id": "tas", - "table_id": "atmos", - "plev": 50000, - "description": "Vertical velocity at 500 hPa", - }, - }, + dataset_attrs=dataset_cfg, log_dir=log_dir, log_name=log_name, ) as session: @@ -139,22 +137,19 @@ class BrandedName: branded_variable_name = BrandedName() - dataset_json_path = Path(__file__).parent.parent / "data" / "cmor_dataset.json" + dataset_cfg = build_dataset_cfg( + model="cesm", + resolution="ne30", + experiment="piControl", + frequency="mon", + ripf="r1i1p1f1", + tables_root=tables_root, + ) log_dir = tmp_path log_name = "cmor_test_multi.log" with CmorSession( tables_root=tables_root, - dataset_json=dataset_json_path, - dataset_attrs={ - "institution_id": "NCAR", - "GLOBAL_IS_CMIP7": True, - "branded_variable": { - "variable_id": "tas", - "table_id": "atmos", - "plev": 50000, - "description": "Vertical velocity at 500 hPa", - }, - }, + dataset_attrs=dataset_cfg, log_dir=log_dir, log_name=log_name, ) as session: @@ -215,19 +210,17 @@ class BrandedName: branded_variable_name = BrandedName() - dataset_json_path = Path(__file__).parent.parent / "data" / "cmor_dataset.json" + dataset_cfg = build_dataset_cfg( + model="cesm", + resolution="ne30", + experiment="piControl", + frequency="mon", + ripf="r1i1p1f1", + tables_root=tables_root, + ) with CmorSession( tables_root=tables_root, - dataset_json=dataset_json_path, - dataset_attrs={ - "institution_id": "NCAR", - "GLOBAL_IS_CMIP7": True, - "branded_variable": { - "variable_id": "ta", - "table_id": "atmos", - "description": "Air Temperature", - }, - }, + dataset_attrs=dataset_cfg, log_dir=tmp_path, log_name="cmor_test_zonal_mean.log", ) as session: From 04da3247e4fba08ce236c0150fcf6f43e5e7f951 Mon Sep 17 00:00:00 2001 From: maritsandstad Date: Sat, 3 Oct 2026 22:44:30 +0200 Subject: [PATCH 3/9] Further adjustments to get the new setup working --- scripts/cmor_driver.py | 17 +++++------------ src/cmip7_prep/cmor_utils.py | 24 +++++++++++++++++++++--- src/cmip7_prep/cmor_writer.py | 16 ++++++++-------- 3 files changed, 34 insertions(+), 23 deletions(-) diff --git a/scripts/cmor_driver.py b/scripts/cmor_driver.py index 71c5b0a7..09394363 100755 --- a/scripts/cmor_driver.py +++ b/scripts/cmor_driver.py @@ -62,7 +62,6 @@ from cmip7_prep.cmor_writer import CmorSession from cmip7_prep.mom6_static import ocean_fx_fields from cmip7_prep.variable_selection import assemble_yaml_defined_cmip_vars -from cmip7_prep.regrid_maps import get_grid_names from dask import delayed @@ -640,9 +639,9 @@ def process_one_var( # Global/dataset attributes assembled from the CV plus the # per-model dataset config; replaces the packaged cmor_dataset - # JSON files. frequency, variant indices and experiment - # metadata are already baked in, so only the per-variable - # region/grid are set below. + # JSON files. frequency, variant indices, experiment metadata + # and the per-realm grid_label are already baked in, so only the + # per-variable region is set below. dataset_cfg = build_dataset_cfg( model=model, resolution=resolution, @@ -650,6 +649,7 @@ def process_one_var( frequency=frequency, ripf=ripf_index, tables_root=tables_root, + realm=realm, ) with CmorSession( tables_root=tables_root, @@ -661,13 +661,6 @@ def process_one_var( ) as cm: region = write_cfg.get("region", "glb") set_cur_dataset_attribute("region", region) - try: - grid_name = get_grid_names(model, resolution, realm) - - except ValueError as e: - logger.error(f"Error setting grid attribute: {e}") - grid_name = grid - set_cur_dataset_attribute("grid", grid_name) logger.info( f"Writing CMOR variable {cmip7name.name} with frequency {frequency}" @@ -685,7 +678,7 @@ def process_one_var( "standard_name": write_cfg.get("standard_name", None), "levels": write_cfg.get("levels", None), "branded_variable_name": cmip7name, - "grid_label": grid_name, + "grid_label": dataset_cfg.get("grid_label"), }, )() # Now use CMOR utility to write out netcdf variable diff --git a/src/cmip7_prep/cmor_utils.py b/src/cmip7_prep/cmor_utils.py index b4a9a18e..2c34cd13 100644 --- a/src/cmip7_prep/cmor_utils.py +++ b/src/cmip7_prep/cmor_utils.py @@ -123,6 +123,7 @@ def build_dataset_cfg( frequency: str, ripf: str, tables_root, + realm: str = "atmos", license_id: str = "CC-BY-4.0", ) -> dict: """Assemble the CMOR dataset-attribute dict for one run. @@ -132,12 +133,28 @@ def build_dataset_cfg( resolution come from the per-model ``dataset`` block in ``data/_regrid_maps.yaml``. Replaces the packaged cmor_dataset*.json. """ - from .regrid_maps import load_regrid_maps # local import avoids import cycle + # Local imports avoid an import cycle with regrid_maps. + from .regrid_maps import load_regrid_maps, get_grid_names cv = _load_controlled_vocabulary(str(tables_root)) meta = load_regrid_maps(model).get("dataset") or {} base_source_id = meta.get("base_source_id") + # grid_label is the per-realm grid code from the regrid-map table; its + # human-readable description comes from the CV keyed by that same code. + try: + grid_label = get_grid_names(model, resolution, realm) + except ValueError: + logger.warning( + "No grid name for model=%s resolution=%s realm=%s; defaulting " + "grid_label to 'gr'", + model, + resolution, + realm, + ) + grid_label = "gr" + grid_desc = (cv.get("grid_label") or {}).get(grid_label, "") + # Branded source_id/nominal_resolution written to the output, by resolution. source_id = _resolve_by_resolution( meta.get("source_ids"), resolution, meta.get("default_source_id", base_source_id) @@ -204,7 +221,7 @@ def build_dataset_cfg( "parent_mip_era": mip_era, "parent_activity_id": parent_activity_id, "parent_experiment_id": parent_experiment_id, - "parent_source_id": base_source_id, + "parent_source_id": source_id, "parent_variant_label": ripf, "parent_time_units": "days since 1850-01-01", "horizontal_label": "hxy", @@ -212,7 +229,8 @@ def build_dataset_cfg( "temporal_label": "tavg", "area_label": "u", "branding_suffix": "tavg-l-hxy-u", - "grid_label": "gr", + "grid_label": grid_label, + "grid": grid_desc, "region": "glb", "tracking_prefix": cv.get("tracking_prefix", "hdl:21.14107"), "output_path_template": _OUTPUT_PATH_TEMPLATE, diff --git a/src/cmip7_prep/cmor_writer.py b/src/cmip7_prep/cmor_writer.py index 20509b1d..3b569a33 100644 --- a/src/cmip7_prep/cmor_writer.py +++ b/src/cmip7_prep/cmor_writer.py @@ -1444,14 +1444,14 @@ def write_variable( missing_value=fillv, ) logger.debug("Now define time dimension and write data") - if "lat" in data.dims and "lon" in data.dims: - # These should be pulled fromt he grid in the tables - cmor.set_cur_dataset_attribute("grid", "1x1 degree") - cmor.set_cur_dataset_attribute("grid_label", vdef.get("grid_label", "gr")) - else: - # These should be pulled fromt he grid in the tables - cmor.set_cur_dataset_attribute("grid", "curvilinear") - cmor.set_cur_dataset_attribute("grid_label", vdef.get("grid_label", "gn")) + # if "lat" in data.dims and "lon" in data.dims: + # # These should be pulled fromt he grid in the tables + # cmor.set_cur_dataset_attribute("grid", "1x1 degree") + # cmor.set_cur_dataset_attribute("grid_label", vdef.get("grid_label", "gr")) + # else: + # # These should be pulled fromt he grid in the tables + # cmor.set_cur_dataset_attribute("grid", "curvilinear") + # cmor.set_cur_dataset_attribute("grid_label", vdef.get("grid_label", "gn")) # ---- Prepare time info for this write (local, not cached) ---- time_da = ds.coords.get("time") if time_da is None: From d399bd94df52a5012253c4fc930f63688552e301 Mon Sep 17 00:00:00 2001 From: maritsandstad Date: Mon, 5 Oct 2026 16:08:01 +0200 Subject: [PATCH 4/9] New cmor-tables version bump, linting --- cmip7-cmor-tables | 2 +- scripts/cmor_driver.py | 5 +---- src/cmip7_prep/cmor_utils.py | 4 +++- src/cmip7_prep/cmor_writer.py | 4 ++-- src/cmip7_prep/regrid_maps.py | 6 ++++-- 5 files changed, 11 insertions(+), 10 deletions(-) diff --git a/cmip7-cmor-tables b/cmip7-cmor-tables index 7d1d15a7..338c9a5a 160000 --- a/cmip7-cmor-tables +++ b/cmip7-cmor-tables @@ -1 +1 @@ -Subproject commit 7d1d15a716772ad627c2dabf3d595948c2075a7c +Subproject commit 338c9a5ad61b9eb1207f5c7cfbf99dcae196e674 diff --git a/scripts/cmor_driver.py b/scripts/cmor_driver.py index 09394363..8c91e47c 100755 --- a/scripts/cmor_driver.py +++ b/scripts/cmor_driver.py @@ -8,6 +8,7 @@ Preserves all comments and error handling from both atm_monthly.py and lnd_monthly.py. """ + from __future__ import annotations import argparse from concurrent.futures import as_completed @@ -66,9 +67,6 @@ from dask import delayed - - - # The only logging configuration in the package: library modules attach # handlers to their own loggers but leave the root config to the entry point. logging.basicConfig( @@ -684,7 +682,6 @@ def process_one_var( # Now use CMOR utility to write out netcdf variable cm.write_variable(ds_cmor_write, cmip_var, vdef) - # A high-water mark for the process, so this is the largest any # variable has needed so far, not this one alone. A jump from the # previous line means this variable exceeded all before it. diff --git a/src/cmip7_prep/cmor_utils.py b/src/cmip7_prep/cmor_utils.py index 2c34cd13..a8b1424d 100644 --- a/src/cmip7_prep/cmor_utils.py +++ b/src/cmip7_prep/cmor_utils.py @@ -157,7 +157,9 @@ def build_dataset_cfg( # Branded source_id/nominal_resolution written to the output, by resolution. source_id = _resolve_by_resolution( - meta.get("source_ids"), resolution, meta.get("default_source_id", base_source_id) + meta.get("source_ids"), + resolution, + meta.get("default_source_id", base_source_id), ) nominal_resolution = _resolve_by_resolution( meta.get("nominal_resolution"), diff --git a/src/cmip7_prep/cmor_writer.py b/src/cmip7_prep/cmor_writer.py index 3b569a33..45a19d02 100644 --- a/src/cmip7_prep/cmor_writer.py +++ b/src/cmip7_prep/cmor_writer.py @@ -1212,12 +1212,12 @@ def _write_fx_2d(self, ds: xr.Dataset, name: str, units: str) -> None: logger.debug("FX variable %s define lon_id %s", name, lon_id) logger.debug("Writing fx variable %s on curvilinear grid", name) cmor.set_cur_dataset_attribute("grid", "curvilinear") - #cmor.set_cur_dataset_attribute("grid_label", "gn") + # cmor.set_cur_dataset_attribute("grid_label", "gn") if name == "deptho": name = "deptho_ti-u-hxy-sea" else: cmor.set_cur_dataset_attribute("grid", "1x1 degree") - #cmor.set_cur_dataset_attribute("grid_label", "gr") + # cmor.set_cur_dataset_attribute("grid_label", "gr") if name in ("areacella_ti-u-hxy-u", "sftlf_ti-u-hxy-u"): self.load_table(self.tables_root, "land") elif name in ("sftof_ti-u-hxy-u", "deptho", "areacello"): diff --git a/src/cmip7_prep/regrid_maps.py b/src/cmip7_prep/regrid_maps.py index fd25f549..4a06a43a 100644 --- a/src/cmip7_prep/regrid_maps.py +++ b/src/cmip7_prep/regrid_maps.py @@ -78,6 +78,7 @@ def get_map_paths(model: str, resolution: str) -> dict[str, Path]: ) return {method: root / name for method, name in entry.items()} + def get_grid_names(model: str, resolution: str, realm: str) -> dict[str, str]: """Return the grid names for one model and resolution. @@ -96,9 +97,10 @@ def get_grid_names(model: str, resolution: str, realm: str) -> dict[str, str]: f"No grid names defined for model={model}, resolution={resolution}, realm={realm}; " f"available: {sorted(realm_grids)}" ) - + return grid_names[resolution][realm] or {} + def get_grid_desc(model: str, resolution: str, realm: str) -> str: """Return the grid description for one model and resolution.""" table = _load_cached(model) @@ -114,4 +116,4 @@ def get_grid_desc(model: str, resolution: str, realm: str) -> str: f"No grid descriptions defined for model={model}, resolution={resolution}, realm={realm}; " f"available: {sorted(realm_descs)}" ) - return grid_descs[resolution][realm] or "" \ No newline at end of file + return grid_descs[resolution][realm] or "" From fe53162954885ca40af30f590256c953387f81f3 Mon Sep 17 00:00:00 2001 From: maritsandstad Date: Mon, 5 Oct 2026 22:23:32 +0200 Subject: [PATCH 5/9] More linting --- src/cmip7_prep/cmor_utils.py | 10 +++++----- src/cmip7_prep/regrid_maps.py | 3 ++- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/cmip7_prep/cmor_utils.py b/src/cmip7_prep/cmor_utils.py index a8b1424d..91444704 100644 --- a/src/cmip7_prep/cmor_utils.py +++ b/src/cmip7_prep/cmor_utils.py @@ -15,6 +15,9 @@ import yaml import xarray as xr +# Local imports avoid an import cycle with regrid_maps. +from .regrid_maps import load_regrid_maps, get_grid_names + _FILL_DEFAULT = 1.0e20 _DATA_DIR = Path(__file__).parent.parent.parent / "data" @@ -108,8 +111,8 @@ def _parse_ripf(ripf: str) -> tuple[str, str, str, str]: match = re.fullmatch(r"r(\d+)i(\d+)p(\d+)f(\d+)", ripf.strip()) if not match: raise ValueError( - f"Invalid realization-initialization-physics-forcing string {ripf!r}; " - "expected rXiYpZfW (e.g. r1i1p1f1)" + "Invalid realization-initialization-physics-forcing; " + f"expected rXiYpZfW (e.g. r1i1p1f1) got {ripf}" ) realization, initialization, physics, forcing = match.groups() return f"r{realization}", f"i{initialization}", f"p{physics}", f"f{forcing}" @@ -133,9 +136,6 @@ def build_dataset_cfg( resolution come from the per-model ``dataset`` block in ``data/_regrid_maps.yaml``. Replaces the packaged cmor_dataset*.json. """ - # Local imports avoid an import cycle with regrid_maps. - from .regrid_maps import load_regrid_maps, get_grid_names - cv = _load_controlled_vocabulary(str(tables_root)) meta = load_regrid_maps(model).get("dataset") or {} base_source_id = meta.get("base_source_id") diff --git a/src/cmip7_prep/regrid_maps.py b/src/cmip7_prep/regrid_maps.py index 4a06a43a..b99f6e3f 100644 --- a/src/cmip7_prep/regrid_maps.py +++ b/src/cmip7_prep/regrid_maps.py @@ -113,7 +113,8 @@ def get_grid_desc(model: str, resolution: str, realm: str) -> str: realm_descs = grid_descs.get(resolution) or {} if realm not in realm_descs: raise ValueError( - f"No grid descriptions defined for model={model}, resolution={resolution}, realm={realm}; " + f"No grid descriptions defined for model={model}, " + f"resolution={resolution}, realm={realm}; " f"available: {sorted(realm_descs)}" ) return grid_descs[resolution][realm] or "" From 456ebea4a8009bf91f03c2f2df60666d06d54887 Mon Sep 17 00:00:00 2001 From: maritsandstad Date: Mon, 5 Oct 2026 22:35:07 +0200 Subject: [PATCH 6/9] Fix tests --- tests/test_convert_csv_to_yaml.py | 2 +- tests/test_validate_cmor_output.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_convert_csv_to_yaml.py b/tests/test_convert_csv_to_yaml.py index de019b5c..d04dc9a8 100644 --- a/tests/test_convert_csv_to_yaml.py +++ b/tests/test_convert_csv_to_yaml.py @@ -578,7 +578,7 @@ def test_dataset_overrides_noresm(self, tmp_path): ] data = read_csv(_write_temp_csv(tmp_path, self.FIELDNAMES, rows), self.CFG) assert data["atmos"]["dataset_overrides"]["institution_id"] == "NCC" - assert data["atmos"]["dataset_overrides"]["source_id"] == "NorESM3" + assert data["atmos"]["dataset_overrides"]["source_id"] == "NorESM3-LM" def test_dims_not_written_to_yaml(self, tmp_path): """The Dimensions column is read but not copied into the YAML. diff --git a/tests/test_validate_cmor_output.py b/tests/test_validate_cmor_output.py index 580baec9..ca77bfcd 100644 --- a/tests/test_validate_cmor_output.py +++ b/tests/test_validate_cmor_output.py @@ -63,7 +63,7 @@ def test_scan_output_tree_inventories_dims(tmp_path): / "CMIP7" / "CMIP" / "NCC" - / "NorESM3" + / "NorESM3-LM" / "piControl" / "r1i1p1f1" / "glb" @@ -97,7 +97,7 @@ def test_scan_output_tree_inventories_dims(tmp_path): expected_variables={"tas_hxy"}, ensemble_member=None, ) - + print(produced) assert sorted(produced) == ["tas_hxy"] assert not errors summary = summarize_dimension_inventory(inventory) From 1aac3e1808dadc25b64d8ca464087e0d14e3ba65 Mon Sep 17 00:00:00 2001 From: maritsandstad Date: Tue, 6 Oct 2026 14:05:25 +0200 Subject: [PATCH 7/9] regrid maps for cesm --- data/cesm_regrid_maps.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/cesm_regrid_maps.yaml b/data/cesm_regrid_maps.yaml index a363d0fc..330b89a7 100644 --- a/data/cesm_regrid_maps.yaml +++ b/data/cesm_regrid_maps.yaml @@ -48,4 +48,4 @@ dataset: source_ids: {} default_source_id: CESM3 nominal_resolution: {} - default_nominal_resolution: 100 km \ No newline at end of file + default_nominal_resolution: 100 km From 930c15c6f16e32e11009302877f03e37000ad722 Mon Sep 17 00:00:00 2001 From: maritsandstad Date: Wed, 7 Oct 2026 09:40:19 +0200 Subject: [PATCH 8/9] Fix test to new CmorSession setup --- tests/test_cmor_writer.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/tests/test_cmor_writer.py b/tests/test_cmor_writer.py index 0d0e40d9..54debbca 100644 --- a/tests/test_cmor_writer.py +++ b/tests/test_cmor_writer.py @@ -270,10 +270,18 @@ class BrandedName: # pylint: disable=too-few-public-methods def _session(tmp_path, realm): """Return a CmorSession for one realm, writing into tmp_path.""" + dataset_cfg = build_dataset_cfg( + model="cesm", + resolution="ne30", + experiment="piControl", + frequency="month", + ripf="r1i1p1f1", + tables_root=Path(__file__).parent.parent / "cmip7-cmor-tables", + realm=realm, + ) return CmorSession( tables_root=Path(__file__).parent.parent / "cmip7-cmor-tables", - dataset_json=Path(__file__).parent.parent / "data" / "cmor_dataset.json", - dataset_attrs={"institution_id": "NCAR", "GLOBAL_IS_CMIP7": True}, + dataset_attrs=dataset_cfg, log_dir=tmp_path, log_name=f"cmor_{realm}.log", realm=realm, From 921b50d5413eaa7a68f5a5aa31896d9903d4ba49 Mon Sep 17 00:00:00 2001 From: maritsandstad Date: Wed, 7 Oct 2026 09:42:37 +0200 Subject: [PATCH 9/9] linting --- tests/test_cmor_writer.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/test_cmor_writer.py b/tests/test_cmor_writer.py index 54debbca..13202458 100644 --- a/tests/test_cmor_writer.py +++ b/tests/test_cmor_writer.py @@ -271,14 +271,14 @@ class BrandedName: # pylint: disable=too-few-public-methods def _session(tmp_path, realm): """Return a CmorSession for one realm, writing into tmp_path.""" dataset_cfg = build_dataset_cfg( - model="cesm", - resolution="ne30", - experiment="piControl", - frequency="month", - ripf="r1i1p1f1", - tables_root=Path(__file__).parent.parent / "cmip7-cmor-tables", - realm=realm, - ) + model="cesm", + resolution="ne30", + experiment="piControl", + frequency="month", + ripf="r1i1p1f1", + tables_root=Path(__file__).parent.parent / "cmip7-cmor-tables", + realm=realm, + ) return CmorSession( tables_root=Path(__file__).parent.parent / "cmip7-cmor-tables", dataset_attrs=dataset_cfg,