diff --git a/cmip7-cmor-tables b/cmip7-cmor-tables index cf46e3c..338c9a5 160000 --- a/cmip7-cmor-tables +++ b/cmip7-cmor-tables @@ -1 +1 @@ -Subproject commit cf46e3c000f0c8d798efd786ae593872f7bf49fe +Subproject commit 338c9a5ad61b9eb1207f5c7cfbf99dcae196e674 diff --git a/data/cesm_regrid_maps.yaml b/data/cesm_regrid_maps.yaml index 56cf927..330b89a 100644 --- a/data/cesm_regrid_maps.yaml +++ b/data/cesm_regrid_maps.yaml @@ -20,3 +20,32 @@ resolutions: regular: conservative: cpl/gridmaps/tx2_3v2/map_t232_TO_1x1d_aave.251023.nc bilinear: cpl/gridmaps/tx2_3v2/map_t232_TO_1x1d_blin.251023.nc + + +grid_names_per_realm: + ne16: + atmos: g123 + aerosol: g123 + atmosChem: g123 + land: g123 + seaIce: g202 + landIce: g194 + ne30: + atmos: g106 + aerosol: g106 + atmosChem: g106 + land: g106 + seaIce: g106 + landIce: g106 + +# Static dataset (global attribute) metadata for the CMOR dataset JSON builder; +# see noresm_regrid_maps.yaml for the schema. CESM writes CESM3 at every +# resolution for now (placeholder). +dataset: + base_source_id: CESM3 + source_type: AOGCM + calendar: noleap + source_ids: {} + default_source_id: CESM3 + nominal_resolution: {} + default_nominal_resolution: 100 km diff --git a/data/cmor_dataset.json b/data/cmor_dataset.json deleted file mode 100644 index fcd2666..0000000 --- a/data/cmor_dataset.json +++ /dev/null @@ -1,50 +0,0 @@ -{ - "project_id": "CMIP7", - "mip_era": "CMIP7", - "activity_id": "CMIP", - "drs_specs": "MIP-DRS7", - "institution_id": "NCAR", - "institution": "NSF National Center for Atmospheric Research, Climate and Global Dynamics Laboratory, 1850 Table Mesa Drive, Boulder, CO 80305, USA", - "contact": "cesm-cmip@ucar.edu", - "product": "model-output", - "_cmip7_option": true, - "horizontal_label": "hxy", - "vertical_label": "l", - "temporal_label": "tavg", - "source_id": "CESM3", - "source": "CESM3 (2025): atmosphere: CAM; ocean: MOM; sea ice: CICE6; land: CTSM; aerosol: MAM4; coupler: CMEPS.", - "source_type": "AOGCM", - "nominal_resolution": "100 km", - "branding_suffix": "tavg-l-hxy-u", - "experiment_id": "piControl", - "experiment": "Pre-industrial control simulation with prescribed carbon dioxide concentrations (for prescribed carbon dioxide emissions, see `esm-piControl`). Used to characterise natural variability and unforced behaviour.", - "branch_method": "no parent", - "branch_time_in_parent": 0.0, - "branch_time_in_child": 0.0, - "parent_time_units": "days since 1850-01-01-00000", - "parent_variant_label": "r1i1p1f1", - "parent_mip_era": "CMIP7", - "parent_source_id": "CESM3", - "parent_activity_id": "CMIP", - "parent_experiment_id": "piControl-spinup", - "sub_experiment_id": "none", - "sub_experiment": "none", - "archive_id": "WCRP", - "area_label" : "u", - "license_id": "CC-BY-4.0", - "realization_index": "r1", - "initialization_index": "i1", - "physics_index": "p1", - "forcing_index": "f1", - "member_id": "r1i1p1f1", - "calendar": "noleap", - "frequency": "mon", - "temporal_subset": "1850", - "grid": "data mapped to a 1x1 degree.", - "grid_label": "gr", - "title": "CESM3 piControl on 1x1 degree grid (regridded from ne30np3 se grid)", - "references": "See CESM3 documentation and CMIP7 references for NCAR/CGD.", - "outpath": ".", - "output_path_template": "", - "output_file_template": "" -} diff --git a/data/cmor_dataset_noresm.json b/data/cmor_dataset_noresm.json deleted file mode 100644 index a73fcaf..0000000 --- a/data/cmor_dataset_noresm.json +++ /dev/null @@ -1,51 +0,0 @@ -{ - "project_id": "CMIP7", - "mip_era": "CMIP7", - "activity_id": "CMIP", - "drs_specs": "MIP-DRS7", - "institution_id": "NCC", - "institution": "NorESM Climate modeling Consortium consisting of CICERO (Center for International Climate and Environmental Research, Oslo 0349), MET-Norway (Norwegian Meteorological Institute, Oslo 0313), NERSC (Nansen Environmental and Remote Sensing Center, Bergen 5006), NILU (Norwegian Institute for Air Research, Kjeller 2027), UiB (University of Bergen, Bergen 5007), UiO (University of Oslo, Oslo 0313) and UNI (Uni Research, Bergen 5008), Norway. Mailing address: NCC, c/o MET-Norway, Henrik Mohns plass 1, Oslo 0313, Norway", - "contact": "Please send any requests or bug reports to noresm-ncc@met.no.", - "product": "model-output", - "_cmip7_option": true, - "horizontal_label": "hxy", - "vertical_label": "l", - "temporal_label": "tavg", - "source_id": "NorESM3", - "source": "NorESM3 (2025): atmosphere: CAM; atmosChem: OsloChemSimp; ocean: BLOM; oceanBgchem: HAMOCC; sea ice: CICE6; land: CLM-FATES; aerosol: OsloAero; coupler: CMEPS.", - "source_type": "AOGCM", - "nominal_resolution": "100 km", - "region": "glb", - "branding_suffix": "tavg-l-hxy-u", - "experiment_id": "piControl", - "experiment": "Pre-industrial control simulation with prescribed carbon dioxide concentrations (for prescribed carbon dioxide emissions, see `esm-piControl`). Used to characterise natural variability and unforced behaviour.", - "branch_method": "no parent", - "branch_time_in_parent": 0.0, - "branch_time_in_child": 0.0, - "parent_time_units": "days since 1850-01-01-00000", - "parent_variant_label": "r1i1p1f1", - "parent_mip_era": "CMIP7", - "parent_source_id": "NorESM3", - "parent_activity_id": "CMIP", - "parent_experiment_id": "piControl-spinup", - "sub_experiment_id": "none", - "sub_experiment": "none", - "archive_id": "WCRP", - "area_label" : "u", - "license_id": "CC-BY-4.0", - "realization_index": "r1", - "initialization_index": "i1", - "physics_index": "p1", - "forcing_index": "f1", - "member_id": "r1i1p1f1", - "calendar": "noleap", - "frequency": "mon", - "temporal_subset": "1850", - "grid": "data mapped to a 2x2 degree.", - "grid_label": "gr", - "title": "NorESM3 piControl on 2x2 degree grid (regridded from ne16np3 se grid)", - "references": "See NorESM3 documentation and CMIP7 references for NCAR/CGD.", - "outpath": ".", - "output_path_template": "", - "output_file_template": "" -} diff --git a/data/noresm_regrid_maps.yaml b/data/noresm_regrid_maps.yaml index 4706653..1240ba1 100644 --- a/data/noresm_regrid_maps.yaml +++ b/data/noresm_regrid_maps.yaml @@ -30,3 +30,37 @@ resolutions: regular: conservative: map_tnx1v4_to_1x1_aave_c260531.nc bilinear: map_tnx1v4_to_1x1_blin_c260531.nc + +grid_names_per_realm: + ne16: + atmos: g123 + aerosol: g123 + atmosChem: g123 + land: g123 + seaIce: g202 + landIce: g194 + ne30: + atmos: g106 + aerosol: g106 + atmosChem: g106 + land: g106 + seaIce: g106 + landIce: g106 + +# Static dataset (global attribute) metadata for the CMOR dataset JSON builder. +# base_source_id is the key looked up in cmor-cvs.json for institution_id and +# source text; it must be a CV source_id entry. source_ids/nominal_resolution +# are the branded, per-resolution values actually written to the output; the +# default_* keys catch any resolution not listed (e.g. non-CMIP test runs). +dataset: + base_source_id: NorESM3 + source_type: AOGCM + calendar: noleap + source_ids: + ne16: NorESM3-LM + ne30: NorESM3-MM + default_source_id: NorESM3 + nominal_resolution: + ne16: 250 km + ne30: 100 km + default_nominal_resolution: 100 km diff --git a/scripts/cmor_driver.py b/scripts/cmor_driver.py index 5506b5e..7f78214 100755 --- a/scripts/cmor_driver.py +++ b/scripts/cmor_driver.py @@ -14,6 +14,11 @@ from concurrent.futures import as_completed +import inspect + +print(inspect.getframeinfo(inspect.getouterframes(inspect.currentframe())[0][0])[0]) + + import os from pathlib import Path import logging @@ -27,13 +32,21 @@ import json import numpy as np import xarray as xr + +# Load all possible cmip vars for this realm and this experiment +# The data_request_api is a CMIP7-specific Python package that is +# separate from CMOR itself but closely related to it. +# It produce lists of variables requested for each CMIP7 experiment +from data_request_api.query import data_request as dr +from data_request_api.content import dump_transformation as dt + from cmor import set_cur_dataset_attribute from cmip7_prep.cmor_utils import ( load_positive_overrides, bounds_from_centers_1d, roll_for_monotonic_with_bounds, - packaged_dataset_json, + build_dataset_cfg, ) from cmip7_prep.include_patterns import ( get_include_patterns, @@ -52,6 +65,7 @@ from cmip7_prep.mom6_static import ocean_fx_fields from cmip7_prep.variable_selection import assemble_yaml_defined_cmip_vars + from dask import delayed # The only logging configuration in the package: library modules attach @@ -484,9 +498,6 @@ def process_one_var( """Compute+write one CMIP variable. Returns a list of (varname, 'ok' or error message) tuples.""" varname = cmip_var.branded_variable_name.name - realization_index, initialization_index, physics_index, forcing_index = ( - parse_realization_initialization_physics_forcing(ripf_index) - ) # At this point you have a cmip_var (metadata from database query for the target variable) # queried a cmor database from the cloud logger.debug(f"Starting processing for variable: {varname}") @@ -624,39 +635,31 @@ def process_one_var( try: log_dir = outdir / "logs" - # TODO: add NorESM institution_id below - # Initialize CMOR class - metadata_json = None - if model == "noresm": - metadata_json = packaged_dataset_json("cmor_dataset_noresm.json") - + # Global/dataset attributes assembled from the CV plus the + # per-model dataset config; replaces the packaged cmor_dataset + # JSON files. frequency, variant indices, experiment metadata + # and the per-realm grid_label are already baked in, so only the + # per-variable region is set below. + dataset_cfg = build_dataset_cfg( + model=model, + resolution=resolution, + experiment=experiment, + frequency=frequency, + ripf=ripf_index, + tables_root=tables_root, + realm=realm, + ) with CmorSession( tables_root=tables_root, log_dir=log_dir, log_name=f"cmor_{datetime.now(UTC).strftime('%Y%m%dT%H%M%SZ')}_{varname}.log", - dataset_json=metadata_json, - dataset_attrs={"institution_id": "NCC", "GLOBAL_IS_CMIP7": True}, + dataset_attrs=dataset_cfg, outdir=outdir, ice_sheet=ice_sheet, realm=realm, ) as cm: - set_cur_dataset_attribute("frequency", frequency) - set_cur_dataset_attribute("realization_index", realization_index) - set_cur_dataset_attribute( - "initialization_index", initialization_index - ) - set_cur_dataset_attribute("physics_index", physics_index) - set_cur_dataset_attribute("forcing_index", forcing_index) region = write_cfg.get("region", "glb") set_cur_dataset_attribute("region", region) - # Updating with correct experiment info from CMIP7 tables - experiment_info = get_experiment_info_from_tables( - experiment, tables_root - ) - for key, value in experiment_info.items(): - if isinstance(value, list): - value = value[0] - set_cur_dataset_attribute(key, value) logger.info( f"Writing CMOR variable {cmip7name.name} with frequency {frequency}" @@ -674,9 +677,9 @@ def process_one_var( "standard_name": write_cfg.get("standard_name", None), "levels": write_cfg.get("levels", None), "branded_variable_name": cmip7name, + "grid_label": dataset_cfg.get("grid_label"), }, )() - # Now use CMOR utility to write out netcdf variable cm.write_variable(ds_cmor_write, cmip_var, vdef) @@ -782,13 +785,6 @@ def main(): mapping = Mapping.from_packaged_default(filename=yaml_filename) mapping.default_freq = frequency - # Load all possible cmip vars for this realm and this experiment - # The data_request_api is a CMIP7-specific Python package that is - # separate from CMOR itself but closely related to it. - # It produce lists of variables requested for each CMIP7 experiment - from data_request_api.query import data_request as dr - from data_request_api.content import dump_transformation as dt - logger.info("Loading data request content %s", realm) content_dic = dt.get_transformed_content() logger.info("Content dictionary obtained") diff --git a/scripts/convert_csv_to_yaml.py b/scripts/convert_csv_to_yaml.py index d8655a6..5a14991 100755 --- a/scripts/convert_csv_to_yaml.py +++ b/scripts/convert_csv_to_yaml.py @@ -82,7 +82,7 @@ def load_intensive_vars() -> set[str]: "normalize_dim_names": True, "dataset_overrides": { "institution_id": "NCC", - "source_id": "NorESM3", + "source_id": "NorESM3-LM", "nominal_resolution": "200 km", "yaml_coax_dummy": [0, 1, 2], }, diff --git a/scripts/validate_cmor_output.py b/scripts/validate_cmor_output.py index bfb03e3..5e06de4 100644 --- a/scripts/validate_cmor_output.py +++ b/scripts/validate_cmor_output.py @@ -40,7 +40,7 @@ logger = logging.getLogger("cmip7_prep.validate_cmor_output") MODEL_NAMING_MAPS = { - "noresm": ["NCC", "NorESM3"], + "noresm": ["NCC", "NorESM3-LM"], "cesm": ["NCAR", "CESM3"], } diff --git a/src/cmip7_prep/cmor_utils.py b/src/cmip7_prep/cmor_utils.py index 958cca6..29847ed 100644 --- a/src/cmip7_prep/cmor_utils.py +++ b/src/cmip7_prep/cmor_utils.py @@ -2,6 +2,7 @@ from functools import lru_cache from pathlib import Path +import json import warnings import re import datetime as dt @@ -14,6 +15,9 @@ import yaml import xarray as xr +# Local imports avoid an import cycle with regrid_maps. +from .regrid_maps import load_regrid_maps, get_grid_names + _FILL_DEFAULT = 1.0e20 _DATA_DIR = Path(__file__).parent.parent.parent / "data" @@ -49,6 +53,193 @@ def packaged_dataset_json(filename: str = "cmor_dataset.json"): return as_file(res) +# --------------------------------------------------------------------- +# CMOR dataset JSON builder +# --------------------------------------------------------------------- +# institution_id per model, matching the CV source_id entries. Used as a +# fallback only, when the CV lookup by base_source_id yields nothing. +_INSTITUTION_ID_BY_MODEL = {"noresm": "NCC", "cesm": "NCAR"} + +# DRS path/filename templates; structural, so they live here rather than in the +# CV or the per-model config. +_OUTPUT_PATH_TEMPLATE = ( + "" + "" +) +_OUTPUT_FILE_TEMPLATE = ( + "" + "" +) + + +@lru_cache(maxsize=None) +def _load_controlled_vocabulary(tables_root: str) -> dict: + """Read and cache the CV block of cmor-cvs.json for a tables root.""" + path = Path(tables_root) / "tables-cvs" / "cmor-cvs.json" + with open(path, encoding="utf-8") as handle: + return json.load(handle).get("CV", {}) + + +def _first(value, default=None): + """Return value[0] for a list/tuple, the value itself otherwise, else default.""" + if isinstance(value, (list, tuple)): + return value[0] if value else default + return value if value is not None else default + + +def _resolve_by_resolution(mapping, resolution: str, default): + """Return mapping[resolution] when mapping is a dict with that key, else default.""" + if isinstance(mapping, dict): + return mapping.get(resolution, default) + return default + + +def _format_license(license_cv: dict, license_id: str, institution_id: str) -> str: + """Expand the CV license_template for one license_id and institution.""" + template = license_cv.get("license_template", "") + entry = (license_cv.get("license_id") or {}).get(license_id, {}) + return ( + template.replace("", license_id) + .replace("", institution_id or "") + .replace("", entry.get("license_type", "")) + .replace("", entry.get("license_url", "")) + ) + + +def _parse_ripf(ripf: str) -> tuple[str, str, str, str]: + """Split 'rXiYpZfW' into its ('rX', 'iY', 'pZ', 'fW') CMIP7 index strings.""" + match = re.fullmatch(r"r(\d+)i(\d+)p(\d+)f(\d+)", ripf.strip()) + if not match: + raise ValueError( + "Invalid realization-initialization-physics-forcing; " + f"expected rXiYpZfW (e.g. r1i1p1f1) got {ripf}" + ) + realization, initialization, physics, forcing = match.groups() + return f"r{realization}", f"i{initialization}", f"p{physics}", f"f{forcing}" + + +def build_dataset_cfg( + *, + model: str, + resolution: str, + experiment: str, + frequency: str, + ripf: str, + tables_root, + realm: str = "atmos", + license_id: str = "CC-BY-4.0", +) -> dict: + """Assemble the CMOR dataset-attribute dict for one run. + + Institutional, source, license and experiment metadata are read from the + controlled vocabulary (cmor-cvs.json); the branded source_id and nominal + resolution come from the per-model ``dataset`` block in + ``data/_regrid_maps.yaml``. Replaces the packaged cmor_dataset*.json. + """ + cv = _load_controlled_vocabulary(str(tables_root)) + meta = load_regrid_maps(model).get("dataset") or {} + base_source_id = meta.get("base_source_id") + + # grid_label is the per-realm grid code from the regrid-map table; its + # human-readable description comes from the CV keyed by that same code. + try: + grid_label = get_grid_names(model, resolution, realm) + except ValueError: + logger.warning( + "No grid name for model=%s resolution=%s realm=%s; defaulting " + "grid_label to 'gr'", + model, + resolution, + realm, + ) + grid_label = "gr" + grid_desc = (cv.get("grid_label") or {}).get(grid_label, "") + + # Branded source_id/nominal_resolution written to the output, by resolution. + source_id = _resolve_by_resolution( + meta.get("source_ids"), + resolution, + meta.get("default_source_id", base_source_id), + ) + nominal_resolution = _resolve_by_resolution( + meta.get("nominal_resolution"), + resolution, + meta.get("default_nominal_resolution"), + ) + + # institution/source derived from the CV via the base (CV-valid) source_id. + source_entry = (cv.get("source_id") or {}).get(base_source_id, {}) + institution_id = _first( + source_entry.get("institution_id"), _INSTITUTION_ID_BY_MODEL.get(model) + ) + institution = (cv.get("institution_id") or {}).get(institution_id, "") + source = source_entry.get("source", base_source_id) + license_str = _format_license(cv.get("license") or {}, license_id, institution_id) + + # experiment metadata (activity_id, parentage) straight from the CV. + experiment_entry = (cv.get("experiment_id") or {}).get(experiment, {}) + activity_id = _first(experiment_entry.get("activity_id"), "CMIP") + parent_experiment_id = _first( + experiment_entry.get("parent_experiment_id"), "no parent" + ) + parent_activity_id = _first(experiment_entry.get("parent_activity_id"), activity_id) + + realization_index, initialization_index, physics_index, forcing_index = _parse_ripf( + ripf + ) + mip_era = cv.get("mip_era", "CMIP7") + + return { + "mip_era": mip_era, + "activity_id": activity_id, + "drs_specs": cv.get("drs_specs", "MIP-DRS7"), + "data_specs_version": cv.get("data_specs_version"), + "Conventions": _first(cv.get("Conventions"), "CF-1.12"), + "product": "model-output", + "_cmip7_option": True, + "institution_id": institution_id, + "institution": institution, + "source_id": source_id, + "source": source, + "source_type": meta.get("source_type", "AOGCM"), + "nominal_resolution": nominal_resolution, + "license_id": license_id, + "license": license_str, + "calendar": meta.get("calendar", "noleap"), + "frequency": frequency, + "experiment_id": experiment, + "experiment": experiment_entry.get("experiment", experiment), + "sub_experiment_id": "none", + "sub_experiment": "none", + "realization_index": realization_index, + "initialization_index": initialization_index, + "physics_index": physics_index, + "forcing_index": forcing_index, + "variant_label": ripf, + "member_id": ripf, + "branch_method": "no parent", + "branch_time_in_parent": 0.0, + "branch_time_in_child": 0.0, + "parent_mip_era": mip_era, + "parent_activity_id": parent_activity_id, + "parent_experiment_id": parent_experiment_id, + "parent_source_id": source_id, + "parent_variant_label": ripf, + "parent_time_units": "days since 1850-01-01", + "horizontal_label": "hxy", + "vertical_label": "l", + "temporal_label": "tavg", + "area_label": "u", + "branding_suffix": "tavg-l-hxy-u", + "grid_label": grid_label, + "grid": grid_desc, + "region": "glb", + "tracking_prefix": cv.get("tracking_prefix", "hdl:21.14107"), + "output_path_template": _OUTPUT_PATH_TEMPLATE, + "output_file_template": _OUTPUT_FILE_TEMPLATE, + } + + def filled_for_cmor( da: xr.DataArray, fill: float | None = None ) -> tuple[xr.DataArray, float]: diff --git a/src/cmip7_prep/cmor_writer.py b/src/cmip7_prep/cmor_writer.py index 002dd0a..f275473 100644 --- a/src/cmip7_prep/cmor_writer.py +++ b/src/cmip7_prep/cmor_writer.py @@ -2,10 +2,10 @@ """Thin CMOR wrapper used by cmip7_prep. This module centralizes CMOR session setup and writing so that the rest of the -pipeline can stay xarray-first. It supports either a dataset JSON file (preferred) -or directly injected global attributes, and creates axes based on the coordinates -present in the provided dataset. It also supports a packaged default -`cmor_dataset.json` living under `cmip7_prep/data/`. +pipeline can stay xarray-first. It takes the global/dataset attributes as an +in-memory dict (built by ``cmor_utils.build_dataset_cfg``), or, as a fallback, a +dataset JSON path, and creates axes based on the coordinates present in the +provided dataset. """ from pathlib import Path @@ -23,7 +23,6 @@ import numpy as np import xarray as xr from .cmor_utils import ( - packaged_dataset_json, get_cmor_attr, set_cmor_attr, encode_time_to_num, @@ -132,17 +131,19 @@ class CmorSession( tables_root : str or Path Directory containing CMOR tables and tables-csv directories with JSONs (e.g., CMIP6_*.json or CMIP7_*.json). - dataset_json : str or Path, optional - Path to a cmor_dataset.json with the experiment/source metadata. dataset_attrs : dict, optional - Alternative to `dataset_json`, allowing direct attribute injection. + Global/dataset attributes to write, typically produced by + ``cmor_utils.build_dataset_cfg``. Preferred over ``dataset_json``. + dataset_json : str or Path, optional + Path to a cmor_dataset.json with the experiment/source metadata; a + fallback for callers that still carry a file. """ def __init__( self, *, tables_root: Path | str, - dataset_attrs: dict[str, str] | None = None, + dataset_attrs: dict[str, Any] | None = None, dataset_json: Optional[DatasetJsonLike] = None, tracking_prefix: str | None = None, # NEW: one log per run (session) @@ -222,22 +223,27 @@ def __enter__(self) -> "CmorSession": getattr(cmor, name)(str(self._log_path)) break - # Resolve dataset_json to a real filesystem path - dj = self.dataset_json - if dj is None: - # packaged file → returns a context manager - cm = packaged_dataset_json("cmor_dataset.json") - self._dataset_json_cm = cm - p = cm.__enter__() # ← ENTER the CM, get a Path - elif isinstance(dj, (str, Path)): - p = Path(dj) + # Resolve the dataset attributes: the in-memory dict is preferred; a + # dataset_json path remains as a fallback for callers that still have a + # file. Either way CMOR is fed a temp JSON, since cmor.dataset_json + # must be called to load the CV before any variable is defined. + if self.dataset_attrs: + cfg = dict(self.dataset_attrs) + elif self.dataset_json is not None: + dj = self.dataset_json + if isinstance(dj, (str, Path)): + p = Path(dj) + else: + # caller passed a context manager directly + self._dataset_json_cm = dj + p = dj.__enter__() # ← ENTER the CM, get a Path + logger.debug("CMOR JSON dataset: %s", p) + with open(p, encoding="utf-8") as f: + cfg = json.load(f) else: - # caller passed a context manager directly - self._dataset_json_cm = dj - p = dj.__enter__() # ← ENTER the CM, get a Path - logger.debug("CMOR JSON dataset: %s", p) - with open(p, encoding="utf-8") as f: - cfg = json.load(f) + raise ValueError( + "CmorSession requires either dataset_attrs or dataset_json" + ) cfg["outpath"] = str(self._outdir) tmp = tempfile.NamedTemporaryFile("w", suffix=".json", delete=False) json.dump(cfg, tmp) @@ -1284,12 +1290,12 @@ def _write_fx_2d(self, ds: xr.Dataset, name: str, units: str) -> None: logger.debug("FX variable %s define lon_id %s", name, lon_id) logger.debug("Writing fx variable %s on curvilinear grid", name) cmor.set_cur_dataset_attribute("grid", "curvilinear") - cmor.set_cur_dataset_attribute("grid_label", "gn") + # cmor.set_cur_dataset_attribute("grid_label", "gn") if name == "deptho": name = "deptho_ti-u-hxy-sea" else: cmor.set_cur_dataset_attribute("grid", "1x1 degree") - cmor.set_cur_dataset_attribute("grid_label", "gr") + # cmor.set_cur_dataset_attribute("grid_label", "gr") if name in ("areacella_ti-u-hxy-u", "sftlf_ti-u-hxy-u"): self.load_table(self.tables_root, "land") elif name in ("sftof_ti-u-hxy-u", "deptho", "areacello"): @@ -1516,12 +1522,14 @@ def write_variable( missing_value=fillv, ) logger.debug("Now define time dimension and write data") - if "lat" in data.dims and "lon" in data.dims: - cmor.set_cur_dataset_attribute("grid", "1x1 degree") - cmor.set_cur_dataset_attribute("grid_label", "gr") - else: - cmor.set_cur_dataset_attribute("grid", "curvilinear") - cmor.set_cur_dataset_attribute("grid_label", "gn") + # if "lat" in data.dims and "lon" in data.dims: + # # These should be pulled fromt he grid in the tables + # cmor.set_cur_dataset_attribute("grid", "1x1 degree") + # cmor.set_cur_dataset_attribute("grid_label", vdef.get("grid_label", "gr")) + # else: + # # These should be pulled fromt he grid in the tables + # cmor.set_cur_dataset_attribute("grid", "curvilinear") + # cmor.set_cur_dataset_attribute("grid_label", vdef.get("grid_label", "gn")) # ---- Prepare time info for this write (local, not cached) ---- time_da = ds.coords.get("time") if time_da is None: diff --git a/src/cmip7_prep/regrid_maps.py b/src/cmip7_prep/regrid_maps.py index 5c4520a..b99f6e3 100644 --- a/src/cmip7_prep/regrid_maps.py +++ b/src/cmip7_prep/regrid_maps.py @@ -77,3 +77,44 @@ def get_map_paths(model: str, resolution: str) -> dict[str, Path]: "every resolution needs one, since fx fields are always conservative" ) return {method: root / name for method, name in entry.items()} + + +def get_grid_names(model: str, resolution: str, realm: str) -> dict[str, str]: + """Return the grid names for one model and resolution. + + The result has keys like 'atmos', 'aerosol', 'land', 'seaIce', 'landIce'. + """ + table = _load_cached(model) + grid_names = table.get("grid_names_per_realm") or {} + if resolution not in grid_names: + raise ValueError( + f"No grid names defined for model={model}, resolution={resolution}; " + f"available: {sorted(grid_names)}" + ) + realm_grids = grid_names.get(resolution) or {} + if realm not in realm_grids: + raise ValueError( + f"No grid names defined for model={model}, resolution={resolution}, realm={realm}; " + f"available: {sorted(realm_grids)}" + ) + + return grid_names[resolution][realm] or {} + + +def get_grid_desc(model: str, resolution: str, realm: str) -> str: + """Return the grid description for one model and resolution.""" + table = _load_cached(model) + grid_descs = table.get("grid_descs_per_realm") or {} + if resolution not in grid_descs: + raise ValueError( + f"No grid descriptions defined for model={model}, resolution={resolution}; " + f"available: {sorted(grid_descs)}" + ) + realm_descs = grid_descs.get(resolution) or {} + if realm not in realm_descs: + raise ValueError( + f"No grid descriptions defined for model={model}, " + f"resolution={resolution}, realm={realm}; " + f"available: {sorted(realm_descs)}" + ) + return grid_descs[resolution][realm] or "" diff --git a/tests/test_cmor_writer.py b/tests/test_cmor_writer.py index f83c9e2..1320245 100644 --- a/tests/test_cmor_writer.py +++ b/tests/test_cmor_writer.py @@ -7,6 +7,7 @@ import xarray as xr from cmip7_prep.cmor_writer import CmorSession +from cmip7_prep.cmor_utils import build_dataset_cfg def test_cmor_session_basic(tmp_path): @@ -57,22 +58,19 @@ class BrandedName: branded_variable_name = BrandedName() - dataset_json_path = Path(__file__).parent.parent / "data" / "cmor_dataset.json" + dataset_cfg = build_dataset_cfg( + model="cesm", + resolution="ne30", + experiment="piControl", + frequency="mon", + ripf="r1i1p1f1", + tables_root=tables_root, + ) log_dir = tmp_path log_name = "cmor_test.log" with CmorSession( tables_root=tables_root, - dataset_json=dataset_json_path, - dataset_attrs={ - "institution_id": "NCAR", - "GLOBAL_IS_CMIP7": True, - "branded_variable": { - "variable_id": "tas", - "table_id": "atmos", - "plev": 50000, - "description": "Vertical velocity at 500 hPa", - }, - }, + dataset_attrs=dataset_cfg, log_dir=log_dir, log_name=log_name, ) as session: @@ -140,22 +138,19 @@ class BrandedName: branded_variable_name = BrandedName() - dataset_json_path = Path(__file__).parent.parent / "data" / "cmor_dataset.json" + dataset_cfg = build_dataset_cfg( + model="cesm", + resolution="ne30", + experiment="piControl", + frequency="mon", + ripf="r1i1p1f1", + tables_root=tables_root, + ) log_dir = tmp_path log_name = "cmor_test_multi.log" with CmorSession( tables_root=tables_root, - dataset_json=dataset_json_path, - dataset_attrs={ - "institution_id": "NCAR", - "GLOBAL_IS_CMIP7": True, - "branded_variable": { - "variable_id": "tas", - "table_id": "atmos", - "plev": 50000, - "description": "Vertical velocity at 500 hPa", - }, - }, + dataset_attrs=dataset_cfg, log_dir=log_dir, log_name=log_name, ) as session: @@ -216,19 +211,17 @@ class BrandedName: branded_variable_name = BrandedName() - dataset_json_path = Path(__file__).parent.parent / "data" / "cmor_dataset.json" + dataset_cfg = build_dataset_cfg( + model="cesm", + resolution="ne30", + experiment="piControl", + frequency="mon", + ripf="r1i1p1f1", + tables_root=tables_root, + ) with CmorSession( tables_root=tables_root, - dataset_json=dataset_json_path, - dataset_attrs={ - "institution_id": "NCAR", - "GLOBAL_IS_CMIP7": True, - "branded_variable": { - "variable_id": "ta", - "table_id": "atmos", - "description": "Air Temperature", - }, - }, + dataset_attrs=dataset_cfg, log_dir=tmp_path, log_name="cmor_test_zonal_mean.log", ) as session: @@ -277,10 +270,18 @@ class BrandedName: # pylint: disable=too-few-public-methods def _session(tmp_path, realm): """Return a CmorSession for one realm, writing into tmp_path.""" + dataset_cfg = build_dataset_cfg( + model="cesm", + resolution="ne30", + experiment="piControl", + frequency="month", + ripf="r1i1p1f1", + tables_root=Path(__file__).parent.parent / "cmip7-cmor-tables", + realm=realm, + ) return CmorSession( tables_root=Path(__file__).parent.parent / "cmip7-cmor-tables", - dataset_json=Path(__file__).parent.parent / "data" / "cmor_dataset.json", - dataset_attrs={"institution_id": "NCAR", "GLOBAL_IS_CMIP7": True}, + dataset_attrs=dataset_cfg, log_dir=tmp_path, log_name=f"cmor_{realm}.log", realm=realm, diff --git a/tests/test_convert_csv_to_yaml.py b/tests/test_convert_csv_to_yaml.py index de019b5..d04dc9a 100644 --- a/tests/test_convert_csv_to_yaml.py +++ b/tests/test_convert_csv_to_yaml.py @@ -578,7 +578,7 @@ def test_dataset_overrides_noresm(self, tmp_path): ] data = read_csv(_write_temp_csv(tmp_path, self.FIELDNAMES, rows), self.CFG) assert data["atmos"]["dataset_overrides"]["institution_id"] == "NCC" - assert data["atmos"]["dataset_overrides"]["source_id"] == "NorESM3" + assert data["atmos"]["dataset_overrides"]["source_id"] == "NorESM3-LM" def test_dims_not_written_to_yaml(self, tmp_path): """The Dimensions column is read but not copied into the YAML. diff --git a/tests/test_validate_cmor_output.py b/tests/test_validate_cmor_output.py index 580baec..ca77bfc 100644 --- a/tests/test_validate_cmor_output.py +++ b/tests/test_validate_cmor_output.py @@ -63,7 +63,7 @@ def test_scan_output_tree_inventories_dims(tmp_path): / "CMIP7" / "CMIP" / "NCC" - / "NorESM3" + / "NorESM3-LM" / "piControl" / "r1i1p1f1" / "glb" @@ -97,7 +97,7 @@ def test_scan_output_tree_inventories_dims(tmp_path): expected_variables={"tas_hxy"}, ensemble_member=None, ) - + print(produced) assert sorted(produced) == ["tas_hxy"] assert not errors summary = summarize_dimension_inventory(inventory)