Trying to sync to more official version of tables

data/cesm_regrid_maps.yaml

@@ -20,3 +20,32 @@ resolutions:
   regular:
     conservative: cpl/gridmaps/tx2_3v2/map_t232_TO_1x1d_aave.251023.nc
     bilinear: cpl/gridmaps/tx2_3v2/map_t232_TO_1x1d_blin.251023.nc
+
+
+grid_names_per_realm:
+  ne16:
+    atmos: g123
+    aerosol: g123
+    atmosChem: g123
+    land: g123
+    seaIce: g202
+    landIce: g194
+  ne30:
+    atmos: g106
+    aerosol: g106
+    atmosChem: g106
+    land: g106
+    seaIce: g106
+    landIce: g106
+
+# Static dataset (global attribute) metadata for the CMOR dataset JSON builder;
+# see noresm_regrid_maps.yaml for the schema. CESM writes CESM3 at every
+# resolution for now (placeholder).
+dataset:
+  base_source_id: CESM3
+  source_type: AOGCM
+  calendar: noleap
+  source_ids: {}
+  default_source_id: CESM3
+  nominal_resolution: {}
+  default_nominal_resolution: 100 km

data/noresm_regrid_maps.yaml

@@ -30,3 +30,37 @@ resolutions:
   regular:
     conservative: map_tnx1v4_to_1x1_aave_c260531.nc
     bilinear: map_tnx1v4_to_1x1_blin_c260531.nc
+
+grid_names_per_realm:
+  ne16:
+    atmos: g123
+    aerosol: g123
+    atmosChem: g123
+    land: g123
+    seaIce: g202
+    landIce: g194
+  ne30:
+    atmos: g106
+    aerosol: g106
+    atmosChem: g106
+    land: g106
+    seaIce: g106
+    landIce: g106
+
+# Static dataset (global attribute) metadata for the CMOR dataset JSON builder.
+# base_source_id is the key looked up in cmor-cvs.json for institution_id and
+# source text; it must be a CV source_id entry. source_ids/nominal_resolution
+# are the branded, per-resolution values actually written to the output; the
+# default_* keys catch any resolution not listed (e.g. non-CMIP test runs).
+dataset:
+  base_source_id: NorESM3
+  source_type: AOGCM
+  calendar: noleap
+  source_ids:
+    ne16: NorESM3-LM
+    ne30: NorESM3-MM
+  default_source_id: NorESM3
+  nominal_resolution:
+    ne16: 250 km
+    ne30: 100 km
+  default_nominal_resolution: 100 km

scripts/cmor_driver.py

@@ -14,6 +14,11 @@
 from concurrent.futures import as_completed
 
 
+import inspect
+
+print(inspect.getframeinfo(inspect.getouterframes(inspect.currentframe())[0][0])[0])
+
+
 import os
 from pathlib import Path
 import logging
@@ -27,13 +32,21 @@
 import json
 import numpy as np
 import xarray as xr
+
+# Load all possible cmip vars for this realm and this experiment
+# The data_request_api is a CMIP7-specific Python package that is
+# separate from CMOR itself but closely related to it.
+# It produce lists of variables requested for each CMIP7 experiment
+from data_request_api.query import data_request as dr
+from data_request_api.content import dump_transformation as dt
+
 from cmor import set_cur_dataset_attribute
 
 from cmip7_prep.cmor_utils import (
     load_positive_overrides,
     bounds_from_centers_1d,
     roll_for_monotonic_with_bounds,
-    packaged_dataset_json,
+    build_dataset_cfg,
 )
 from cmip7_prep.include_patterns import (
     get_include_patterns,
@@ -52,6 +65,7 @@
 from cmip7_prep.mom6_static import ocean_fx_fields
 from cmip7_prep.variable_selection import assemble_yaml_defined_cmip_vars
 
+
 from dask import delayed
 
 # The only logging configuration in the package: library modules attach
@@ -484,9 +498,6 @@ def process_one_var(
     """Compute+write one CMIP variable. Returns a list of (varname, 'ok' or error message) tuples."""
     varname = cmip_var.branded_variable_name.name
 
-    realization_index, initialization_index, physics_index, forcing_index = (
-        parse_realization_initialization_physics_forcing(ripf_index)
-    )
     # At this point you have a cmip_var (metadata from database query for the target variable)
     # queried a cmor database from the cloud
     logger.debug(f"Starting processing for variable: {varname}")
@@ -624,39 +635,31 @@ def process_one_var(
             try:
                 log_dir = outdir / "logs"
 
-                # TODO: add NorESM institution_id below
-                # Initialize CMOR class
-                metadata_json = None
-                if model == "noresm":
-                    metadata_json = packaged_dataset_json("cmor_dataset_noresm.json")
-
+                # Global/dataset attributes assembled from the CV plus the
+                # per-model dataset config; replaces the packaged cmor_dataset
+                # JSON files.  frequency, variant indices, experiment metadata
+                # and the per-realm grid_label are already baked in, so only the
+                # per-variable region is set below.
+                dataset_cfg = build_dataset_cfg(
+                    model=model,
+                    resolution=resolution,
+                    experiment=experiment,
+                    frequency=frequency,
+                    ripf=ripf_index,
+                    tables_root=tables_root,
+                    realm=realm,
+                )
                 with CmorSession(
                     tables_root=tables_root,
                     log_dir=log_dir,
                     log_name=f"cmor_{datetime.now(UTC).strftime('%Y%m%dT%H%M%SZ')}_{varname}.log",
-                    dataset_json=metadata_json,
-                    dataset_attrs={"institution_id": "NCC", "GLOBAL_IS_CMIP7": True},
+                    dataset_attrs=dataset_cfg,
                     outdir=outdir,
                     ice_sheet=ice_sheet,
                     realm=realm,
                 ) as cm:
-                    set_cur_dataset_attribute("frequency", frequency)
-                    set_cur_dataset_attribute("realization_index", realization_index)
-                    set_cur_dataset_attribute(
-                        "initialization_index", initialization_index
-                    )
-                    set_cur_dataset_attribute("physics_index", physics_index)
-                    set_cur_dataset_attribute("forcing_index", forcing_index)
                     region = write_cfg.get("region", "glb")
                     set_cur_dataset_attribute("region", region)
-                    # Updating with correct experiment info from CMIP7 tables
-                    experiment_info = get_experiment_info_from_tables(
-                        experiment, tables_root
-                    )
-                    for key, value in experiment_info.items():
-                        if isinstance(value, list):
-                            value = value[0]
-                        set_cur_dataset_attribute(key, value)
 
                     logger.info(
                         f"Writing CMOR variable {cmip7name.name} with frequency {frequency}"
@@ -674,9 +677,9 @@ def process_one_var(
                             "standard_name": write_cfg.get("standard_name", None),
                             "levels": write_cfg.get("levels", None),
                             "branded_variable_name": cmip7name,
+                            "grid_label": dataset_cfg.get("grid_label"),
                         },
                     )()
-
                     # Now use CMOR utility to write out netcdf variable
                     cm.write_variable(ds_cmor_write, cmip_var, vdef)
 
@@ -782,13 +785,6 @@ def main():
         mapping = Mapping.from_packaged_default(filename=yaml_filename)
     mapping.default_freq = frequency
 
-    # Load all possible cmip vars for this realm and this experiment
-    # The data_request_api is a CMIP7-specific Python package that is
-    # separate from CMOR itself but closely related to it.
-    # It produce lists of variables requested for each CMIP7 experiment
-    from data_request_api.query import data_request as dr
-    from data_request_api.content import dump_transformation as dt
-
     logger.info("Loading data request content %s", realm)
     content_dic = dt.get_transformed_content()
     logger.info("Content dictionary obtained")

scripts/convert_csv_to_yaml.py

@@ -82,7 +82,7 @@ def load_intensive_vars() -> set[str]:
         "normalize_dim_names": True,
         "dataset_overrides": {
             "institution_id": "NCC",
-            "source_id": "NorESM3",
+            "source_id": "NorESM3-LM",
             "nominal_resolution": "200 km",
             "yaml_coax_dummy": [0, 1, 2],
         },

scripts/validate_cmor_output.py

@@ -40,7 +40,7 @@
 logger = logging.getLogger("cmip7_prep.validate_cmor_output")
 
 MODEL_NAMING_MAPS = {
-    "noresm": ["NCC", "NorESM3"],
+    "noresm": ["NCC", "NorESM3-LM"],
     "cesm": ["NCAR", "CESM3"],
 }