From b8ee36352d4d13c58819998b9ad4b6afa1058f63 Mon Sep 17 00:00:00 2001 From: Heiko Goelzer <35115552+hgoelzer@users.noreply.github.com> Date: Thu, 18 Jun 2026 19:06:35 +0200 Subject: [PATCH] Fix ST time encoding to use Jan 1 of year+1 per ISMIP7 convention MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit State variable (ST) timestamps now follow the ismip7-time-encoding reference: end-of-year snapshots are encoded as Jan 1 of the following year (e.g. simulation year 2015 → 2016-01-01), not Dec 31. Flux variable (FL) encoding is unchanged: mid-year timestamp Jul 1 with time bounds [Jan 1, Jan 1 next year]. experiments_ismip7.csv is redesigned to store nominal simulation years (start_year_min, start_year_max, end_year) instead of raw date strings. The checker derives expected FL/ST timestamps at runtime via the new _nominal_to_timestamp() helper, applying ±1 day tolerance for float32 encoding jitter. _check_naming() and _check_time() both receive var_type so the ST year offset (filename year N = timestamp year N+1) is handled correctly. The test suite and README are updated accordingly. --- README.md | 25 +++++++++++- compliance_checker.py | 66 +++++++++++++++++++++----------- experiments_ismip7.csv | 12 +++--- generate/generate_test_files.py | 14 ++++--- tests/test_compliance_checker.py | 4 +- 5 files changed, 83 insertions(+), 38 deletions(-) diff --git a/README.md b/README.md index ed7c1be..88d86f7 100644 --- a/README.md +++ b/README.md @@ -5,10 +5,29 @@ Checks ISMIP7 NetCDF simulation datasets for compliance with the [ISMIP7 data re 1. **Naming** — variable name, region field, ISM member id (`mNNN`), ESM name (CMIP6/CMIP7 registry), forcing member id (`fNNN`), set counter (`[C|E|P]NNN`), and year range (`YYYY-YYYY` matching the actual time axis). 2. **Numerical** — units match the data request; all values lie within the allowed min/max range for the relevant region; array is not entirely fill values. 3. **Spatial** *(xyt variables only)* — grid corners lie within the expected AIS or GrIS extents; resolution is one of the allowed values; x and y cell size are equal. -4. **Time** — time dimension is present, unlimited, and monotonically increasing; annual cadence; experiment end date and duration match `experiments_ismip7.csv`. +4. **Time** — time dimension is present, unlimited, and monotonically increasing; annual cadence; experiment start/end dates and duration match `experiments_ismip7.csv`, accounting for the variable type (ST or FL, see [Time encoding](#time-encoding)). 5. **Attributes** — required global and coordinate attributes are present and have correct values; `standard_name` matches data request; `_FillValue` equals the NetCDF4 default for the variable's dtype; variable and time are float32; `scale_factor` and `add_offset` are not allowed. -Compliance criteria are defined in `conventions/ISMIP7_variable_request.xlsx` (variable metadata) and `experiments_ismip7.csv` (valid experiment date ranges). +Compliance criteria are defined in `conventions/ISMIP7_variable_request.xlsx` (variable metadata) and `experiments_ismip7.csv` (valid experiment year ranges and durations). + +--- + +## Time encoding + +ISMIP7 uses the standard (Gregorian) CF calendar with time recorded as **days since 1850-01-01 00:00:00**. The encoding convention differs by variable type: + +| Variable type | Time coordinate | Time bounds | +|---|---|---| +| **State (ST)** — snapshots | Jan 1 of year N+1 (= end of year N) | none | +| **Flux (FL)** — annual averages | Jul 1 of year N (mid-year) | Jan 1 of year N → Jan 1 of year N+1 | + +For example, a 286-year `ctrl` simulation covering nominal years 2015–2300: +- ST files carry timestamps `2016-01-01` … `2301-01-01` +- FL files carry timestamps `2015-07-01` … `2300-07-01`, with bounds `[2015-01-01, 2016-01-01]` … `[2300-01-01, 2301-01-01]` + +The filename year range (`YYYY-YYYY`) always refers to the **nominal simulation years** (2015–2300 in the example above), regardless of variable type. The checker accounts for this when validating the filename against the time axis. + +Reference lookup tables are available in the companion repository [`ismip7-time-encoding`](https://github.com/ismip/ismip7-time-encoding). --- @@ -43,6 +62,8 @@ python compliance_checker.py --source-path ./Models/GrIS/ISMIP7/SYNTH1/CORE --va | `--source-path` | `./Models/GrIS/ISMIP7/SYNTH1/CORE` | Directory containing `.nc` files to check | | `--variable-list` | `ismip7_scalars` | `ismip7_xyt`, `ismip7_scalars`, or `ismip7` (both) | +`experiments_ismip7.csv` defines the allowed nominal year ranges and durations for each experiment. The checker derives the expected FL and ST timestamps from these year values at runtime (see [Time encoding](#time-encoding)). + --- ## Generating synthetic test files diff --git a/compliance_checker.py b/compliance_checker.py index 1eef924..9877508 100755 --- a/compliance_checker.py +++ b/compliance_checker.py @@ -254,17 +254,27 @@ def _load_experiments_csv(file_path: str): for _, row in frame.iterrows(): experiments.append( { - "experiment": row["experiment"], - "startinf": datetime.datetime.strptime(row["startinf"], "%Y-%m-%d"), - "startsup": datetime.datetime.strptime(row["startsup"], "%Y-%m-%d"), - "endinf": datetime.datetime.strptime(row["endinf"], "%Y-%m-%d"), - "endsup": datetime.datetime.strptime(row["endsup"], "%Y-%m-%d"), - "duration": int(row["duration"]), + "experiment": row["experiment"], + "start_year_min": int(row["start_year_min"]), + "start_year_max": int(row["start_year_max"]), + "end_year": int(row["end_year"]), + "duration": int(row["duration"]), } ) return experiments +def _nominal_to_timestamp(year: int, var_type: str) -> datetime.datetime: + """Return the expected encoded timestamp for a nominal simulation year. + + ST variables: Jan 1 of the following year (end-of-year snapshot). + FL variables: Jul 1 of the same year (mid-year average). + """ + if var_type == "ST": + return datetime.datetime(year + 1, 1, 1) + return datetime.datetime(year, 7, 1) + + def _run_compliance_checker( source_path: str, commit_num: str, @@ -667,6 +677,7 @@ def _check_naming( dim: set, isscalar: bool, report_naming_issues: list, + var_type: str = "", ) -> int: errors = 0 @@ -745,8 +756,10 @@ def _check_naming( errors += 1 elif "time" in ds.coords: _decoded_time = xr.decode_cf(ds, decode_times=xr.coders.CFDatetimeCoder(use_cftime=True))["time"] - actual_start = min(_decoded_time).item().year - actual_end = max(_decoded_time).item().year + # ST timestamps are Jan 1 of year+1; subtract 1 to recover the nominal simulation year. + _yr_offset = 1 if var_type == "ST" else 0 + actual_start = min(_decoded_time).item().year - _yr_offset + actual_end = max(_decoded_time).item().year - _yr_offset if fn_start_year != actual_start: log_file.write( f" - ERROR: filename start year {fn_start_year} does not match first time step year {actual_start}.\n" @@ -896,6 +909,7 @@ def _check_time( dim: set, experiments: list, experiment_name: str, + var_type: str = "", ) -> int: errors = 0 @@ -977,6 +991,13 @@ def _check_time( end_exp.item().year, end_exp.item().month, end_exp.item().day ) + # Compute expected timestamp windows from nominal years + var_type (±1 day tolerance). + _tol = datetime.timedelta(days=1) + startinf = _nominal_to_timestamp(exp["start_year_min"], var_type) - _tol + startsup = _nominal_to_timestamp(exp["start_year_max"], var_type) + _tol + endinf = _nominal_to_timestamp(exp["end_year"], var_type) - _tol + endsup = _nominal_to_timestamp(exp["end_year"], var_type) + _tol + if exp["duration"] == -1: # Undetermined length: only require >= 1 year, start and end date in range if duration_years >= 1: @@ -994,7 +1015,7 @@ def _check_time( dateformat_start_exp = datetime.datetime( start_exp.item().year, start_exp.item().month, start_exp.item().day ) - if exp["startinf"] <= dateformat_start_exp <= exp["startsup"]: + if startinf <= dateformat_start_exp <= startsup: log_file.write( " - Experiment starts correctly on " + start_exp.item().strftime("%Y-%m-%d") @@ -1005,14 +1026,14 @@ def _check_time( " - ERROR: the experiment starts the " + start_exp.item().strftime("%Y-%m-%d") + ". The date should be comprised between " - + exp["startinf"].strftime("%Y-%m-%d") + + startinf.strftime("%Y-%m-%d") + " and " - + exp["startsup"].strftime("%Y-%m-%d") + + startsup.strftime("%Y-%m-%d") + "\n" ) errors += 1 - if exp["endinf"] <= dateformat_end_exp <= exp["endsup"]: + if endinf <= dateformat_end_exp <= endsup: log_file.write( " - Experiment ends correctly on " + end_exp.item().strftime("%Y-%m-%d") @@ -1023,9 +1044,9 @@ def _check_time( " - ERROR: the experiment ends on " + end_exp.item().strftime("%Y-%m-%d") + ". The date should be comprised between " - + exp["endinf"].strftime("%Y-%m-%d") + + endinf.strftime("%Y-%m-%d") + " and " - + exp["endsup"].strftime("%Y-%m-%d") + + endsup.strftime("%Y-%m-%d") + "\n" ) errors += 1 @@ -1037,7 +1058,7 @@ def _check_time( start_exp.item().month, start_exp.item().day, ) - if exp["startinf"] <= dateformat_start_exp <= exp["startsup"]: + if startinf <= dateformat_start_exp <= startsup: log_file.write( " - Experiment starts correctly on " + start_exp.item().strftime("%Y-%m-%d") @@ -1048,14 +1069,14 @@ def _check_time( " - ERROR: the experiment starts the " + start_exp.item().strftime("%Y-%m-%d") + ". The date should be comprised between " - + exp["startinf"].strftime("%Y-%m-%d") + + startinf.strftime("%Y-%m-%d") + " and " - + exp["startsup"].strftime("%Y-%m-%d") + + startsup.strftime("%Y-%m-%d") + "\n" ) errors += 1 - if exp["endinf"] <= dateformat_end_exp <= exp["endsup"]: + if endinf <= dateformat_end_exp <= endsup: log_file.write( " - Experiment ends correctly on " + end_exp.item().strftime("%Y-%m-%d") @@ -1066,9 +1087,9 @@ def _check_time( " - ERROR: the experiment ends on " + end_exp.item().strftime("%Y-%m-%d") + ". The date should be comprised between " - + exp["endinf"].strftime("%Y-%m-%d") + + endinf.strftime("%Y-%m-%d") + " and " - + exp["endsup"].strftime("%Y-%m-%d") + + endsup.strftime("%Y-%m-%d") + "\n" ) errors += 1 @@ -1295,8 +1316,9 @@ def _run_variable_checks( index = ismip_var.index(considered_variable) isscalar = ismip_meta[index]["dim"] == "t" + var_type = ismip_meta[index].get("type", "") - n_err = _check_naming(log_file, ds, file_name, region, dim, isscalar, report_naming_issues) + n_err = _check_naming(log_file, ds, file_name, region, dim, isscalar, report_naming_issues, var_type) var_naming_errors += n_err if n_err > 0: return var_naming_errors, var_num_errors, var_spatial_errors, var_time_errors, var_attr_errors @@ -1315,7 +1337,7 @@ def _run_variable_checks( if not isscalar: var_spatial_errors += _check_spatial(log_file, ds, grid_extent, possible_resolution) - var_time_errors += _check_time(log_file, ds, dim, experiments, experiment_name) + var_time_errors += _check_time(log_file, ds, dim, experiments, experiment_name, ismip_meta[var_index].get("type", "")) var_attr_errors += _check_attributes(log_file, ds, ivar, ismip_meta, var_index, isscalar, ismip_meta[var_index].get("type", ""), region) diff --git a/experiments_ismip7.csv b/experiments_ismip7.csv index 8b55547..ecaa1d6 100644 --- a/experiments_ismip7.csv +++ b/experiments_ismip7.csv @@ -1,6 +1,6 @@ -experiment;startinf;startsup;endinf;endsup;duration -historical;1850-01-01;2013-12-31;2014-06-30;2015-01-01;-1 -ssp370;2015-01-01;2016-01-02;2100-06-30;2101-01-01;86 -ssp126;2015-01-01;2016-01-02;2300-06-30;2301-01-01;286 -ssp585;2015-01-01;2016-01-02;2300-06-30;2301-01-01;286 -ctrl;2015-01-01;2016-01-02;2300-06-30;2301-01-01;286 +experiment;start_year_min;start_year_max;end_year;duration +historical;1850;2014;2014;-1 +ssp370;2015;2015;2100;86 +ssp126;2015;2015;2300;286 +ssp585;2015;2015;2300;286 +ctrl;2015;2015;2300;286 diff --git a/generate/generate_test_files.py b/generate/generate_test_files.py index 897c8c8..348a827 100644 --- a/generate/generate_test_files.py +++ b/generate/generate_test_files.py @@ -319,12 +319,12 @@ def create_netcdf_file(output_file, grid_name='GrIS_16000m', scenario='ctrl', st # Select appropriate time coordinate (days since 1850, calendar: standard) origin = datetime(1850, 1, 1).date() if var_type == 'ST': - # End of year (Dec 31) + # End of year: Jan 1 of next year (ISMIP7 convention) time_days = [] time_bounds = None for i in range(nyears): year = start_year + i - dt = datetime(year, 12, 31).date() + dt = datetime(year + 1, 1, 1).date() time_days.append(float((dt - origin).days)) time_coord = np.array(time_days, dtype=np.float32) elif var_type == 'FL': @@ -341,12 +341,12 @@ def create_netcdf_file(output_file, grid_name='GrIS_16000m', scenario='ctrl', st time_bounds[i, 1] = float((t1 - origin).days) time_coord = np.array(time_days, dtype=np.float32) else: - # Default to state variable timing (end of year) + # Default to state variable timing (Jan 1 of next year) time_days = [] time_bounds = None for i in range(nyears): year = start_year + i - dt = datetime(year, 12, 31).date() + dt = datetime(year + 1, 1, 1).date() time_days.append(float((dt - origin).days)) time_coord = np.array(time_days, dtype=np.float32) @@ -459,11 +459,12 @@ def create_netcdf_file(output_file, grid_name='GrIS_16000m', scenario='ctrl', st # Select appropriate time coordinate (days since 1850) origin = datetime(1850, 1, 1).date() if var_type == 'ST': + # End of year: Jan 1 of next year (ISMIP7 convention) time_days = [] time_bounds = None for i in range(nyears): year = start_year + i - dt = datetime(year, 12, 31).date() + dt = datetime(year + 1, 1, 1).date() time_days.append(float((dt - origin).days)) time_coord = np.array(time_days, dtype=np.float32) elif var_type == 'FL': @@ -479,11 +480,12 @@ def create_netcdf_file(output_file, grid_name='GrIS_16000m', scenario='ctrl', st time_bounds[i, 1] = float((t1 - origin).days) time_coord = np.array(time_days, dtype=np.float32) else: + # Default to state variable timing (Jan 1 of next year) time_days = [] time_bounds = None for i in range(nyears): year = start_year + i - dt = datetime(year, 12, 31).date() + dt = datetime(year + 1, 1, 1).date() time_days.append(float((dt - origin).days)) time_coord = np.array(time_days, dtype=np.float32) diff --git a/tests/test_compliance_checker.py b/tests/test_compliance_checker.py index 399c2ba..2609866 100644 --- a/tests/test_compliance_checker.py +++ b/tests/test_compliance_checker.py @@ -154,8 +154,8 @@ def test_checker_reports_historical_time_range_violation(case_dir): set_time_values( target_file, [ - datetime(2015, 12, 31), - datetime(2016, 12, 31), + datetime(2016, 1, 1), + datetime(2017, 1, 1), ], ) rename_file_part(