Fix ST time encoding to use Jan 1 of year+1 per ISMIP7 convention

README.md

@@ -5,10 +5,29 @@ Checks ISMIP7 NetCDF simulation datasets for compliance with the [ISMIP7 data re
 1. **Naming** — variable name, region field, ISM member id (`mNNN`), ESM name (CMIP6/CMIP7 registry), forcing member id (`fNNN`), set counter (`[C|E|P]NNN`), and year range (`YYYY-YYYY` matching the actual time axis).
 2. **Numerical** — units match the data request; all values lie within the allowed min/max range for the relevant region; array is not entirely fill values.
 3. **Spatial** *(xyt variables only)* — grid corners lie within the expected AIS or GrIS extents; resolution is one of the allowed values; x and y cell size are equal.
-4. **Time** — time dimension is present, unlimited, and monotonically increasing; annual cadence; experiment end date and duration match `experiments_ismip7.csv`.
+4. **Time** — time dimension is present, unlimited, and monotonically increasing; annual cadence; experiment start/end dates and duration match `experiments_ismip7.csv`, accounting for the variable type (ST or FL, see [Time encoding](#time-encoding)).
 5. **Attributes** — required global and coordinate attributes are present and have correct values; `standard_name` matches data request; `_FillValue` equals the NetCDF4 default for the variable's dtype; variable and time are float32; `scale_factor` and `add_offset` are not allowed.
 
-Compliance criteria are defined in `conventions/ISMIP7_variable_request.xlsx` (variable metadata) and `experiments_ismip7.csv` (valid experiment date ranges).
+Compliance criteria are defined in `conventions/ISMIP7_variable_request.xlsx` (variable metadata) and `experiments_ismip7.csv` (valid experiment year ranges and durations).
+
+---
+
+## Time encoding
+
+ISMIP7 uses the standard (Gregorian) CF calendar with time recorded as **days since 1850-01-01 00:00:00**. The encoding convention differs by variable type:
+
+| Variable type | Time coordinate | Time bounds |
+|---|---|---|
+| **State (ST)** — snapshots | Jan 1 of year N+1 (= end of year N) | none |
+| **Flux (FL)** — annual averages | Jul 1 of year N (mid-year) | Jan 1 of year N → Jan 1 of year N+1 |
+
+For example, a 286-year `ctrl` simulation covering nominal years 2015–2300:
+- ST files carry timestamps `2016-01-01` … `2301-01-01`
+- FL files carry timestamps `2015-07-01` … `2300-07-01`, with bounds `[2015-01-01, 2016-01-01]` … `[2300-01-01, 2301-01-01]`
+
+The filename year range (`YYYY-YYYY`) always refers to the **nominal simulation years** (2015–2300 in the example above), regardless of variable type. The checker accounts for this when validating the filename against the time axis.
+
+Reference lookup tables are available in the companion repository [`ismip7-time-encoding`](https://github.com/ismip/ismip7-time-encoding).
 
 ---
 
@@ -43,6 +62,8 @@ python compliance_checker.py --source-path ./Models/GrIS/ISMIP7/SYNTH1/CORE --va
 | `--source-path` | `./Models/GrIS/ISMIP7/SYNTH1/CORE` | Directory containing `.nc` files to check |
 | `--variable-list` | `ismip7_scalars` | `ismip7_xyt`, `ismip7_scalars`, or `ismip7` (both) |
 
+`experiments_ismip7.csv` defines the allowed nominal year ranges and durations for each experiment. The checker derives the expected FL and ST timestamps from these year values at runtime (see [Time encoding](#time-encoding)).
+
 ---
 
 ## Generating synthetic test files

compliance_checker.py

@@ -254,17 +254,27 @@ def _load_experiments_csv(file_path: str):
     for _, row in frame.iterrows():
         experiments.append(
             {
-                "experiment": row["experiment"],
-                "startinf": datetime.datetime.strptime(row["startinf"], "%Y-%m-%d"),
-                "startsup": datetime.datetime.strptime(row["startsup"], "%Y-%m-%d"),
-                "endinf": datetime.datetime.strptime(row["endinf"], "%Y-%m-%d"),
-                "endsup": datetime.datetime.strptime(row["endsup"], "%Y-%m-%d"),
-                "duration": int(row["duration"]),
+                "experiment":    row["experiment"],
+                "start_year_min": int(row["start_year_min"]),
+                "start_year_max": int(row["start_year_max"]),
+                "end_year":      int(row["end_year"]),
+                "duration":      int(row["duration"]),
             }
         )
     return experiments
 
 
+def _nominal_to_timestamp(year: int, var_type: str) -> datetime.datetime:
+    """Return the expected encoded timestamp for a nominal simulation year.
+
+    ST variables: Jan 1 of the following year (end-of-year snapshot).
+    FL variables: Jul 1 of the same year (mid-year average).
+    """
+    if var_type == "ST":
+        return datetime.datetime(year + 1, 1, 1)
+    return datetime.datetime(year, 7, 1)
+
+
 def _run_compliance_checker(
     source_path: str,
     commit_num: str,
@@ -667,6 +677,7 @@ def _check_naming(
     dim: set,
     isscalar: bool,
     report_naming_issues: list,
+    var_type: str = "",
 ) -> int:
     errors = 0
 
@@ -745,8 +756,10 @@ def _check_naming(
                 errors += 1
             elif "time" in ds.coords:
                 _decoded_time = xr.decode_cf(ds, decode_times=xr.coders.CFDatetimeCoder(use_cftime=True))["time"]
-                actual_start = min(_decoded_time).item().year
-                actual_end = max(_decoded_time).item().year
+                # ST timestamps are Jan 1 of year+1; subtract 1 to recover the nominal simulation year.
+                _yr_offset = 1 if var_type == "ST" else 0
+                actual_start = min(_decoded_time).item().year - _yr_offset
+                actual_end = max(_decoded_time).item().year - _yr_offset
                 if fn_start_year != actual_start:
                     log_file.write(
                         f" - ERROR: filename start year {fn_start_year} does not match first time step year {actual_start}.\n"
@@ -896,6 +909,7 @@ def _check_time(
     dim: set,
     experiments: list,
     experiment_name: str,
+    var_type: str = "",
 ) -> int:
     errors = 0
 
@@ -977,6 +991,13 @@ def _check_time(
         end_exp.item().year, end_exp.item().month, end_exp.item().day
     )
 
+    # Compute expected timestamp windows from nominal years + var_type (±1 day tolerance).
+    _tol = datetime.timedelta(days=1)
+    startinf = _nominal_to_timestamp(exp["start_year_min"], var_type) - _tol
+    startsup  = _nominal_to_timestamp(exp["start_year_max"], var_type) + _tol
+    endinf    = _nominal_to_timestamp(exp["end_year"],       var_type) - _tol
+    endsup    = _nominal_to_timestamp(exp["end_year"],       var_type) + _tol
+
     if exp["duration"] == -1:
         # Undetermined length: only require >= 1 year, start and end date in range
         if duration_years >= 1:
@@ -994,7 +1015,7 @@ def _check_time(
         dateformat_start_exp = datetime.datetime(
             start_exp.item().year, start_exp.item().month, start_exp.item().day
         )
-        if exp["startinf"] <= dateformat_start_exp <= exp["startsup"]:
+        if startinf <= dateformat_start_exp <= startsup:
             log_file.write(
                 " - Experiment starts correctly on "
                 + start_exp.item().strftime("%Y-%m-%d")
@@ -1005,14 +1026,14 @@ def _check_time(
                 " - ERROR: the experiment starts the "
                 + start_exp.item().strftime("%Y-%m-%d")
                 + ". The date should be comprised between "
-                + exp["startinf"].strftime("%Y-%m-%d")
+                + startinf.strftime("%Y-%m-%d")
                 + " and "
-                + exp["startsup"].strftime("%Y-%m-%d")
+                + startsup.strftime("%Y-%m-%d")
                 + "\n"
             )
             errors += 1
 
-        if exp["endinf"] <= dateformat_end_exp <= exp["endsup"]:
+        if endinf <= dateformat_end_exp <= endsup:
             log_file.write(
                 " - Experiment ends correctly on "
                 + end_exp.item().strftime("%Y-%m-%d")
@@ -1023,9 +1044,9 @@ def _check_time(
                 " - ERROR: the experiment ends on "
                 + end_exp.item().strftime("%Y-%m-%d")
                 + ". The date should be comprised between "
-                + exp["endinf"].strftime("%Y-%m-%d")
+                + endinf.strftime("%Y-%m-%d")
                 + " and "
-                + exp["endsup"].strftime("%Y-%m-%d")
+                + endsup.strftime("%Y-%m-%d")
                 + "\n"
             )
             errors += 1
@@ -1037,7 +1058,7 @@ def _check_time(
                 start_exp.item().month,
                 start_exp.item().day,
             )
-            if exp["startinf"] <= dateformat_start_exp <= exp["startsup"]:
+            if startinf <= dateformat_start_exp <= startsup:
                 log_file.write(
                     " - Experiment starts correctly on "
                     + start_exp.item().strftime("%Y-%m-%d")
@@ -1048,14 +1069,14 @@ def _check_time(
                     " - ERROR: the experiment starts the "
                     + start_exp.item().strftime("%Y-%m-%d")
                     + ". The date should be comprised between "
-                    + exp["startinf"].strftime("%Y-%m-%d")
+                    + startinf.strftime("%Y-%m-%d")
                     + " and "
-                    + exp["startsup"].strftime("%Y-%m-%d")
+                    + startsup.strftime("%Y-%m-%d")
                     + "\n"
                 )
                 errors += 1
 
-            if exp["endinf"] <= dateformat_end_exp <= exp["endsup"]:
+            if endinf <= dateformat_end_exp <= endsup:
                 log_file.write(
                     " - Experiment ends correctly on "
                     + end_exp.item().strftime("%Y-%m-%d")
@@ -1066,9 +1087,9 @@ def _check_time(
                     " - ERROR: the experiment ends on "
                     + end_exp.item().strftime("%Y-%m-%d")
                     + ". The date should be comprised between "
-                    + exp["endinf"].strftime("%Y-%m-%d")
+                    + endinf.strftime("%Y-%m-%d")
                     + " and "
-                    + exp["endsup"].strftime("%Y-%m-%d")
+                    + endsup.strftime("%Y-%m-%d")
                     + "\n"
                 )
                 errors += 1
@@ -1295,8 +1316,9 @@ def _run_variable_checks(
 
     index = ismip_var.index(considered_variable)
     isscalar = ismip_meta[index]["dim"] == "t"
+    var_type = ismip_meta[index].get("type", "")
 
-    n_err = _check_naming(log_file, ds, file_name, region, dim, isscalar, report_naming_issues)
+    n_err = _check_naming(log_file, ds, file_name, region, dim, isscalar, report_naming_issues, var_type)
     var_naming_errors += n_err
     if n_err > 0:
         return var_naming_errors, var_num_errors, var_spatial_errors, var_time_errors, var_attr_errors
@@ -1315,7 +1337,7 @@ def _run_variable_checks(
             if not isscalar:
                 var_spatial_errors += _check_spatial(log_file, ds, grid_extent, possible_resolution)
 
-            var_time_errors += _check_time(log_file, ds, dim, experiments, experiment_name)
+            var_time_errors += _check_time(log_file, ds, dim, experiments, experiment_name, ismip_meta[var_index].get("type", ""))
 
             var_attr_errors += _check_attributes(log_file, ds, ivar, ismip_meta, var_index, isscalar, ismip_meta[var_index].get("type", ""), region)
 

experiments_ismip7.csv

@@ -1,6 +1,6 @@
-experiment;startinf;startsup;endinf;endsup;duration
-historical;1850-01-01;2013-12-31;2014-06-30;2015-01-01;-1
-ssp370;2015-01-01;2016-01-02;2100-06-30;2101-01-01;86
-ssp126;2015-01-01;2016-01-02;2300-06-30;2301-01-01;286
-ssp585;2015-01-01;2016-01-02;2300-06-30;2301-01-01;286
-ctrl;2015-01-01;2016-01-02;2300-06-30;2301-01-01;286
+experiment;start_year_min;start_year_max;end_year;duration
+historical;1850;2014;2014;-1
+ssp370;2015;2015;2100;86
+ssp126;2015;2015;2300;286
+ssp585;2015;2015;2300;286
+ctrl;2015;2015;2300;286

generate/generate_test_files.py

@@ -319,12 +319,12 @@ def create_netcdf_file(output_file, grid_name='GrIS_16000m', scenario='ctrl', st
             # Select appropriate time coordinate (days since 1850, calendar: standard)
             origin = datetime(1850, 1, 1).date()
             if var_type == 'ST':
-                # End of year (Dec 31)
+                # End of year: Jan 1 of next year (ISMIP7 convention)
                 time_days = []
                 time_bounds = None
                 for i in range(nyears):
                     year = start_year + i
-                    dt = datetime(year, 12, 31).date()
+                    dt = datetime(year + 1, 1, 1).date()
                     time_days.append(float((dt - origin).days))
                 time_coord = np.array(time_days, dtype=np.float32)
             elif var_type == 'FL':
@@ -341,12 +341,12 @@ def create_netcdf_file(output_file, grid_name='GrIS_16000m', scenario='ctrl', st
                     time_bounds[i, 1] = float((t1 - origin).days)
                 time_coord = np.array(time_days, dtype=np.float32)
             else:
-                # Default to state variable timing (end of year)
+                # Default to state variable timing (Jan 1 of next year)
                 time_days = []
                 time_bounds = None
                 for i in range(nyears):
                     year = start_year + i
-                    dt = datetime(year, 12, 31).date()
+                    dt = datetime(year + 1, 1, 1).date()
                     time_days.append(float((dt - origin).days))
                 time_coord = np.array(time_days, dtype=np.float32)
 
@@ -459,11 +459,12 @@ def create_netcdf_file(output_file, grid_name='GrIS_16000m', scenario='ctrl', st
             # Select appropriate time coordinate (days since 1850)
             origin = datetime(1850, 1, 1).date()
             if var_type == 'ST':
+                # End of year: Jan 1 of next year (ISMIP7 convention)
                 time_days = []
                 time_bounds = None
                 for i in range(nyears):
                     year = start_year + i
-                    dt = datetime(year, 12, 31).date()
+                    dt = datetime(year + 1, 1, 1).date()
                     time_days.append(float((dt - origin).days))
                 time_coord = np.array(time_days, dtype=np.float32)
             elif var_type == 'FL':
@@ -479,11 +480,12 @@ def create_netcdf_file(output_file, grid_name='GrIS_16000m', scenario='ctrl', st
                     time_bounds[i, 1] = float((t1 - origin).days)
                 time_coord = np.array(time_days, dtype=np.float32)
             else:
+                # Default to state variable timing (Jan 1 of next year)
                 time_days = []
                 time_bounds = None
                 for i in range(nyears):
                     year = start_year + i
-                    dt = datetime(year, 12, 31).date()
+                    dt = datetime(year + 1, 1, 1).date()
                     time_days.append(float((dt - origin).days))
                 time_coord = np.array(time_days, dtype=np.float32)
 

tests/test_compliance_checker.py

@@ -154,8 +154,8 @@ def test_checker_reports_historical_time_range_violation(case_dir):
     set_time_values(
         target_file,
         [
-            datetime(2015, 12, 31),
-            datetime(2016, 12, 31),
+            datetime(2016, 1, 1),
+            datetime(2017, 1, 1),
         ],
     )
     rename_file_part(