Migrate ILAMB regression baselines to Framework B (cmip6 batch)

.github/workflows/regression-drift.yaml

@@ -4,8 +4,8 @@
 # catching baselines that no longer reproduce within tolerance (e.g. after a dependency upgrade)
 # even when no pull request touched them.
 #
-# Run on the public runners with no credentials (read-only access to the public repository and public read access to the native blobs),
-# so it is safe on untrusted fork pull-request code.
+# Runs on the self-hosted ARC runners to reuse the persistent dataset and intake-esgf caches
+# (REF_DATASET_CACHE_DIR=/data/cache/datasets, ~/.esgf and the read-only /data/cmip6 archive).
 # A failure here means a stored baseline has drifted and needs investigation (and possibly a re-mint).
 # See docs/background/regression-baselines.md.
 name: Regression baselines (nightly drift)
@@ -28,7 +28,10 @@ jobs:
   drift:
     # Don't run the scheduled check on forks.
     if: github.repository == 'Climate-REF/climate-ref'
-    runs-on: ubuntu-latest
+    runs-on: "arc-climate-ref"
+    env:
+      # Avoid the huge amounts of TQDM output from the download
+      TQDM_DISABLE: "1"
     strategy:
       # A drift in one provider must not cancel the others' checks.
       fail-fast: false

.github/workflows/regression-mint.yaml

@@ -47,11 +47,17 @@ concurrency:
 jobs:
   mint:
     if: github.repository == 'Climate-REF/climate-ref'
-    runs-on: ubuntu-latest
+    # Run on the self-hosted ARC runners.
+    # They mount the persistent dataset, software, and intake-esgf caches.
+    # This requires approval by a maintainer to be run.
+    runs-on: "arc-climate-ref"
     environment: native-baselines
     defaults:
       run:
         shell: bash
+    env:
+      # Keep the cache-warming download steps from flooding the log with progress bars.
+      TQDM_DISABLE: "1"
     steps:
       - name: Refuse to mint on the default branch
         if: github.ref_name == github.event.repository.default_branch
@@ -66,10 +72,27 @@ jobs:
           python-version: "3.13"
       - name: Fetch sample data
         run: uv run ref datasets fetch-sample-data
+      - name: Set up provider (environment + reference data)
+        env:
+          PROVIDER: ${{ inputs.provider }}
+        # Provisions the provider's conda environment (where applicable) and its
+        # reference-data registries. ILAMB/PMP/ESMValTool diagnostics read observational
+        # reference data that is not part of the ESGF model fetch above; without this the
+        # diagnostic fails to open its reference files during the mint run.
+        run: uv run ref providers setup --provider "${PROVIDER}"
       - name: Restore test-case catalog paths
         env:
           PROVIDER: ${{ inputs.provider }}
-        run: uv run ref test-cases fetch --provider "${PROVIDER}" || echo "::warning::test-case fetch incomplete"
+          DIAGNOSTIC: ${{ inputs.diagnostic }}
+          TEST_CASE: ${{ inputs.test_case }}
+        run: |
+          # Scope the fetch to the dispatched diagnostic/test case so a single-case
+          # mint does not re-fetch every case for the provider (cmip7 cases in
+          # particular can stall on ESGF). Mirrors the mint step's argument handling.
+          args=(--provider "${PROVIDER}")
+          if [ -n "${DIAGNOSTIC}" ]; then args+=(--diagnostic "${DIAGNOSTIC}"); fi
+          if [ -n "${TEST_CASE}" ]; then args+=(--test-case "${TEST_CASE}"); fi
+          uv run ref test-cases fetch "${args[@]}" || echo "::warning::test-case fetch incomplete"
       - name: Verify store credentials
         env:
           REF_NATIVE_STORE_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
@@ -99,9 +122,12 @@ jobs:
           git config user.name "github-actions[bot]"
           git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
 
-          # Stage only the committed bundle and manifest; native blobs live in the
-          # object store, not the tree, and nothing else should change during a mint.
+          # Stage the committed bundle, its manifest, and the input catalog.
+          # The catalog.yaml is included because the mint regenerates it:
+          #  datasets.hash (its _metadata.hash, recorded as the manifest's catalog_hash) is not reproducible across pandas versions,
+          #  so the runner's value can differ from the developer-authored one.
           git add ':(glob)packages/**/test-data/**/manifest.json' \
+                  ':(glob)packages/**/test-data/**/catalog.yaml' \
                   ':(glob)packages/**/test-data/**/regression/**'
           if git diff --cached --quiet; then
             echo "No baseline changes to commit; the native blobs were already current."
@@ -111,4 +137,19 @@ jobs:
           # Pushed with GITHUB_TOKEN, so GitHub will NOT start new workflow runs for this
           # commit: the PR gate does not auto-replay the freshly minted baseline. Push a
           # follow-up commit (or re-run the PR gate) to verify it. See docs/background/regression-baselines.md.
-          git push origin "HEAD:${GITHUB_REF_NAME}"
+          #
+          # The branch may have advanced (e.g. mint of another diagnostic).
+          # Different diagnostics touch disjoint test-data paths, so the rebase does not conflict.
+          for attempt in 1 2 3 4 5; do
+            if git push origin "HEAD:${GITHUB_REF_NAME}"; then
+              exit 0
+            fi
+            echo "push rejected (attempt ${attempt}); rebasing onto origin/${GITHUB_REF_NAME}"
+            git fetch origin "${GITHUB_REF_NAME}"
+            git rebase "origin/${GITHUB_REF_NAME}" || {
+              echo "::error::rebase conflict while committing regenerated baselines"
+              exit 1
+            }
+          done
+          echo "::error::failed to push regenerated baselines after retries"
+          exit 1

.github/workflows/regression-pr-gate.yaml

@@ -40,8 +40,6 @@ jobs:
         with:
           python-version: "3.13"
       - name: Decide and verify baselines
-        # The gate fetches sample data only when a case actually needs replaying,
-        # so a PR that touches no baselines completes without any downloads.
         env:
           GITHUB_BASE_REF: ${{ github.base_ref }}
         run: bash scripts/ci/regression-pr-gate.sh

Makefile

@@ -145,6 +145,10 @@ test-diagnostics: test-diagnostic-example test-diagnostic-esmvaltool test-diagno
 .PHONY: test-executors
 test-executors: test-celery
 
+.PHONY: regression-gate
+regression-gate:  ## run the regression baseline coupling gate + replay (compares against origin/main)
+	bash scripts/ci/regression-pr-gate.sh
+
 .PHONY: test
 test: clean test-core test-ref test-executors test-diagnostics test-integration ## run the tests
 	uv run coverage report

changelog/738.feature.md

@@ -0,0 +1 @@
+ILAMB regression baselines are migrated to the per-package Framework B layout, starting with the `mrsos-wangmao`, `gpp-fluxnet2015` and `lai-avh15c1` cmip6 test cases. Each carries a committed CMEC bundle and a manifest; native blobs are published separately through the gated mint workflow. ILAMB executions now also declare their scalar CSV and netCDF outputs in the CMEC output bundle so they are persisted with the results and can be replayed.

changelog/738.fix.md

@@ -0,0 +1 @@
+Regression `series.json` baselines are now written with a stable, dimension-sorted order. Previously a diagnostic could emit its series in a platform-dependent order, causing a committed bundle minted on one platform to falsely fail the regression gate on another even when every value matched within tolerance.

changelog/742.trivial.md

@@ -0,0 +1 @@
+Sped up the per-pull-request regression baseline gate by no longer downloading the input data that replay never reads, and made the gate runnable locally via `make regression-gate`.

docs/background/regression-baselines.md

@@ -182,6 +182,25 @@ branch and acts on each decision:
 The job runs on the public `ubuntu-latest` runner with no secrets, so it is safe on fork pull requests.
 The decision-to-replay fan-out lives in `scripts/ci/regression-pr-gate.sh`.
 
+Neither stage downloads input datasets:
+`ci-gate` reads only manifests and the git diff,
+and `replay` rebuilds the committed bundle from the public native blobs plus the committed `catalog.yaml`/`manifest.json`
+(`build_execution_result` reads only its output directory).
+The gate therefore fetches no sample data or ESGF inputs — only the small native blobs for the cases it actually replays.
+
+#### Running the gate locally
+
+The same script is the local entry point, so you can reproduce a pull request's verdict before pushing:
+
+```bash
+make regression-gate                          # gate + replay against origin/main
+bash scripts/ci/regression-pr-gate.sh main    # or pass any base ref
+```
+
+The base ref defaults to `origin/${GITHUB_BASE_REF:-main}`.
+Under GitHub Actions the script emits log groups and `::error::`/`::warning::` annotations;
+run locally it prints plain output instead.
+
 ### Gated mint (`regression-mint.yaml`)
 
 Minting is the only step that writes to the object store,

docs/how-to-guides/testing-diagnostics.md

@@ -460,6 +460,9 @@ flowchart TD
 
 Bump `test_case_version` whenever you *intend* to change a baseline — it tells reviewers and CI the new output is correct.
 
+!!! tip "Check the gate locally before pushing"
+    Run the same gate your pull request will hit with `make regression-gate` (it compares against `origin/main`).
+
 #### Publishing a native baseline
 
 Native files are written to the object store only by `ref test-case mint`, which needs write credentials.

packages/climate-ref-core/src/climate_ref_core/metric_values/typing.py

@@ -115,9 +115,19 @@ def dump_to_json(cls, path: Path, series: Sequence["SeriesMetricValue"]) -> None
         series
             The series values to dump.
         """
+        # Sort the series by their dimensions before serialising so the order is
+        # deterministic across platforms and runs. Diagnostics may emit series in an
+        # implementation-defined order (e.g. set or dict iteration that differs by
+        # platform), which otherwise produces spurious diffs and breaks the positional
+        # regression comparator. ``dimensions`` uniquely identifies a series; ``index_name``
+        # is a stable tie-breaker.
+        ordered = sorted(
+            series,
+            key=lambda s: (json.dumps(s.dimensions, sort_keys=True), s.index_name),
+        )
         with open(path, "w") as f:
             json.dump(
-                [s.model_dump(mode="json") for s in series],
+                [s.model_dump(mode="json") for s in ordered],
                 f,
                 indent=2,
                 allow_nan=False,

packages/climate-ref-core/tests/unit/metric_values/test_typing.py

@@ -1,3 +1,4 @@
+import json
 import re
 from pathlib import Path
 
@@ -91,6 +92,37 @@ def test_dump_and_load_json(self, tmp_path: Path):
 
         assert loaded_series == series
 
+    def test_dump_orders_series_deterministically(self, tmp_path: Path):
+        # series.json must serialise in a stable, dimension-sorted order so the output
+        # does not depend on the (platform/run-dependent) order a diagnostic emitted its
+        # series in. The regression baseline comparator compares series positionally, so
+        # an unstable order would produce spurious cross-platform mismatches.
+        def make(model: str, region: str) -> SeriesMetricValue:
+            return SeriesMetricValue(
+                dimensions={"region": region, "source_id": model},
+                values=[1.0, 2.0],
+                index=[0, 1],
+                index_name="time",
+            )
+
+        series = [make("CanESM5", "tropical"), make("Reference", "global"), make("CanESM5", "global")]
+        shuffled = [series[1], series[2], series[0]]
+
+        path1 = tmp_path / "order1.json"
+        path2 = tmp_path / "order2.json"
+        SeriesMetricValue.dump_to_json(path1, series)
+        SeriesMetricValue.dump_to_json(path2, shuffled)
+
+        # Byte-identical output regardless of the input ordering.
+        assert path1.read_text() == path2.read_text()
+
+        # All series preserved and emitted in the canonical dimension-sorted order.
+        loaded = SeriesMetricValue.load_from_json(path1)
+        assert len(loaded) == len(series)
+        assert loaded == sorted(
+            loaded, key=lambda s: (json.dumps(s.dimensions, sort_keys=True), s.index_name)
+        )
+
     def test_load_from_json_not_a_list(self, tmp_path: Path):
         path = tmp_path / "test.json"
         path.write_text('{"not": "a list"}')

packages/climate-ref-ilamb/src/climate_ref_ilamb/standard.py

@@ -479,6 +479,26 @@ def _load_csv_and_merge(output_directory: Path) -> pd.DataFrame:
     return df
 
 
+def _register_data_outputs(output_bundle: dict[str, Any], definition: ExecutionDefinition) -> None:
+    """
+    Register ILAMB's scalar CSV and netCDF outputs in the data section of a CMEC output bundle.
+
+    These files are re-read by ``build_execution_result`` (via ``_load_csv_and_merge`` and the
+    netCDF series loop) to reconstruct the metrics and series, so they must be persisted with the
+    execution outputs and captured in the regression baseline rather than left in the scratch
+    directory.
+    """
+    for datafile in sorted(
+        [*definition.output_directory.glob("*.csv"), *definition.output_directory.glob("*.nc")]
+    ):
+        relative_path = str(definition.as_relative_path(datafile))
+        output_bundle[OutputCV.DATA.value][relative_path] = {
+            OutputCV.FILENAME.value: relative_path,
+            OutputCV.LONG_NAME.value: datafile.name,
+            OutputCV.DESCRIPTION.value: "Scalar and time-series data produced by ILAMB.",
+        }
+
+
 class ILAMBStandard(Diagnostic):
     """
     Apply the standard ILAMB analysis with respect to a given reference dataset.
@@ -752,6 +772,11 @@ def build_execution_result(self, definition: ExecutionDefinition) -> ExecutionRe
                 OutputCV.DIMENSIONS.value: figure_dimensions,
             }
 
+        # Register the scalar CSV files and the netCDF time-trace files in the data section so
+        # they are persisted with the execution outputs and captured in the regression baseline;
+        # build_execution_result re-reads them to reconstruct the metrics and series.
+        _register_data_outputs(output_bundle, definition)
+
         # Add the html page to the output
         index_html = definition.to_output_path("index.html")
         if index_html.exists():

packages/climate-ref-ilamb/tests/integration/test_diagnostics.py

@@ -89,6 +89,14 @@ def test_validate_test_case_regression(
         test_data_dir=provider_test_data_dir,
     )
 
+    # TODO: remove this once we have migrated test cases to use the committed bundles.
+    # Under Framework B the native baselines live in the object store, not the repo, so
+    # build_execution_result cannot be replayed from the committed bundle alone. The
+    # online equivalent (`ref test-cases replay`) materialises native blobs from the
+    # store; this offline test skips until that path is shared here.
+    if not any(paths.regression.glob("*.nc")):
+        pytest.skip(f"No committed native baseline for {diagnostic.slug}/{test_case_name}")
+
     definition = validator.load_regression_definition(tmp_path / diagnostic.slug / test_case_name)
     validator.validate(definition)