feat(test-cases): output slots, stage decomposition, and CLI package split

.gitignore

@@ -167,6 +167,9 @@ packages/*/NOTICE
 # User-specific catalog paths (test data)
 *.paths.yaml
 
+# Materialised test-case output slots (local inspection only; never committed)
+**/test-data/**/output/
+
 # Debug log files
 **/main_log_debug.txt
 

changelog/748.feature.md

@@ -0,0 +1,6 @@
+`ref test-cases` runs now materialise their outputs into an inspectable, gitignored output slot
+(`<case>/output/<label>/`) holding the curated native files, the rebuilt committed bundle, and a source stamp,
+so you can see and diff exactly what a run produced.
+Added a `build` verb that rebuilds the committed bundle from an existing slot without re-executing the diagnostic,
+a `mint --from-replay` option that re-authors a baseline from the stored native instead of re-running it,
+and a `--label` option that keeps several runs side by side for comparison.

changelog/748.trivial.md

@@ -0,0 +1,3 @@
+Reorganised the `ref test-cases` CLI into a package split by concern
+(discovery, run, baseline lifecycle, store, and the CI gate),
+with no change to the commands or their behaviour.

docs/background/regression-baselines.md

@@ -61,12 +61,49 @@ flowchart LR
 
 | Verb | Credentials | What it does |
 | --- | --- | --- |
-| `run` | none | Execute the diagnostic, curate outputs, write the committed bundle and seed `manifest.json` (`native = {}`). |
-| `mint` | write | Upload the curated native files to the object store and populate `manifest.native`. Generally run by CI. |
-| `sync` | public read | Fetch the native blobs referenced by the manifest into the local tree. |
-| `replay` | public read | Materialise the native baseline, re-run only `build_execution_result`, and tolerantly compare to the committed bundle. |
+| `run` | none | Execute the diagnostic, curate its native into the output slot `output/<label>/`, rebuild the committed bundle, and — when seeding or `--force-regen` — promote it to `regression/` and update `manifest.json` (first seed uses `native = {}`; existing manifests keep their native block). |
+| `build` | none | Rebuild the committed bundle from an existing output slot (no execution), under the same promotion gate as `run`. Handy for regenerating the bundle after an extraction-code change, from already-materialised native. |
+| `mint` | write | Execute (or, with `--from-replay`, replay the stored native), upload the curated native to the object store, and populate `manifest.native`. Generally run by CI. |
+| `sync` | public read | Fetch the native blobs referenced by the manifest into the local store cache. |
+| `replay` | public read | Materialise the native baseline into a slot, re-run only `build_execution_result`, and tolerantly compare to the committed bundle. |
 | `check-store` | write | Preflight the writable native store (credentials + bucket) without uploading anything. Run this before a slow `mint` to confirm credentials are correct. |
 
+### Output slots
+
+Every `run`, `build`, `replay`, and `mint` materialises what it produced into a gitignored
+*output slot*, so a developer can actually see and diff what a run or replay generated:
+
+```text
+<test case>/output/<label>/
+├── diagnostic.json     # the curated native set, flat at manifest-relative paths
+├── output.json
+├── series.json
+├── ...                 # plus any referenced plots/data/html and native .nc files
+├── regression/         # the committed bundle rebuilt from this slot
+│   ├── diagnostic.json
+│   ├── output.json
+│   └── series.json
+└── .source.json        # {label, verb, source, test_case_version, created}
+```
+
+Slots are never committed — the whole `**/test-data/**/output/` tree is gitignored —
+they exist purely for local inspection.
+
+The `--label` option names a slot (default `latest`, overwritten on every run).
+Named slots persist, so two runs can be compared side by side:
+for example `run --label fresh` then `replay --label fromstore`,
+or `run --label before`, edit the diagnostic, then `run --label after`.
+
+`run` and `build` write the slot on every invocation
+but only *promote* the rebuilt bundle to the tracked `regression/` baseline
+when `--force-regen` is given or no baseline exists yet,
+so a labelled run never silently clobbers a committed baseline;
+`mint` always promotes and uploads.
+When a promoted bundle's underlying native digests differ from the minted baseline,
+`run` and `build` warn (they never block) that a re-mint is needed.
+`mint --from-replay` re-authors the manifest from the stored native instead of re-executing,
+and uploads only blobs whose digest actually changed.
+
 ## CMIP7 test data
 
 CMIP7 data is not yet available on ESGF,

packages/climate-ref-core/src/climate_ref_core/logging.py

@@ -70,6 +70,23 @@ def emit(self, record: logging.LogRecord) -> None:
         logger.opt(depth=depth, exception=record.exc_info).log(level, record.getMessage())
 
 
+def _retention_count_ignore_missing(logs: list[str]) -> None:  # pragma: no cover
+    """Remove old log files, tolerating concurrent cleanup by another process."""
+    keep = 10
+    existing_logs = []
+    for log in logs:
+        path = Path(log)
+        try:
+            mtime = path.stat().st_mtime
+        except FileNotFoundError:
+            continue
+        existing_logs.append((mtime, path))
+
+    for _, path in sorted(existing_logs, key=lambda item: (-item[0], item[1].as_posix()))[keep:]:
+        with contextlib.suppress(FileNotFoundError):
+            path.unlink()
+
+
 def initialise_logging(level: int | str, format: str, log_directory: str | Path) -> None:  # noqa: A002 # pragma: no cover
     """
     Initialise the logging for the REF
@@ -88,7 +105,7 @@ def initialise_logging(level: int | str, format: str, log_directory: str | Path)
     filename = f"climate-ref_{{time:YYYY-MM-DD_HH-mm}}_{process_name}.log"
     logger.add(
         sink=log_directory / filename,
-        retention=10,
+        retention=_retention_count_ignore_missing,
         level="DEBUG",
         format=VERBOSE_LOG_FORMAT,
         colorize=False,

packages/climate-ref-core/src/climate_ref_core/regression/_quantise.py

@@ -1,12 +1,17 @@
 """
 Float quantisation for committed regression bundles.
 
-Committed regression JSON (``series.json`` / ``diagnostic.json`` / ``output.json``)
+The rounded committed JSON (``series.json`` / ``diagnostic.json``)
 records full-precision floats whose least-significant digits are platform-dependent
 (CPU, BLAS, library versions).
 Those last digits churn byte-for-byte between CI and local runs even when the result is numerically identical,
 producing noisy, unreviewable diffs in the committed bundle.
 
+``output.json`` is excluded from rounding: it is the CMEC *output* bundle (metadata only, no
+float leaves by construction) and is serialised by Pydantic's ``model_dump_json(indent=2)`` which
+does not sort keys, so re-dumping it with ``json.dumps(sort_keys=True)`` would reorder its keys and
+churn the committed bytes (see :data:`climate_ref_core.regression.capture._UNROUNDED_COMMITTED_FILES`).
+
 Rounding every float to a fixed number of significant figures at write time gives
 stable, reviewable committed bytes.
 We round to seven significant figures: one digit finer than the regression compare

packages/climate-ref-core/src/climate_ref_core/regression/capture.py

@@ -25,6 +25,8 @@
 from pathlib import Path
 from typing import TYPE_CHECKING
 
+from loguru import logger
+
 from climate_ref_core.output_files import copy_execution_outputs, to_placeholders
 from climate_ref_core.paths import safe_path
 
@@ -41,26 +43,43 @@
     from climate_ref_core.regression.store import NativeStore
 
 
-# Per-file serialisation parameters for the committed bundle,
-# mirroring exactly how each artefact is written natively.
+# Serialisation parameters for committed-bundle files that contain floats.
 _COMMITTED_FLOAT_JSON_KWARGS: dict[str, dict[str, object]] = {
     "series.json": {"indent": 2, "allow_nan": False, "sort_keys": True},
     "diagnostic.json": {"indent": 2, "allow_nan": False, "sort_keys": True},
 }
 
+# Committed files deliberately NOT float-rounded
+# output.json is the CMEC *output* bundle, which by construction carries no float values.
+_UNROUNDED_COMMITTED_FILES: frozenset[str] = frozenset({"output.json"})
+
+
+def _contains_float(obj: object) -> bool:
+    """Return True if ``obj`` (a parsed-JSON structure) has any float leaf (bool excluded)."""
+    if isinstance(obj, bool):
+        return False
+    if isinstance(obj, float):
+        return True
+    if isinstance(obj, dict):
+        return any(_contains_float(v) for v in obj.values())
+    if isinstance(obj, list):
+        return any(_contains_float(v) for v in obj)
+    return False
+
 
 def _round_committed_floats(regression_dir: Path) -> None:
     """
     Round floats in the committed JSON bundle to seven significant figures in place.
 
-    Full-precision floats in ``series.json`` / ``diagnostic.json`` churn byte-for-byte
+    Only ``series.json`` / ``diagnostic.json`` are rounded.
+    Full-precision floats in those files churn byte-for-byte
     between CI and local runs even when numerically identical,
     producing noisy diffs in the committed (git-tracked) bundle.
     Rounding stabilises those bytes;
     seven figures stays an order of magnitude under the regression compare tolerance (``rtol=1e-6``),
     so a gate verdict is never flipped (see :mod:`climate_ref_core.regression._quantise`).
 
-    Each file is re-serialised with the same JSON parameters used to write it natively,
+    Each rounded file is re-serialised with the same JSON parameters used to write it natively,
     so the only byte difference versus the copied file is reduced float precision.
     A file is rewritten only when rounding actually changes its parsed content,
     keeping float-free artefacts byte-identical to the copy.
@@ -82,6 +101,12 @@ def _round_committed_floats(regression_dir: Path) -> None:
             continue
         path.write_text(json.dumps(rounded, **dump_kwargs), encoding="utf-8")  # type: ignore[arg-type]
 
+    # Check that the unrounded files don't contain floats
+    for filename in _UNROUNDED_COMMITTED_FILES:
+        path = regression_dir / filename
+        if path.exists() and _contains_float(json.loads(path.read_text(encoding="utf-8"))):
+            logger.warning(f"{filename} contains float values but is not float-rounded.")
+
 
 def write_committed_bundle(
     source_dir: Path,

packages/climate-ref-core/src/climate_ref_core/regression/manifest.py

@@ -202,6 +202,13 @@ def loads(cls, text: str, *, source: str = "<string>") -> Manifest:
                 "The manifest may be corrupted or written by an incompatible version; "
                 "regenerate it with `ref test-cases run --force-regen`."
             )
+        schema = data["schema"]
+        if isinstance(schema, bool) or not isinstance(schema, int) or schema != SCHEMA_VERSION:
+            raise ValueError(
+                f"Invalid manifest {source}: unsupported schema {schema!r}, "
+                f"expected {SCHEMA_VERSION}. The manifest was written by an incompatible "
+                "version; regenerate it with `ref test-cases run --force-regen`."
+            )
         try:
             native = {
                 relpath: NativeEntry(sha256=entry["sha256"], size=entry["size"])
@@ -220,6 +227,11 @@ def loads(cls, text: str, *, source: str = "<string>") -> Manifest:
             except ValueError as exc:
                 raise ValueError(f"Invalid manifest {source}: {exc}") from exc
             _validate_digest(entry.sha256)
+            if isinstance(entry.size, bool) or not isinstance(entry.size, int) or entry.size < 0:
+                raise ValueError(
+                    f"Invalid manifest {source}: native entry {relpath!r} has invalid size "
+                    f"{entry.size!r}; expected a non-negative integer."
+                )
         return cls(
             schema=data["schema"],
             test_case_version=data["test_case_version"],

packages/climate-ref-core/src/climate_ref_core/testing.py

@@ -31,6 +31,7 @@
 from climate_ref_core.esgf.base import ESGFRequest
 from climate_ref_core.metric_values.typing import SeriesMetricValue
 from climate_ref_core.output_files import from_placeholders, ordered_replacements
+from climate_ref_core.paths import safe_path
 from climate_ref_core.pycmec.metric import CMECMetric
 from climate_ref_core.pycmec.output import CMECOutput
 from climate_ref_core.regression.manifest import Manifest
@@ -201,6 +202,28 @@ def manifest(self) -> Path:
         """Path to manifest.json (the regression bundle manifest, tracked in git)."""
         return self.root / "manifest.json"
 
+    @property
+    def output(self) -> Path:
+        """Path to the output/ directory (gitignored; holds materialised native slots)."""
+        return self.root / "output"
+
+    def output_slot(self, label: str = "latest") -> Path:
+        """
+        Path to a named output slot under ``output/`` (gitignored).
+
+        A slot is a self-contained, inspectable snapshot of one execute/materialise:
+        the curated native set (flat, at manifest-relative paths) plus a ``regression/``
+        subdirectory holding the rebuilt committed bundle and a ``.source.json`` stamp.
+        ``latest`` (the default) is overwritten on every run; named slots persist so two
+        runs can be diffed (e.g. ``--label before`` vs ``--label after``).
+
+        Parameters
+        ----------
+        label
+            Slot name. Must be a single path segment (no separators or ``..``).
+        """
+        return safe_path(label, self.output, label="output slot label", single_segment=True)
+
     @property
     def test_data_dir(self) -> Path:
         """Path to the test-data directory (parent of diagnostic slug dir)."""

packages/climate-ref-core/tests/unit/regression/test_capture.py

@@ -9,12 +9,15 @@
 from climate_ref_core.output_files import PLACEHOLDER_OUTPUT_DIR, PLACEHOLDER_TEST_DATA_DIR
 from climate_ref_core.pycmec.output import CMECOutput
 from climate_ref_core.regression.capture import (
+    _COMMITTED_FLOAT_JSON_KWARGS,
+    _UNROUNDED_COMMITTED_FILES,
+    _contains_float,
     build_native_snapshot,
     capture_execution,
     materialise_native,
     write_committed_bundle,
 )
-from climate_ref_core.regression.manifest import NativeEntry, sha256_file
+from climate_ref_core.regression.manifest import COMMITTED_BUNDLE_FILES, NativeEntry, sha256_file
 from climate_ref_core.regression.store import LocalFilesystemStore
 
 
@@ -106,6 +109,66 @@ def test_write_committed_bundle_rounds_floats(tmp_path):
     assert series[0]["values"] == [1.761334, 9.876543]
 
 
+def test_committed_float_classification_is_exhaustive():
+    assert set(_COMMITTED_FLOAT_JSON_KWARGS) | _UNROUNDED_COMMITTED_FILES == set(COMMITTED_BUNDLE_FILES)
+    assert set(_COMMITTED_FLOAT_JSON_KWARGS).isdisjoint(_UNROUNDED_COMMITTED_FILES)
+
+
+def test_output_json_is_float_free_by_construction(tmp_path):
+    # Codifies the invariant the output.json exclusion relies on: a representative output bundle,
+    # serialised the way REF writes it natively, carries no float leaves.
+    output = CMECOutput.model_validate(CMECOutput.create_template())
+    output.update(
+        "data",
+        short_name="example",
+        dict_content={
+            "filename": "data/example.nc",
+            "long_name": "Example output",
+            "description": "An example data file",
+        },
+    )
+    output.update(
+        "html",
+        short_name="index",
+        dict_content={
+            "filename": "index.html",
+            "long_name": "Index page",
+            "description": "The landing page",
+        },
+    )
+    json_path = tmp_path / "output.json"
+    output.dump_to_json(json_path)
+    loaded = json.loads(json_path.read_text(encoding="utf-8"))
+
+    assert _contains_float(loaded) is False
+
+
+def test_write_committed_bundle_leaves_output_json_bytes_unchanged(tmp_path):
+    # output.json must pass through write_committed_bundle byte-for-byte (no key reorder / rewrite),
+    # while the float-bearing artefacts are rounded.
+    output_dir = (tmp_path / "scratch" / "frag").resolve()
+    test_data_dir = (tmp_path / "test-data").resolve()
+    source = tmp_path / "scratch" / "frag"
+    source.mkdir(parents=True)
+    # output.json written the native (Pydantic) way: indent=2, keys NOT sorted, no floats.
+    CMECOutput.model_validate(CMECOutput.create_template()).dump_to_json(source / "output.json")
+    source_output_bytes = (source / "output.json").read_bytes()
+    # Float-bearing artefacts that rounding must rewrite.
+    (source / "diagnostic.json").write_text(json.dumps({"PROVENANCE": {"score": 1.843240715970751}}))
+    (source / "series.json").write_text(
+        json.dumps([{"dimensions": {"region": "global"}, "values": [1.761333624017425]}])
+    )
+    regression_dir = tmp_path / "regression"
+
+    write_committed_bundle(source, regression_dir, output_dir=output_dir, test_data_dir=test_data_dir)
+
+    # output.json is byte-identical to the copied source: not re-dumped, keys not reordered.
+    assert (regression_dir / "output.json").read_bytes() == source_output_bytes
+    # The float-bearing artefacts were rounded (proving the bundle was processed, not skipped).
+    assert json.loads((regression_dir / "diagnostic.json").read_text())["PROVENANCE"]["score"] == 1.843241
+    assert json.loads((regression_dir / "series.json").read_text())[0]["values"] == [1.761334]
+
+
 def test_build_native_snapshot_digests_relpaths(tmp_path):
     base = tmp_path / "results" / "frag"
     base.mkdir(parents=True)

packages/climate-ref-core/tests/unit/regression/test_manifest.py

@@ -191,6 +191,28 @@ def test_load_rejects_bad_native_digest(self, tmp_path: Path, bad_digest: str) -
         with pytest.raises(ValueError, match="Invalid sha256 digest"):
             Manifest.load(p)
 
+    @pytest.mark.parametrize("bad_schema", [2, 0, "1", True, 1.0])
+    def test_load_rejects_unknown_schema(self, bad_schema: object) -> None:
+        payload = {
+            "schema": bad_schema,
+            "test_case_version": 1,
+            "committed": {},
+            "native": {},
+        }
+        with pytest.raises(ValueError, match="unsupported schema"):
+            Manifest.loads(json.dumps(payload))
+
+    @pytest.mark.parametrize("bad_size", [-1, 3.0, "10", True])
+    def test_load_rejects_invalid_native_size(self, bad_size: object) -> None:
+        payload = {
+            "schema": SCHEMA_VERSION,
+            "test_case_version": 1,
+            "committed": {},
+            "native": {"file.nc": {"sha256": "aa" * 32, "size": bad_size}},
+        }
+        with pytest.raises(ValueError, match="invalid size"):
+            Manifest.loads(json.dumps(payload))
+
 
 class TestComputeCommittedDigests:
     def test_all_three_present(self, regression_dir: Path) -> None: