feat: ingest ESMValTool reference datasets for provenance

changelog/+esmvaltool-reference-datasets.feature.md

@@ -0,0 +1,5 @@
+ESMValTool reference (observational/reanalysis) datasets are now ingested into the database under a dedicated `esmvaltool-reference` dataset type during `ref providers setup`, so the data each diagnostic uses can be recorded for provenance and surfaced in the frontend.
+Because this data is not CMOR/obs4MIPs compliant, its metadata is parsed from the ESMValTool `OBS`/`OBS6`, `native6` and `obs4MIPs` layout conventions rather than from file global attributes.
+ESMValTool diagnostics now declare the reference datasets they use via a `reference_datasets` specification instead of hardcoding them inside recipe construction, giving a single source of truth for provenance.
+Each execution now records the reference datasets the diagnostic compared against, so they appear alongside the model datasets in the execution's dataset list.
+The near-identical obs4MIPs and PMP climatology dataset models were also consolidated onto a shared mixin while remaining distinct dataset types.

packages/climate-ref-core/src/climate_ref_core/diagnostics.py

@@ -5,7 +5,7 @@
 from __future__ import annotations
 
 import pathlib
-from collections.abc import Iterable, Sequence
+from collections.abc import Iterable, Mapping, Sequence
 from typing import TYPE_CHECKING, Any, Protocol, runtime_checkable
 
 from attrs import field, frozen
@@ -414,6 +414,29 @@ def apply_filters(self, data_catalog: pd.DataFrame) -> pd.DataFrame:
         return data_catalog[select]
 
 
+@frozen
+class ReferenceDatasetSelector:
+    """
+    Identifies ingested reference datasets a diagnostic uses, for provenance recording.
+
+    Unlike :class:`DataRequirement`, this does not drive the solver's input selection. It is a
+    provider-agnostic description of reference (observational/reanalysis) datasets that should be
+    recorded against each execution of the diagnostic so their use can be tracked and surfaced,
+    even though the diagnostic references them internally rather than receiving them as inputs.
+    """
+
+    source_type: SourceDatasetType
+    """The dataset type the reference datasets are ingested as (e.g. ``esmvaltool-reference``)."""
+
+    facets: Mapping[str, str]
+    """
+    Metadata facets that must all match (AND) an ingested dataset for it to be linked.
+
+    Keys are columns on the corresponding dataset model (e.g. ``project``, ``source_id``,
+    ``table_id``). All datasets matching the facets are linked, across their variables/versions.
+    """
+
+
 @runtime_checkable
 class AbstractDiagnostic(Protocol):
     """
@@ -636,6 +659,23 @@ def run(self, definition: ExecutionDefinition, *, capture_regression: bool = Fal
         # Build the result from the output bundle
         return self.build_execution_result(definition)
 
+    def reference_dataset_selectors(self) -> Sequence[ReferenceDatasetSelector]:
+        """
+        Return the reference datasets to record as provenance for each execution.
+
+        These are datasets the diagnostic compares against but does not receive as solver inputs
+        (e.g. observational data baked into a recipe). The solver links every ingested dataset that
+        matches a selector to the execution, so the reference data used can be tracked and surfaced.
+
+        The default returns no selectors; diagnostics with reference data override this.
+
+        Returns
+        -------
+        :
+            The reference dataset selectors for this diagnostic.
+        """
+        return ()
+
     def prepare_regression_output(self, definition: ExecutionDefinition) -> None:
         """
         Normalise native output for deterministic regression capture.

packages/climate-ref-core/src/climate_ref_core/source_types.py

@@ -30,6 +30,7 @@ class SourceDatasetType(enum.Enum):
     CMIP7 = "cmip7"
     obs4MIPs = "obs4mips"
     PMPClimatology = "pmp-climatology"
+    ESMValToolReference = "esmvaltool-reference"
 
     @classmethod
     @functools.lru_cache(maxsize=1)

packages/climate-ref-esmvaltool/src/climate_ref_esmvaltool/__init__.py

@@ -5,7 +5,7 @@
 from __future__ import annotations
 
 from pathlib import Path
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Any
 
 import pooch
 from loguru import logger
@@ -65,6 +65,46 @@ def get_data_path(self) -> Path | None:
         """Get the path where ESMValTool data is cached."""
         return resolve_cache_dir("esmvaltool")
 
+    def ingest_data(self, config: Config, db: Any) -> None:
+        """
+        Ingest fetched ESMValTool reference data into the database.
+
+        This records the observational/reanalysis datasets bundled with ESMValTool so
+        their use can be tracked for provenance and surfaced in the frontend. The data is
+        not CMOR compliant, so it is registered under its own dataset type.
+
+        Note: requires the ``climate-ref`` package. When ``climate-ref-esmvaltool`` is used
+        standalone the ingestion is skipped with a warning, mirroring the PMP provider.
+        """
+        try:
+            from climate_ref.datasets import ingest_datasets  # noqa: PLC0415
+            from climate_ref.datasets.esmvaltool_reference import (  # noqa: PLC0415
+                ESMValToolReferenceDatasetAdapter,
+            )
+        except ImportError:
+            logger.info(
+                f"Skipping {self.slug} data ingestion: climate-ref package not installed. "
+                "Run `ref datasets ingest --source-type esmvaltool-reference` manually if needed."
+            )
+            return
+
+        # Reference data is fetched under ``<datasets-registry-cache>/ESMValTool``; the same
+        # location build_cmd() configures as the ESMValCore rootpath.
+        registry = dataset_registry_manager[_DATASETS_REGISTRY_NAME]
+        data_path = registry.abspath / "ESMValTool"  # type: ignore[attr-defined]
+        if not data_path.exists():
+            logger.warning(
+                f"ESMValTool reference data not found at {data_path}. "
+                f"Run `ref providers setup --provider {self.slug}` first."
+            )
+            return
+
+        try:
+            stats = ingest_datasets(ESMValToolReferenceDatasetAdapter(), data_path, db, skip_invalid=True)
+            stats.log_summary("ESMValTool reference ingestion complete:")
+        except ValueError as e:
+            logger.warning(f"No valid ESMValTool reference datasets found: {e}")
+
 
 # Initialise the diagnostics manager.
 provider = ESMValToolProvider(

packages/climate-ref-esmvaltool/src/climate_ref_esmvaltool/diagnostics/base.py

@@ -18,10 +18,12 @@
     CommandLineDiagnostic,
     ExecutionDefinition,
     ExecutionResult,
+    ReferenceDatasetSelector,
 )
 from climate_ref_core.metric_values.typing import MetricValueKind, SeriesMetricValue
 from climate_ref_core.pycmec.metric import CMECMetric, MetricCV
 from climate_ref_core.pycmec.output import CMECOutput, OutputCV
+from climate_ref_esmvaltool.diagnostics.reference import ESMValToolReferenceSpec
 from climate_ref_esmvaltool.recipe import (
     fix_annual_statistics_keep_year,
     load_recipe,
@@ -73,6 +75,16 @@ class ESMValToolDiagnostic(CommandLineDiagnostic):
 
     base_recipe: ClassVar[str]
 
+    reference_datasets: ClassVar[tuple[ESMValToolReferenceSpec, ...]] = ()
+    """
+    Reference (observational/reanalysis) datasets this diagnostic compares against.
+
+    These are the non-CMIP datasets the diagnostic uses; they are declared here as the single
+    source of truth so they can be recorded for provenance and surfaced in the frontend.
+    ``update_recipe`` builds the recipe entries from these specs rather than hardcoding them.
+    Diagnostics whose references have not yet been declared leave this empty.
+    """
+
     reconstruction_inputs = (f"executions/{_STABLE_SESSION_NAME}/{_PROVENANCE_GLOB}",)
     """Raw provenance YAML that :meth:`build_execution_result` re-scans to discover the run's outputs.
 
@@ -83,6 +95,26 @@ class ESMValToolDiagnostic(CommandLineDiagnostic):
     the native sanitiser rewrites them to ``<OUTPUT_DIR>`` and the captured blobs stay portable.
     """
 
+    def reference_dataset_selectors(self) -> tuple[ReferenceDatasetSelector, ...]:
+        """
+        Map the declared :attr:`reference_datasets` to provenance selectors.
+
+        Each spec resolves to the ingested ESMValTool reference datasets matching its project and
+        dataset name (and MIP table where declared), across their variables.
+        """
+        selectors = []
+        for spec in self.reference_datasets:
+            facets = {"project": spec.project, "source_id": spec.dataset}
+            if spec.mip is not None:
+                facets["table_id"] = spec.mip
+            selectors.append(
+                ReferenceDatasetSelector(
+                    source_type=SourceDatasetType.ESMValToolReference,
+                    facets=facets,
+                )
+            )
+        return tuple(selectors)
+
     @staticmethod
     @abstractmethod
     def update_recipe(

packages/climate-ref-esmvaltool/src/climate_ref_esmvaltool/diagnostics/enso.py

@@ -17,9 +17,20 @@
 from climate_ref_core.pycmec.output import CMECOutput
 from climate_ref_core.testing import TestCase, TestDataSpecification
 from climate_ref_esmvaltool.diagnostics.base import ESMValToolDiagnostic, get_cmip_source_type
+from climate_ref_esmvaltool.diagnostics.reference import ESMValToolReferenceSpec
 from climate_ref_esmvaltool.recipe import dataframe_to_recipe
 from climate_ref_esmvaltool.types import MetricBundleArgs, OutputBundleArgs, Recipe
 
+# Reference reanalysis for ENSO characteristics; one observational dataset per run.
+_TROPFLUX = ESMValToolReferenceSpec(
+    project="OBS6",
+    dataset="TROPFLUX",
+    mip="Omon",
+    tier=2,
+    obs_type="reanaly",
+    version="v1",
+)
+
 
 class ENSOBasicClimatology(ESMValToolDiagnostic):
     """
@@ -302,6 +313,8 @@ class ENSOCharacteristics(ESMValToolDiagnostic):
     slug = "enso-characteristics"
     base_recipe = "ref/recipe_enso_characteristics.yml"
 
+    reference_datasets = (_TROPFLUX,)
+
     data_requirements = (
         (
             DataRequirement(
@@ -421,21 +434,11 @@ def update_recipe(
         # TODO: update the observational data requirement once available on ESGF.
         # Observations - use only one per run
         recipe["datasets"].append(
-            # {
-            #     "dataset": "NOAA-ERSSTv5",
-            #     "version": "v5",
-            #     "project": "OBS6",
-            #     "type": "reanaly",
-            #     "tier": 2,
-            # }
-            {
-                "dataset": "TROPFLUX",
-                "version": "v1",
-                "project": "OBS6",
-                "type": "reanaly",
-                "tier": 2,
-                "mip": "Omon",
-            }
+            # Alternative observational dataset:
+            # ESMValToolReferenceSpec(
+            #     project="OBS6", dataset="NOAA-ERSSTv5", tier=2, obs_type="reanaly", version="v5"
+            # )
+            _TROPFLUX.to_recipe_dataset()
         )
 
     @staticmethod

packages/climate-ref-esmvaltool/src/climate_ref_esmvaltool/diagnostics/reference.py

@@ -0,0 +1,69 @@
+"""
+Declarative reference-dataset specifications for ESMValTool diagnostics.
+
+ESMValTool diagnostics compare model output against reference (observational/reanalysis)
+datasets that are not published as CMIP data. Historically each diagnostic hardcoded these
+references as raw dictionaries inside ``update_recipe``. :class:`ESMValToolReferenceSpec`
+makes them a declared, inspectable property of the diagnostic instead: a single source of
+truth that feeds both recipe construction (:meth:`ESMValToolReferenceSpec.to_recipe_dataset`)
+and dataset provenance.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from attrs import frozen
+
+
+@frozen
+class ESMValToolReferenceSpec:
+    """
+    A single reference dataset that an ESMValTool diagnostic depends on.
+
+    The fields mirror the ESMValTool/ESMValCore dataset keys used in a recipe. Optional fields
+    are omitted from the generated recipe entry when unset, matching how the references were
+    previously written by hand.
+    """
+
+    project: str
+    """ESMValCore project, e.g. ``OBS``, ``OBS6``, ``native6`` or ``obs4MIPs``."""
+
+    dataset: str
+    """Reference dataset name, e.g. ``OSI-450-nh``, ``HadCRUT5``, ``ERA5``."""
+
+    mip: str | None = None
+    """MIP table where applicable, e.g. ``OImon``, ``Amon``."""
+
+    tier: int | None = None
+    """ESMValTool data tier (accessibility)."""
+
+    obs_type: str | None = None
+    """ESMValTool observation ``type`` where applicable, e.g. ``reanaly``, ``sat``, ``ground``."""
+
+    version: str | None = None
+    """Dataset version, e.g. ``v3``, ``5.0.1.0-analysis``."""
+
+    supplementary_variables: tuple[dict[str, str], ...] | None = None
+    """Optional supplementary variables (e.g. cell measures) attached to the recipe entry."""
+
+    def to_recipe_dataset(self) -> dict[str, Any]:
+        """
+        Build the ESMValTool recipe dataset entry for this reference.
+
+        Keys are emitted in alphabetical order to match the historical hardcoded dictionaries,
+        so the generated recipe YAML (dumped with ``sort_keys=False``) is byte-for-byte unchanged.
+        """
+        entry: dict[str, Any] = {"dataset": self.dataset}
+        if self.mip is not None:
+            entry["mip"] = self.mip
+        entry["project"] = self.project
+        if self.supplementary_variables is not None:
+            entry["supplementary_variables"] = [dict(sv) for sv in self.supplementary_variables]
+        if self.tier is not None:
+            entry["tier"] = self.tier
+        if self.obs_type is not None:
+            entry["type"] = self.obs_type
+        if self.version is not None:
+            entry["version"] = self.version
+        return entry