From 19489eda11b6133fac094e88500b808789e6d7cb Mon Sep 17 00:00:00 2001 From: Jared Lewis Date: Wed, 1 Jul 2026 19:06:09 +1000 Subject: [PATCH 01/23] feat: add climate_ref.results read layer and executions values CLI Introduce climate_ref.results as the single source of truth for filtering and querying metric-value results, so notebooks, the CLI and (later) the ref-app API share one query layer instead of re-implementing it. - _query.py: MetricValueFilter plus select_scalar_values/select_series_values Select builders, count_values and latest_execution_for_group. Exact vs substring diagnostic/provider matching and a promoted-only default close the slug/name and promotion drift between the CLI and the API. - outliers.py: source-id-aware IQR detection as read-model logic. - frames.py: DataFrame and facet-collection helpers. - values.py: typed ScalarValue/SeriesValue DTOs and collections with to_pandas(), plus the Reader facade. Scalar reads SQL-paginate when outlier detection is off and materialise the full set only when it is on. - cli/executions.py: new "ref executions values " command that reads through Reader, with scalar/series output, dimension filters, optional outlier detection, pagination and table/json formats. --- .../src/climate_ref/cli/executions.py | 208 +++++++++- .../src/climate_ref/results/__init__.py | 66 +++ .../src/climate_ref/results/_query.py | 187 +++++++++ .../src/climate_ref/results/frames.py | 85 ++++ .../src/climate_ref/results/outliers.py | 167 ++++++++ .../src/climate_ref/results/values.py | 386 ++++++++++++++++++ .../tests/unit/cli/test_executions.py | 152 ++++++- .../tests/unit/results/test_results.py | 270 ++++++++++++ 8 files changed, 1519 insertions(+), 2 deletions(-) create mode 100644 packages/climate-ref/src/climate_ref/results/__init__.py create mode 100644 packages/climate-ref/src/climate_ref/results/_query.py create mode 100644 packages/climate-ref/src/climate_ref/results/frames.py create mode 100644 packages/climate-ref/src/climate_ref/results/outliers.py create mode 100644 packages/climate-ref/src/climate_ref/results/values.py create mode 100644 packages/climate-ref/tests/unit/results/test_results.py diff --git a/packages/climate-ref/src/climate_ref/cli/executions.py b/packages/climate-ref/src/climate_ref/cli/executions.py index 252b14ad2..b38b43078 100644 --- a/packages/climate-ref/src/climate_ref/cli/executions.py +++ b/packages/climate-ref/src/climate_ref/cli/executions.py @@ -5,12 +5,13 @@ import pathlib import shutil from dataclasses import dataclass +from enum import StrEnum from typing import Annotated from urllib.parse import quote import typer from loguru import logger -from rich.console import Group +from rich.console import Console, Group from rich.filesize import decimal from rich.markup import escape from rich.panel import Panel @@ -30,6 +31,12 @@ from climate_ref.models.diagnostic import Diagnostic from climate_ref.models.execution import execution_datasets, get_execution_group_and_latest_filtered from climate_ref.models.provider import Provider +from climate_ref.results import ( + MetricValueFilter, + OutlierPolicy, + Reader, + latest_execution_for_group, +) from climate_ref_core.logging import EXECUTION_LOG_FILENAME app = typer.Typer(help=__doc__) @@ -49,6 +56,13 @@ class ListGroupsFilterOptions: """Filter by facet key-value pairs (AND across keys, OR within key)""" +class ValueKind(StrEnum): + """Which kind of metric value the ``values`` command should show.""" + + scalar = "scalar" + series = "series" + + # Default columns for the ``list-groups`` table. _DEFAULT_LIST_GROUPS_COLUMNS = [ "id", @@ -557,6 +571,198 @@ def inspect(ctx: typer.Context, execution_id: int) -> None: console.print(_log_panel(result_directory)) +@app.command() +def values( # noqa: PLR0913 + ctx: typer.Context, + group_id: Annotated[int, typer.Argument(help="Execution group ID to show metric values for.")], + kind: Annotated[ + ValueKind, + typer.Option(help="Which kind of value to show: scalar (default) or series."), + ] = ValueKind.scalar, + execution_id: Annotated[ + int | None, + typer.Option(help="Show values for a specific execution instead of the latest one in the group."), + ] = None, + dimension: Annotated[ + list[str] | None, + typer.Option( + "--dimension", + "-d", + help="Filter by CV dimension key=value (exact match). Repeat for AND across keys; " + "repeat the same key for OR within a key.", + ), + ] = None, + outliers: Annotated[ + bool, + typer.Option( + "--outliers/--no-outliers", + help="Run source-id-aware IQR outlier detection (scalar values only).", + ), + ] = False, + include_unverified: Annotated[ + bool, + typer.Option( + "--include-unverified", + help="Include values flagged as outliers (only meaningful with --outliers).", + ), + ] = False, + limit: Annotated[ + int | None, + typer.Option(help="Maximum number of values to display."), + ] = None, + offset: Annotated[ + int, + typer.Option(help="Number of values to skip before displaying."), + ] = 0, + output_format: Annotated[ + OutputFormat, + typer.Option("--format", help="Output format: table (default) or json."), + ] = OutputFormat.table, +) -> None: + """ + Show computed metric values for an execution group. + + By default this shows the scalar values from the latest execution in the group. + Use --kind series for 1-d series, --execution-id to target a specific execution, + and --dimension key=value to filter by controlled-vocabulary dimensions. + + Values are read through the shared climate_ref.results layer, so the same filtering + is applied here as in the API. + """ + database = ctx.obj.database + console = ctx.obj.console + reader = Reader(database) + + group = database.session.get(ExecutionGroup, group_id) + if group is None: + logger.error(f"Execution group not found: {group_id}") + raise typer.Exit(code=1) + + if execution_id is None: + latest = latest_execution_for_group(database.session, group_id) + if latest is None: + logger.error(f"No executions found for group {group_id}.") + raise typer.Exit(code=1) + execution_id = latest.id + else: + execution = database.session.get(Execution, execution_id) + if execution is None or execution.execution_group_id != group_id: + logger.error(f"Execution {execution_id} does not belong to group {group_id}.") + raise typer.Exit(code=1) + + try: + dimensions = parse_facet_filters(dimension) + except ValueError as e: + logger.error(str(e)) + raise typer.Exit(code=1) + + # Scope to the concrete execution: promotion/retraction gating would otherwise hide the + # values the user explicitly asked to inspect. + filters = MetricValueFilter( + execution_ids=[execution_id], + dimensions=dimensions or None, + promoted_only=False, + include_retracted=True, + ) + + try: + if kind == ValueKind.scalar: + _render_scalar_values( + reader, + filters, + console=console, + output_format=output_format, + outliers=outliers, + include_unverified=include_unverified, + offset=offset, + limit=limit, + ) + else: + _render_series_values( + reader, + filters, + console=console, + output_format=output_format, + offset=offset, + limit=limit, + ) + except KeyError as e: + # Raised by the filter when a --dimension key is not a registered CV dimension. + logger.error(f"Unknown dimension: {e}") + raise typer.Exit(code=1) + + +def _render_scalar_values( # noqa: PLR0913 + reader: Reader, + filters: MetricValueFilter, + *, + console: Console, + output_format: OutputFormat, + outliers: bool, + include_unverified: bool, + offset: int, + limit: int | None, +) -> None: + collection = reader.scalar_values( + filters, + outliers=OutlierPolicy() if outliers else None, + include_unverified=include_unverified, + offset=offset, + limit=limit, + with_facets=False, + ) + if not len(collection): + console.print("No scalar values found.") + return + + render_dataframe(collection.to_pandas(), console=console, output_format=output_format) + + if collection.had_outliers: + state = "shown" if include_unverified else "hidden" + logger.warning( + f"{collection.outlier_count} value(s) flagged as outliers ({state}). " + f"Use --include-unverified to show them." + ) + displayed = offset + len(collection) + if displayed < collection.total_count: + logger.warning( + f"Displaying {len(collection)} of {collection.total_count} values. " + f"Use --limit / --offset to see more." + ) + + +def _render_series_values( # noqa: PLR0913 + reader: Reader, + filters: MetricValueFilter, + *, + console: Console, + output_format: OutputFormat, + offset: int, + limit: int | None, +) -> None: + collection = reader.series_values(filters, offset=offset, limit=limit, with_facets=False) + if not len(collection): + console.print("No series values found.") + return + + if output_format == OutputFormat.json: + # Full long-form data for scripting. + render_dataframe(collection.to_pandas(explode=True), output_format=output_format) + else: + # Compact one-row-per-series summary for the terminal; the raw arrays are too wide. + df = collection.to_pandas(explode=False) + df["n_points"] = df["values"].apply(len) + df = df.drop(columns=["values", "index"]) + render_dataframe(df, console=console, output_format=output_format) + + displayed = offset + len(collection) + if displayed < collection.total_count: + logger.warning( + f"Displaying {len(collection)} of {collection.total_count} series. " + f"Use --limit / --offset to see more." + ) + + @app.command() def fail_running( ctx: typer.Context, diff --git a/packages/climate-ref/src/climate_ref/results/__init__.py b/packages/climate-ref/src/climate_ref/results/__init__.py new file mode 100644 index 000000000..0c01e6916 --- /dev/null +++ b/packages/climate-ref/src/climate_ref/results/__init__.py @@ -0,0 +1,66 @@ +""" +Data access layer for REF results. + +This package is the single source of truth for how result data is filtered and queried. It serves +notebooks / local users (via pandas-friendly, detached value objects) and, in future, the ref-app +API (which can reuse the shared ``Select`` builders directly). + +Layers: + +* [_query][climate_ref.results._query] -- ``MetricValueFilter`` + ``select_*`` builders + (return a ``Select``; the source of truth for filter/join logic) + ``count_values`` + + ``latest_execution_for_group``. +* [frames][climate_ref.results.frames] -- ``*_to_frame`` + ``collect_facets``. +* [outliers][climate_ref.results.outliers] -- source-id-aware IQR outlier detection. +* [values][climate_ref.results.values] -- typed DTOs + collections + the ``Reader`` facade. + +The typical notebook entry point:: + + from climate_ref.config import Config + from climate_ref.database import Database + from climate_ref.results import Reader + + with Database.from_config(Config.default(), read_only=True) as db: + df = Reader(db).scalar_values().to_pandas() +""" + +from climate_ref.results._query import ( + MetricValueFilter, + count_values, + latest_execution_for_group, + select_scalar_values, + select_series_values, +) +from climate_ref.results.frames import ( + collect_facets, + scalar_values_to_frame, + series_values_to_frame, +) +from climate_ref.results.outliers import OutlierPolicy, detect_scalar_outliers +from climate_ref.results.values import ( + Facet, + Reader, + ScalarValue, + ScalarValueCollection, + SeriesValue, + SeriesValueCollection, +) + +__all__ = [ + "Facet", + "MetricValueFilter", + "OutlierPolicy", + "Reader", + "ScalarValue", + "ScalarValueCollection", + "SeriesValue", + "SeriesValueCollection", + "collect_facets", + "count_values", + "detect_scalar_outliers", + "latest_execution_for_group", + "scalar_values_to_frame", + "select_scalar_values", + "select_series_values", + "series_values_to_frame", +] diff --git a/packages/climate-ref/src/climate_ref/results/_query.py b/packages/climate-ref/src/climate_ref/results/_query.py new file mode 100644 index 000000000..92f71fa35 --- /dev/null +++ b/packages/climate-ref/src/climate_ref/results/_query.py @@ -0,0 +1,187 @@ +""" +Filter and ``Select`` builders for metric-value queries. + +This module is the single source of truth for *how* metric values are filtered and joined. +Both the typed facade ([climate_ref.results.values.Reader][]) and any power-user / API +caller build on the ``select_*`` functions here, so the filter/join semantics live in exactly +one place. This mirrors the existing house style of module-level query functions that take a +``Session`` (see [climate_ref.models.execution.get_execution_group_and_latest_filtered][]). + +The functions return a SQLAlchemy ``Select`` and never touch a ``Session`` -- callers add +their own pagination, wrap in ``exists()``, or hand them to the facade to materialise. +""" + +from collections.abc import Mapping, Sequence +from typing import Any + +import attrs +from sqlalchemy import Select, func, or_, select +from sqlalchemy.orm import Session + +from climate_ref.models import Execution, ExecutionGroup +from climate_ref.models.diagnostic import Diagnostic +from climate_ref.models.metric_value import ( + MetricValue, + ScalarMetricValue, + SeriesMetricValue, +) +from climate_ref.models.provider import Provider + + +@attrs.frozen(kw_only=True) +class MetricValueFilter: + """ + Declarative filter over metric values. + + Every field is optional; ``None``/empty means "do not constrain on this axis". The same + object drives scalar and series queries, frame conversion and facet collection, so a filter + is written once and reused across output shapes. + + Two styles of diagnostic/provider filtering are offered deliberately: + + * ``diagnostic_slug`` / ``provider_slug`` are **exact** matches, for API path-scoped queries + (``/diagnostics/{provider_slug}/{diagnostic_slug}/values``). + * ``diagnostic_contains`` / ``provider_contains`` are case-insensitive **substring** matches + (OR-combined), for CLI-style search. + + Keeping them separate avoids the ``Diagnostic.slug`` vs ``Diagnostic.name`` divergence that + exists today between the CLI and the ref-app API. + """ + + # scoping to executions / groups + execution_ids: Sequence[int] | None = None + execution_group_ids: Sequence[int] | None = None + + # provenance filters -- exact (API) vs substring (CLI search) + diagnostic_slug: str | None = None + provider_slug: str | None = None + diagnostic_contains: Sequence[str] | None = None + provider_contains: Sequence[str] | None = None + + # CV dimension filters, keyed by registered dimension name; str -> equality, sequence -> IN + dimensions: Mapping[str, str | Sequence[str]] | None = None + + # id isolate / exclude (isolate takes precedence, matching the API) + isolate_ids: Sequence[int] | None = None + exclude_ids: Sequence[int] | None = None + + # series-only: reference_id IS NOT NULL (observations) / IS NULL (model) + reference_only: bool | None = None + + # version / retraction gating + promoted_only: bool = True + include_retracted: bool = False + + def dimension_clauses(self, entity: type[MetricValue]) -> list[Any]: + """ + Build validated SQLAlchemy clauses for the dynamic CV dimension columns. + + Raises + ------ + KeyError + If a key is not a registered CV dimension on ``entity``. + """ + clauses: list[Any] = [] + for key, value in (self.dimensions or {}).items(): + if key not in entity._cv_dimensions: + raise KeyError(f"Unknown dimension column {key!r}") + col = getattr(entity, key) + if isinstance(value, str) or not isinstance(value, Sequence): + clauses.append(col == value) + else: + vals = list(value) + clauses.append(col == vals[0] if len(vals) == 1 else col.in_(vals)) + return clauses + + +def _apply_common( # noqa: PLR0912 + stmt: Select[Any], entity: type[MetricValue], f: MetricValueFilter +) -> Select[Any]: + """ + Apply every filter that is identical for scalar and series to a ``Select``. + + This is the ONLY place provenance, version, retraction, dimension and id filtering is + expressed, so scalar and series can never diverge. + """ + needs_diag = bool( + f.diagnostic_slug or f.provider_slug or f.diagnostic_contains or f.provider_contains + ) or (f.promoted_only) + needs_exec = bool(f.execution_ids or f.execution_group_ids or needs_diag or not f.include_retracted) + + if needs_exec: + stmt = stmt.join(Execution, entity.execution_id == Execution.id) + if not f.include_retracted: + stmt = stmt.where(Execution.retracted.is_(False)) + if f.execution_ids: + stmt = stmt.where(entity.execution_id.in_(list(f.execution_ids))) + if f.execution_group_ids: + stmt = stmt.where(Execution.execution_group_id.in_(list(f.execution_group_ids))) + + if needs_diag: + stmt = stmt.join(ExecutionGroup, Execution.execution_group_id == ExecutionGroup.id) + stmt = stmt.join(Diagnostic, ExecutionGroup.diagnostic_id == Diagnostic.id) + if f.promoted_only: + stmt = stmt.where(ExecutionGroup.diagnostic_version == Diagnostic.promoted_version) + if f.diagnostic_slug: + stmt = stmt.where(Diagnostic.slug == f.diagnostic_slug) + if f.diagnostic_contains: + stmt = stmt.where(or_(*(Diagnostic.slug.ilike(f"%{s.lower()}%") for s in f.diagnostic_contains))) + if f.provider_slug or f.provider_contains: + stmt = stmt.join(Provider, Diagnostic.provider_id == Provider.id) + if f.provider_slug: + stmt = stmt.where(Provider.slug == f.provider_slug) + if f.provider_contains: + stmt = stmt.where(or_(*(Provider.slug.ilike(f"%{s.lower()}%") for s in f.provider_contains))) + + for clause in f.dimension_clauses(entity): + stmt = stmt.where(clause) + + if f.isolate_ids: + stmt = stmt.where(entity.id.in_(list(f.isolate_ids))) + elif f.exclude_ids: + stmt = stmt.where(~entity.id.in_(list(f.exclude_ids))) + + return stmt + + +def select_scalar_values(f: MetricValueFilter | None = None) -> Select[tuple[ScalarMetricValue]]: + """Build a ``Select`` over ``ScalarMetricValue`` for the given filter. No session required.""" + f = f or MetricValueFilter() + return _apply_common(select(ScalarMetricValue), ScalarMetricValue, f) + + +def select_series_values(f: MetricValueFilter | None = None) -> Select[tuple[SeriesMetricValue]]: + """ + Build a ``Select`` over ``SeriesMetricValue``. + + The shared index axis is eager-loaded via the model relationship (``lazy="joined"``), so + ``.index`` / ``.index_name`` are safe to read for the returned rows. + """ + f = f or MetricValueFilter() + stmt = _apply_common(select(SeriesMetricValue), SeriesMetricValue, f) + if f.reference_only is True: + stmt = stmt.where(SeriesMetricValue.reference_id.is_not(None)) + elif f.reference_only is False: + stmt = stmt.where(SeriesMetricValue.reference_id.is_(None)) + return stmt + + +def count_values(session: Session, stmt: Select[Any]) -> int: + """Total row count for a builder's ``Select``, ignoring any offset/limit. For pagination.""" + return session.execute(select(func.count()).select_from(stmt.order_by(None).subquery())).scalar_one() + + +def latest_execution_for_group(session: Session, execution_group_id: int) -> Execution | None: + """ + Return the most recent [Execution][climate_ref.models.execution.Execution] for a group. + + This is the read-side primitive behind the API's ``/executions/{group_id}/values`` behaviour + of defaulting to the latest execution when no ``execution_id`` is supplied. Centralising it + here means consumers stop re-deriving the "latest execution" lookup. + """ + return session.execute( + select(Execution) + .where(Execution.execution_group_id == execution_group_id) + .order_by(Execution.created_at.desc(), Execution.id.desc()) + .limit(1) + ).scalar_one_or_none() diff --git a/packages/climate-ref/src/climate_ref/results/frames.py b/packages/climate-ref/src/climate_ref/results/frames.py new file mode 100644 index 000000000..851753100 --- /dev/null +++ b/packages/climate-ref/src/climate_ref/results/frames.py @@ -0,0 +1,85 @@ +""" +DataFrame conversion and facet collection for metric values. + +These pure helpers replace logic duplicated across the CLI (hand-rolled ``pd.DataFrame([...])`` +comprehensions) and the ref-app API (CSV flattening + ``collect_facets_from_query``). They take +rows / a ``Select`` and return plain data, so they can be reused by any consumer. +""" + +from collections.abc import Sequence +from typing import Any + +import pandas as pd +from sqlalchemy import Select, distinct +from sqlalchemy.orm import Session + +from climate_ref.models.metric_value import ( + MetricValue, + ScalarMetricValue, + SeriesMetricValue, +) + + +def scalar_values_to_frame(rows: Sequence[ScalarMetricValue]) -> pd.DataFrame: + """ + Flatten scalar rows to a tidy DataFrame. + + One row per value; one column per non-null CV dimension, plus ``value``, ``id``, + ``execution_id`` and ``type``. ``value`` is left raw (NaN/inf preserved); callers that + serialise to JSON/CSV are responsible for any sanitisation. + """ + records = [] + for r in rows: + rec: dict[str, Any] = dict(r.dimensions) + rec.update(id=r.id, execution_id=r.execution_id, value=r.value, type="scalar") + records.append(rec) + return pd.DataFrame.from_records(records) + + +def series_values_to_frame(rows: Sequence[SeriesMetricValue]) -> pd.DataFrame: + """ + Flatten series rows to a long-form (tidy) DataFrame. + + One row per (series, index point). Columns: the non-null CV dimensions, plus ``value``, + ``index``, ``index_name``, ``reference_id``, ``id``, ``execution_id`` and ``type``. The index + is resolved from the shared axis; when a series has no index the positional integer is used. + """ + records = [] + for r in rows: + dims = r.dimensions + idx = r.index + name = r.index_name or "index" + for i, v in enumerate(r.values or []): + rec: dict[str, Any] = dict(dims) + rec.update( + id=r.id, + execution_id=r.execution_id, + value=v, + index=idx[i] if idx is not None and i < len(idx) else i, + index_name=name, + reference_id=r.reference_id, + type="series", + ) + records.append(rec) + return pd.DataFrame.from_records(records) + + +def collect_facets( + session: Session, + stmt: Select[Any], + entity: type[MetricValue], +) -> dict[str, list[str]]: + """ + Distinct non-null values for each registered CV dimension of a filtered query. + + Runs one ``DISTINCT`` per dimension over the (pre-pagination) ``stmt`` so cost scales with + cardinality rather than row count. Returns only dimensions that have at least one value. + """ + facets: dict[str, list[str]] = {} + for key in entity._cv_dimensions: + col = getattr(entity, key) + sub = stmt.with_only_columns(distinct(col)).order_by(None) + values = [v for (v,) in session.execute(sub) if v is not None] + if values: + facets[key] = sorted(values) + return facets diff --git a/packages/climate-ref/src/climate_ref/results/outliers.py b/packages/climate-ref/src/climate_ref/results/outliers.py new file mode 100644 index 000000000..d6a38e283 --- /dev/null +++ b/packages/climate-ref/src/climate_ref/results/outliers.py @@ -0,0 +1,167 @@ +""" +Outlier detection for scalar metric values. + +Outlier detection is *read-model* logic, not presentation policy: whether a value is flagged +determines which rows a view returns and what the collection-level counts mean. It therefore +lives in ``climate_ref`` (the source of truth) rather than in any single consumer. + +The algorithm is source-id-aware IQR: within each ``group_by`` group, IQR bounds are computed on +the per-``source_id`` mean value (so each model is weighted equally regardless of ensemble size), +then applied to individual values. ``"Reference"`` values are never flagged; non-finite values +(NaN/inf) are always flagged. + +This is a framework-agnostic port of the logic previously living in the ref-app backend; it takes +and returns plain data and imports no web framework. +""" + +import math +import statistics +from collections.abc import Sequence + +import attrs +import pandas as pd + +from climate_ref.models.metric_value import ScalarMetricValue + + +@attrs.frozen(kw_only=True) +class OutlierPolicy: + """ + Configuration for scalar outlier detection. + + The defaults reproduce the behaviour users currently see through the ref-app API + (``factor=10.0``, source-id-aware IQR grouped by ``("statistic", "metric")``). + """ + + method: str = "iqr" + """Detection method. ``"off"`` disables detection; ``"iqr"`` enables it.""" + + factor: float = 10.0 + """Multiplier on the IQR to set the outlier bounds (``Q1 - factor*IQR``, ``Q3 + factor*IQR``).""" + + min_n: int = 4 + """Minimum number of source_ids (or values) in a group required to run detection.""" + + group_by: tuple[str, ...] = ("statistic", "metric") + """CV dimensions to group by before computing bounds. Missing dimensions are ignored.""" + + @property + def enabled(self) -> bool: + """Whether detection should run.""" + return self.method == "iqr" + + +@attrs.frozen(kw_only=True) +class AnnotatedScalar: + """A scalar ORM row paired with its outlier verdict.""" + + value: ScalarMetricValue + is_outlier: bool + verification_status: str # "verified" | "unverified" + + +def _flag_outliers_iqr(values: Sequence[float], factor: float, min_n: int) -> list[bool]: + """Flag outliers with a plain IQR test (fallback when no source_id is available).""" + n = len(values) + if n < min_n: + return [False] * n + quantiles = statistics.quantiles(values, n=4, method="inclusive") + q1, q3 = quantiles[0], quantiles[2] + iqr = q3 - q1 + lower, upper = q1 - factor * iqr, q3 + factor * iqr + return [v < lower or v > upper for v in values] + + +def _iqr_bounds_by_source_id(df: pd.DataFrame, factor: float, min_n: int) -> tuple[float, float] | None: + """IQR bounds computed on per-source_id means (equal model weighting).""" + if "source_id" not in df.columns: + return None + non_reference = df[df["source_id"] != "Reference"] + source_id_means = non_reference.groupby("source_id")["value"].mean() + if len(source_id_means) < min_n: + return None + quantiles = statistics.quantiles(source_id_means.tolist(), n=4, method="inclusive") + q1, q3 = quantiles[0], quantiles[2] + iqr = q3 - q1 + return q1 - factor * iqr, q3 + factor * iqr + + +def _is_finite(x: object) -> bool: + return isinstance(x, int | float) and not math.isinf(x) and not math.isnan(x) + + +def detect_scalar_outliers( + scalar_values: Sequence[ScalarMetricValue], + policy: OutlierPolicy, +) -> tuple[list[AnnotatedScalar], int]: + """ + Annotate scalar values with outlier verdicts. + + Parameters + ---------- + scalar_values + The full (unpaginated) set of scalar rows for the query scope. Detection must run over + the whole set so IQR bounds are globally consistent. + policy + Detection configuration. + + Returns + ------- + : + A tuple of (annotated values in input order, total number of outliers). + """ + if not scalar_values: + return [], 0 + + if not policy.enabled: + return ( + [ + AnnotatedScalar(value=sv, is_outlier=False, verification_status="verified") + for sv in scalar_values + ], + 0, + ) + + df = pd.DataFrame( + [{"scalar_value": sv, "value": sv.value, **sv.dimensions, "id": sv.id} for sv in scalar_values] + ) + group_by = [g for g in policy.group_by if g in df.columns] + + verdict_by_id: dict[int, bool] = {} + groups = df.groupby(list(group_by)) if group_by else [(None, df)] + for _, group in groups: + if "source_id" in group.columns and len(group) >= policy.min_n: + bounds = _iqr_bounds_by_source_id(group, factor=policy.factor, min_n=policy.min_n) + if bounds is not None: + lower, upper = bounds + source_flags = [ + (row["value"] < lower or row["value"] > upper) + if row["source_id"] != "Reference" + else False + for _, row in group.iterrows() + ] + else: + source_flags = [False] * len(group) + elif len(group) >= policy.min_n: + source_flags = _flag_outliers_iqr( + group["value"].to_list(), factor=policy.factor, min_n=policy.min_n + ) + else: + source_flags = [False] * len(group) + + for (_, row), flagged in zip(group.iterrows(), source_flags): + verdict_by_id[row["id"]] = bool(flagged) or not _is_finite(row["value"]) + + annotated: list[AnnotatedScalar] = [] + total = 0 + for sv in scalar_values: + is_outlier = verdict_by_id.get(sv.id, False) + annotated.append( + AnnotatedScalar( + value=sv, + is_outlier=is_outlier, + verification_status="unverified" if is_outlier else "verified", + ) + ) + total += int(is_outlier) + return annotated, total diff --git a/packages/climate-ref/src/climate_ref/results/values.py b/packages/climate-ref/src/climate_ref/results/values.py new file mode 100644 index 000000000..f888c2499 --- /dev/null +++ b/packages/climate-ref/src/climate_ref/results/values.py @@ -0,0 +1,386 @@ +""" +Typed, ORM-free surface for metric-value results. + +[Reader][climate_ref.results.values.Reader] is the intent-named entry point that notebooks +and (eventually) the API use. Its read methods return frozen, detached value objects wrapped in +collections that offer ``to_pandas()``. The objects are fully materialised, so they remain valid +after the originating session closes -- a notebook can build a DataFrame inside a +``with Database(...)`` block and keep using it afterwards. + +Everything here is a thin layer over [climate_ref.results._query][] (the shared ``Select`` +builders) and [climate_ref.results.outliers][]; power users who need the raw ``Select`` or ORM +rows use those modules directly. +""" + +from collections.abc import Iterator, Mapping, Sequence +from typing import Any + +import attrs +import pandas as pd +from sqlalchemy.orm import Session, joinedload + +from climate_ref.database import Database +from climate_ref.models.diagnostic import Diagnostic +from climate_ref.models.execution import Execution, ExecutionGroup +from climate_ref.models.metric_value import ScalarMetricValue, SeriesMetricValue +from climate_ref.results._query import ( + MetricValueFilter, + count_values, + select_scalar_values, + select_series_values, +) +from climate_ref.results.frames import collect_facets +from climate_ref.results.outliers import OutlierPolicy, detect_scalar_outliers + + +def _kind_of(dimensions: Mapping[str, str]) -> str: + """Resolve the model/reference ``kind`` from the CV dimensions (defaults to ``"model"``).""" + return dimensions.get("kind", "model") + + +@attrs.frozen(kw_only=True) +class Facet: + """Distinct values observed for one CV dimension across a filtered query.""" + + key: str + values: tuple[str, ...] + + +@attrs.frozen(kw_only=True) +class ScalarValue: + """A single scalar metric value, detached from the ORM.""" + + id: int + execution_id: int + execution_group_id: int + value: float | None + kind: str + dimensions: Mapping[str, str] + attributes: Mapping[str, Any] + # per-row outlier verdict (populated when detection ran) + is_outlier: bool | None = None + verification_status: str | None = None + # optional provenance, only when include_context=True + diagnostic_slug: str | None = None + provider_slug: str | None = None + + +@attrs.frozen(kw_only=True) +class SeriesValue: + """A 1-d series metric value, detached from the ORM. The index is snapshotted from the shared axis.""" + + id: int + execution_id: int + execution_group_id: int + values: Sequence[float | int] + index: Sequence[float | int | str] | None + index_name: str | None + reference_id: str | None + kind: str + dimensions: Mapping[str, str] + attributes: Mapping[str, Any] + diagnostic_slug: str | None = None + provider_slug: str | None = None + + @property + def is_reference(self) -> bool: + """True for observation/reference series (``reference_id`` is set).""" + return self.reference_id is not None + + +@attrs.frozen(kw_only=True) +class ScalarValueCollection: + """An immutable page of scalar values plus collection-level, outlier-aware metadata.""" + + items: tuple[ScalarValue, ...] + total_count: int + facets: tuple[Facet, ...] + offset: int + limit: int | None + had_outliers: bool + outlier_count: int + + def __iter__(self) -> Iterator[ScalarValue]: + return iter(self.items) + + def __len__(self) -> int: + return len(self.items) + + def facets_dict(self) -> dict[str, list[str]]: + """Facets as a plain ``{dimension: [values]}`` mapping.""" + return {f.key: list(f.values) for f in self.facets} + + def to_pandas(self) -> pd.DataFrame: + """Tidy DataFrame: one row per value, one column per CV dimension present, plus metadata.""" + detection_ran = any(v.is_outlier is not None for v in self.items) + records = [] + for v in self.items: + rec: dict[str, Any] = dict(v.dimensions) + rec.update( + id=v.id, + execution_id=v.execution_id, + execution_group_id=v.execution_group_id, + kind=v.kind, + value=v.value, + ) + if detection_ran: + rec.update(is_outlier=v.is_outlier, verification_status=v.verification_status) + if v.diagnostic_slug is not None: + rec.update(diagnostic_slug=v.diagnostic_slug, provider_slug=v.provider_slug) + records.append(rec) + return pd.DataFrame.from_records(records) + + +@attrs.frozen(kw_only=True) +class SeriesValueCollection: + """An immutable page of series values plus collection-level metadata.""" + + items: tuple[SeriesValue, ...] + total_count: int + facets: tuple[Facet, ...] + offset: int + limit: int | None + + def __iter__(self) -> Iterator[SeriesValue]: + return iter(self.items) + + def __len__(self) -> int: + return len(self.items) + + def facets_dict(self) -> dict[str, list[str]]: + """Facets as a plain ``{dimension: [values]}`` mapping.""" + return {f.key: list(f.values) for f in self.facets} + + def to_pandas(self, *, explode: bool = True) -> pd.DataFrame: + """ + DataFrame of the series values. + + With ``explode=True`` (default) the result is long-form: one row per (series, index point), + matching the API's CSV shape. With ``explode=False`` each series is one row with list-valued + ``values``/``index`` cells. + """ + records = [] + for v in self.items: + base: dict[str, Any] = dict(v.dimensions) + base.update( + id=v.id, + execution_id=v.execution_id, + execution_group_id=v.execution_group_id, + kind=v.kind, + index_name=v.index_name or "index", + reference_id=v.reference_id, + ) + if explode: + idx = v.index + for i, value in enumerate(v.values): + rec = dict(base) + rec.update(value=value, index=idx[i] if idx is not None and i < len(idx) else i) + records.append(rec) + else: + rec = dict(base) + rec.update(values=list(v.values), index=list(v.index) if v.index is not None else None) + records.append(rec) + return pd.DataFrame.from_records(records) + + +class Reader: + """ + Typed entry point to REF query results. + + Constructed from a [Database][climate_ref.database.Database], which owns the session and the + read-only story. All read methods return detached collections that outlive the session. + """ + + def __init__(self, database: Database) -> None: + self._db = database + + @property + def session(self) -> Session: + """The underlying database session.""" + return self._db.session + + def _facets(self, base_stmt: Any, entity: Any) -> tuple[Facet, ...]: + facet_map = collect_facets(self.session, base_stmt, entity) + return tuple(Facet(key=k, values=tuple(v)) for k, v in facet_map.items()) + + def scalar_values( # noqa: PLR0913 + self, + filters: MetricValueFilter | None = None, + *, + outliers: OutlierPolicy | None = None, + include_unverified: bool = False, + offset: int = 0, + limit: int | None = None, + with_facets: bool = True, + include_context: bool = False, + ) -> ScalarValueCollection: + """ + Query scalar values, returning an outlier-aware collection. + + When outlier detection is disabled (the default) pagination and counting happen in SQL, + so cost scales with the requested page rather than the whole result set. When ``outliers`` + is enabled, detection runs over the FULL filtered set so IQR bounds are globally + consistent; with ``include_unverified`` False, flagged values are then removed before + pagination and excluded from ``total_count``. ``facets`` are always computed over the full + filtered set (before any outlier removal and pagination). + """ + filters = filters or MetricValueFilter() + policy = outliers or OutlierPolicy(method="off") + + base_stmt = select_scalar_values(filters) + facets = self._facets(base_stmt, ScalarMetricValue) if with_facets else () + + if not policy.enabled: + # No detection: page and count in SQL so we never materialise the whole table. + total_count = count_values(self.session, base_stmt) + load_stmt = base_stmt.options(self._scalar_loader(include_context)) + if limit is not None: + load_stmt = load_stmt.offset(offset).limit(limit) + elif offset: + load_stmt = load_stmt.offset(offset) + rows = list(self.session.execute(load_stmt).scalars().all()) + items = tuple( + self._to_scalar_dto(r, False, "verified", include_context, detection_ran=False) for r in rows + ) + return ScalarValueCollection( + items=items, + total_count=total_count, + facets=facets, + offset=offset, + limit=limit, + had_outliers=False, + outlier_count=0, + ) + + # Detection enabled: materialise the full filtered set so IQR bounds are globally + # consistent, then filter and paginate in Python. + load_stmt = base_stmt.options(self._scalar_loader(include_context)) + rows = list(self.session.execute(load_stmt).scalars().all()) + + annotated, outlier_count = detect_scalar_outliers(rows, policy) + had_outliers = outlier_count > 0 + if not include_unverified: + annotated = [a for a in annotated if not a.is_outlier] + + total_count = len(annotated) + page = annotated[offset : offset + limit] if limit is not None else annotated[offset:] + items = tuple( + self._to_scalar_dto( + a.value, a.is_outlier, a.verification_status, include_context, detection_ran=True + ) + for a in page + ) + return ScalarValueCollection( + items=items, + total_count=total_count, + facets=facets, + offset=offset, + limit=limit, + had_outliers=had_outliers, + outlier_count=outlier_count, + ) + + def series_values( + self, + filters: MetricValueFilter | None = None, + *, + offset: int = 0, + limit: int | None = None, + with_facets: bool = True, + include_context: bool = False, + ) -> SeriesValueCollection: + """ + Query series values with SQL-level pagination. + + The shared index axis is resolved so ``index``/``index_name`` are populated. ``facets`` are + computed over the full filtered set before pagination. + """ + filters = filters or MetricValueFilter() + + base_stmt = select_series_values(filters) + facets = self._facets(base_stmt, SeriesMetricValue) if with_facets else () + total_count = count_values(self.session, base_stmt) + + load_stmt = base_stmt.options(self._series_loader(include_context)) + if limit is not None: + load_stmt = load_stmt.offset(offset).limit(limit) + elif offset: + load_stmt = load_stmt.offset(offset) + rows = list(self.session.execute(load_stmt).scalars().unique().all()) + + items = tuple(self._to_series_dto(r, include_context) for r in rows) + return SeriesValueCollection( + items=items, total_count=total_count, facets=facets, offset=offset, limit=limit + ) + + # -- loaders / materialisers ------------------------------------------------ + @staticmethod + def _scalar_loader(include_context: bool) -> Any: + exec_load = joinedload(ScalarMetricValue.execution) + if include_context: + return ( + exec_load.joinedload(Execution.execution_group) + .joinedload(ExecutionGroup.diagnostic) + .joinedload(Diagnostic.provider) + ) + return exec_load + + @staticmethod + def _series_loader(include_context: bool) -> Any: + exec_load = joinedload(SeriesMetricValue.execution) + if include_context: + return ( + exec_load.joinedload(Execution.execution_group) + .joinedload(ExecutionGroup.diagnostic) + .joinedload(Diagnostic.provider) + ) + return exec_load + + @staticmethod + def _context_slugs(execution: Execution, include_context: bool) -> tuple[str | None, str | None]: + if not include_context: + return None, None + diagnostic = execution.execution_group.diagnostic + return diagnostic.slug, diagnostic.provider.slug + + def _to_scalar_dto( + self, + row: ScalarMetricValue, + is_outlier: bool, + verification_status: str, + include_context: bool, + detection_ran: bool, + ) -> ScalarValue: + diagnostic_slug, provider_slug = self._context_slugs(row.execution, include_context) + dims = dict(row.dimensions) + return ScalarValue( + id=row.id, + execution_id=row.execution_id, + execution_group_id=row.execution.execution_group_id, + value=row.value, + kind=_kind_of(dims), + dimensions=dims, + attributes=dict(row.attributes or {}), + is_outlier=is_outlier if detection_ran else None, + verification_status=verification_status if detection_ran else None, + diagnostic_slug=diagnostic_slug, + provider_slug=provider_slug, + ) + + def _to_series_dto(self, row: SeriesMetricValue, include_context: bool) -> SeriesValue: + diagnostic_slug, provider_slug = self._context_slugs(row.execution, include_context) + dims = dict(row.dimensions) + return SeriesValue( + id=row.id, + execution_id=row.execution_id, + execution_group_id=row.execution.execution_group_id, + values=list(row.values or []), + index=list(row.index) if row.index is not None else None, + index_name=row.index_name, + reference_id=row.reference_id, + kind=_kind_of(dims), + dimensions=dims, + attributes=dict(row.attributes or {}), + diagnostic_slug=diagnostic_slug, + provider_slug=provider_slug, + ) diff --git a/packages/climate-ref/tests/unit/cli/test_executions.py b/packages/climate-ref/tests/unit/cli/test_executions.py index b2573a9b7..ae223e0a1 100644 --- a/packages/climate-ref/tests/unit/cli/test_executions.py +++ b/packages/climate-ref/tests/unit/cli/test_executions.py @@ -1,5 +1,7 @@ import datetime +import json import pathlib +from typing import ClassVar from unittest.mock import patch import pytest @@ -12,7 +14,7 @@ from climate_ref.models.dataset import CMIP6Dataset from climate_ref.models.diagnostic import Diagnostic from climate_ref.models.execution import ExecutionOutput, ResultOutputType, execution_datasets -from climate_ref.models.metric_value import ScalarMetricValue +from climate_ref.models.metric_value import ScalarMetricValue, SeriesIndex, SeriesMetricValue from climate_ref.provider_registry import _register_provider from climate_ref_core.datasets import SourceDatasetType @@ -1186,3 +1188,151 @@ def test_reingest_success_path(self, db_with_executions, invoke_cli, config): assert result.exit_code == 0 assert "1 succeeded" in result.stdout assert "0 skipped" in result.stdout + + +class TestExecutionValues: + """The ``executions values`` command, backed by the shared ``climate_ref.results`` layer.""" + + _MODELS: ClassVar[dict[str, float]] = { + "ACCESS-CM2": 10.0, + "CESM3": 10.5, + "MIROC6": 9.5, + "MPI-ESM": 10.2, + "NorESM": 9.8, + } + + def _setup(self, db): + """Seed one group / execution with six scalar tas values (one wild) and one series.""" + with db.session.begin(): + diagnostic = db.session.query(Diagnostic).first() + assert diagnostic is not None + group = ExecutionGroup(key="valkey", diagnostic_id=diagnostic.id, selectors={}) + db.session.add(group) + db.session.flush() + execution = Execution( + execution_group_id=group.id, + output_fragment="frag", + dataset_hash="valhash", + successful=True, + ) + db.session.add(execution) + db.session.flush() + + for source_id, value in self._MODELS.items(): + db.session.add( + ScalarMetricValue.build( + execution_id=execution.id, + value=value, + attributes=None, + dimensions={"statistic": "mean", "metric": "tas", "source_id": source_id}, + ) + ) + db.session.add( + ScalarMetricValue.build( + execution_id=execution.id, + value=1000.0, # wild outlier + attributes=None, + dimensions={"statistic": "mean", "metric": "tas", "source_id": "WILD"}, + ) + ) + + axis = SeriesIndex.get_or_create(db.session, "time", [0, 1, 2]) + db.session.add( + SeriesMetricValue.build( + execution_id=execution.id, + values=[1.0, 2.0, 3.0], + index_axis=axis, + dimensions={"source_id": "ACCESS-CM2", "metric": "tas"}, + attributes=None, + ) + ) + db.session.flush() + group_id = group.id + execution_id = execution.id + return group_id, execution_id + + def test_scalar_table(self, db_seeded, invoke_cli): + group_id, _ = self._setup(db_seeded) + + result = invoke_cli(["executions", "values", str(group_id)]) + + assert result.exit_code == 0 + # Dimension columns and the raw values are rendered; outliers shown by default. + assert "source_id" in result.stdout + assert "WILD" in result.stdout + assert "1000.0" in result.stdout + + def test_scalar_json(self, db_seeded, invoke_cli): + group_id, _ = self._setup(db_seeded) + + result = invoke_cli(["executions", "values", str(group_id), "--format", "json"]) + + assert result.exit_code == 0 + records = json.loads(result.stdout) + assert len(records) == len(self._MODELS) + 1 # models + wild + assert any(r["value"] == 1000.0 for r in records) + + def test_scalar_outliers_hidden(self, db_seeded, invoke_cli): + group_id, _ = self._setup(db_seeded) + + result = invoke_cli(["executions", "values", str(group_id), "--outliers"]) + + assert result.exit_code == 0 + assert "WILD" not in result.stdout + assert "flagged as outliers" in result.stderr + + def test_scalar_outliers_shown(self, db_seeded, invoke_cli): + group_id, _ = self._setup(db_seeded) + + result = invoke_cli(["executions", "values", str(group_id), "--outliers", "--include-unverified"]) + + assert result.exit_code == 0 + assert "WILD" in result.stdout + + def test_dimension_filter(self, db_seeded, invoke_cli): + group_id, _ = self._setup(db_seeded) + + result = invoke_cli(["executions", "values", str(group_id), "-d", "source_id=WILD"]) + + assert result.exit_code == 0 + assert "WILD" in result.stdout + assert "ACCESS-CM2" not in result.stdout + + def test_unknown_dimension(self, db_seeded, invoke_cli): + group_id, _ = self._setup(db_seeded) + + result = invoke_cli( + ["executions", "values", str(group_id), "-d", "not_a_dim=1"], expected_exit_code=1 + ) + + assert "Unknown dimension" in result.stderr + + def test_series(self, db_seeded, invoke_cli): + group_id, _ = self._setup(db_seeded) + + result = invoke_cli(["executions", "values", str(group_id), "--kind", "series"]) + + assert result.exit_code == 0 + # The terminal shows a compact per-series summary rather than raw arrays. + assert "n_points" in result.stdout + assert "3" in result.stdout + + def test_specific_execution_wrong_group(self, db_seeded, invoke_cli): + _group_id, execution_id = self._setup(db_seeded) + with db_seeded.session.begin(): + other = ExecutionGroup(key="othergroup", diagnostic_id=1) + db_seeded.session.add(other) + db_seeded.session.flush() + other_id = other.id + + result = invoke_cli( + ["executions", "values", str(other_id), "--execution-id", str(execution_id)], + expected_exit_code=1, + ) + + assert "does not belong" in result.stderr + + def test_missing_group(self, db_seeded, invoke_cli): + result = invoke_cli(["executions", "values", "99999"], expected_exit_code=1) + + assert "Execution group not found" in result.stderr diff --git a/packages/climate-ref/tests/unit/results/test_results.py b/packages/climate-ref/tests/unit/results/test_results.py new file mode 100644 index 000000000..b790dd9c1 --- /dev/null +++ b/packages/climate-ref/tests/unit/results/test_results.py @@ -0,0 +1,270 @@ +"""Unit tests for the metric-value data access layer (spike).""" + +import math + +import pytest + +from climate_ref.models import Execution, ExecutionGroup +from climate_ref.models.diagnostic import Diagnostic +from climate_ref.models.metric_value import ScalarMetricValue, SeriesIndex, SeriesMetricValue +from climate_ref.results import ( + MetricValueFilter, + OutlierPolicy, + Reader, + latest_execution_for_group, +) + + +@pytest.fixture +def dal_db(db_seeded): + """A seeded DB with one execution group, execution, and scalar + series values.""" + session = db_seeded.session + + with session.begin(): + diagnostic = session.query(Diagnostic).first() + assert diagnostic is not None, "example provider should seed at least one diagnostic" + group = ExecutionGroup(key="dal-key", diagnostic_id=diagnostic.id, selectors={}) + session.add(group) + session.flush() + execution = Execution( + execution_group_id=group.id, + output_fragment="frag", + dataset_hash="hash-dal", + successful=True, + ) + session.add(execution) + session.flush() + + # A well-behaved cluster of five models, one wild outlier, a NaN, and a Reference value. + base_sources = { + "ACCESS-CM2": 10.0, + "CESM3": 10.5, + "MIROC6": 9.5, + "MPI-ESM": 10.2, + "NorESM": 9.8, + } + for source_id, value in base_sources.items(): + session.add( + ScalarMetricValue.build( + execution_id=execution.id, + value=value, + attributes=None, + dimensions={"statistic": "mean", "metric": "tas", "source_id": source_id}, + ) + ) + session.add( + ScalarMetricValue.build( + execution_id=execution.id, + value=1000.0, # wild outlier + attributes=None, + dimensions={"statistic": "mean", "metric": "tas", "source_id": "WILD"}, + ) + ) + session.add( + ScalarMetricValue.build( + execution_id=execution.id, + value=math.nan, # non-finite -> always flagged (own group so it can't poison the tas IQR) + attributes=None, + dimensions={"statistic": "mean", "metric": "broken", "source_id": "BROKEN"}, + ) + ) + session.add( + ScalarMetricValue.build( + execution_id=execution.id, + value=99999.0, # Reference -> never flagged + attributes=None, + dimensions={"statistic": "mean", "metric": "tas", "source_id": "Reference"}, + ) + ) + + axis = SeriesIndex.get_or_create(session, "time", [0, 1, 2]) + session.add( + SeriesMetricValue.build( + execution_id=execution.id, + values=[1.0, 2.0, 3.0], + index_axis=axis, + dimensions={"source_id": "ACCESS-CM2", "metric": "tas"}, + attributes=None, + ) + ) + ref_series = SeriesMetricValue.build( + execution_id=execution.id, + values=[1.1, 2.1, 3.1], + index_axis=axis, + dimensions={"source_id": "Reference", "metric": "tas"}, + attributes=None, + ) + ref_series.reference_id = "ref-hash-1" + session.add(ref_series) + session.flush() + group_id = group.id # capture inside the txn; reading it post-commit would autobegin + + db_seeded.execution_group_id = group_id + return db_seeded + + +class TestScalarValues: + def test_returns_all_without_outlier_detection(self, dal_db): + ref = Reader(dal_db) + coll = ref.scalar_values(MetricValueFilter(execution_group_ids=[dal_db.execution_group_id])) + assert coll.total_count == 8 # 5 base + wild + nan + reference + assert len(coll) == 8 + assert coll.had_outliers is False + assert coll.outlier_count == 0 + # execution_group_id is projected onto every row + assert all(v.execution_group_id == dal_db.execution_group_id for v in coll) + + def test_outlier_detection_removes_flagged(self, dal_db): + ref = Reader(dal_db) + coll = ref.scalar_values( + MetricValueFilter(execution_group_ids=[dal_db.execution_group_id]), + outliers=OutlierPolicy(), + include_unverified=False, + ) + # WILD and BROKEN (NaN) removed; Reference retained. + assert coll.had_outliers is True + assert coll.outlier_count >= 2 + assert coll.total_count == 8 - coll.outlier_count + source_ids = {v.dimensions["source_id"] for v in coll} + assert "WILD" not in source_ids + assert "BROKEN" not in source_ids + assert "Reference" in source_ids + + def test_include_unverified_keeps_flagged_with_annotation(self, dal_db): + ref = Reader(dal_db) + coll = ref.scalar_values( + MetricValueFilter(execution_group_ids=[dal_db.execution_group_id]), + outliers=OutlierPolicy(), + include_unverified=True, + ) + assert coll.total_count == 8 + flagged = {v.dimensions["source_id"] for v in coll if v.is_outlier} + assert "WILD" in flagged + assert "BROKEN" in flagged + # verification_status mirrors the flag + assert all((v.verification_status == "unverified") == bool(v.is_outlier) for v in coll) + + def test_to_pandas_columns(self, dal_db): + ref = Reader(dal_db) + coll = ref.scalar_values(MetricValueFilter(execution_group_ids=[dal_db.execution_group_id])) + df = coll.to_pandas() + expected_cols = ( + "source_id", + "statistic", + "metric", + "value", + "id", + "execution_id", + "execution_group_id", + "kind", + ) + for col in expected_cols: + assert col in df.columns + assert len(df) == 8 + + def test_facets(self, dal_db): + ref = Reader(dal_db) + coll = ref.scalar_values(MetricValueFilter(execution_group_ids=[dal_db.execution_group_id])) + facets = coll.facets_dict() + assert "source_id" in facets + assert "WILD" in facets["source_id"] + assert "metric" in facets + + def test_pagination(self, dal_db): + ref = Reader(dal_db) + coll = ref.scalar_values( + MetricValueFilter(execution_group_ids=[dal_db.execution_group_id]), + limit=3, + offset=0, + ) + assert len(coll) == 3 + assert coll.total_count == 8 + + def test_dimension_filter(self, dal_db): + ref = Reader(dal_db) + coll = ref.scalar_values( + MetricValueFilter( + execution_group_ids=[dal_db.execution_group_id], + dimensions={"source_id": ["ACCESS-CM2", "CESM3"]}, + ) + ) + assert coll.total_count == 2 + assert {v.dimensions["source_id"] for v in coll} == {"ACCESS-CM2", "CESM3"} + + def test_unknown_dimension_raises(self, dal_db): + ref = Reader(dal_db) + with pytest.raises(KeyError, match="not_a_dim"): + ref.scalar_values(MetricValueFilter(dimensions={"not_a_dim": "x"})) + + def test_include_context_adds_slugs(self, dal_db): + ref = Reader(dal_db) + coll = ref.scalar_values( + MetricValueFilter(execution_group_ids=[dal_db.execution_group_id]), + include_context=True, + limit=1, + ) + v = coll.items[0] + assert v.diagnostic_slug is not None + assert v.provider_slug is not None + + def test_detached_after_session_expunge(self, dal_db): + ref = Reader(dal_db) + coll = ref.scalar_values(MetricValueFilter(execution_group_ids=[dal_db.execution_group_id])) + dal_db.session.expunge_all() + # DTOs are detached; building a frame must not touch the ORM/session. + df = coll.to_pandas() + assert len(df) == 8 + + +class TestSeriesValues: + def test_returns_series_with_resolved_index(self, dal_db): + ref = Reader(dal_db) + coll = ref.series_values(MetricValueFilter(execution_group_ids=[dal_db.execution_group_id])) + assert coll.total_count == 2 + v = coll.items[0] + assert v.index == [0, 1, 2] + assert v.index_name == "time" + assert v.execution_group_id == dal_db.execution_group_id + + def test_reference_only_filter(self, dal_db): + ref = Reader(dal_db) + refs = ref.series_values( + MetricValueFilter(execution_group_ids=[dal_db.execution_group_id], reference_only=True) + ) + assert refs.total_count == 1 + assert refs.items[0].is_reference + assert refs.items[0].reference_id == "ref-hash-1" + + models = ref.series_values( + MetricValueFilter(execution_group_ids=[dal_db.execution_group_id], reference_only=False) + ) + assert models.total_count == 1 + assert not models.items[0].is_reference + + def test_to_pandas_long_form(self, dal_db): + ref = Reader(dal_db) + coll = ref.series_values(MetricValueFilter(execution_group_ids=[dal_db.execution_group_id])) + df = coll.to_pandas(explode=True) + # 2 series x 3 points + assert len(df) == 6 + for col in ("value", "index", "index_name", "source_id", "reference_id"): + assert col in df.columns + + +class TestLatestExecutionForGroup: + def test_returns_most_recent(self, dal_db): + session = dal_db.session + group_id = dal_db.execution_group_id + with session.begin(): + newer = Execution( + execution_group_id=group_id, + output_fragment="frag2", + dataset_hash="hash-dal-2", + successful=True, + ) + session.add(newer) + session.flush() + newer_id = newer.id + latest = latest_execution_for_group(session, group_id) + assert latest is not None + assert latest.id == newer_id From 80dbab3746f9c362caa5f525641744029ac2221e Mon Sep 17 00:00:00 2001 From: Jared Lewis Date: Wed, 1 Jul 2026 21:14:19 +1000 Subject: [PATCH 02/23] refactor(results): namespace metric-value reads under reader.values Move the scalar/series read methods onto a ValuesReader sub-reader exposed as Reader.values, making Reader a thin entry point that will hold further per-domain sub-readers as the layer grows. Curate the top-level package surface to export only what a caller names to make a call: the Reader entry point, the filters you pass in, and the value objects you pass in. DTOs, collections, and sub-reader classes now live in their domain submodule (imported by path when a name is needed), and the Select builders stay private in _query. This keeps the namespace small as new domains are added. --- .../src/climate_ref/cli/executions.py | 15 ++--- .../src/climate_ref/results/__init__.py | 56 +++++++------------ .../src/climate_ref/results/values.py | 37 ++++++++++-- .../tests/unit/results/test_results.py | 30 +++++----- 4 files changed, 75 insertions(+), 63 deletions(-) diff --git a/packages/climate-ref/src/climate_ref/cli/executions.py b/packages/climate-ref/src/climate_ref/cli/executions.py index b38b43078..335a39cfe 100644 --- a/packages/climate-ref/src/climate_ref/cli/executions.py +++ b/packages/climate-ref/src/climate_ref/cli/executions.py @@ -35,8 +35,9 @@ MetricValueFilter, OutlierPolicy, Reader, - latest_execution_for_group, ) +from climate_ref.results._query import latest_execution_for_group +from climate_ref.results.values import ValuesReader from climate_ref_core.logging import EXECUTION_LOG_FILENAME app = typer.Typer(help=__doc__) @@ -668,7 +669,7 @@ def values( # noqa: PLR0913 try: if kind == ValueKind.scalar: _render_scalar_values( - reader, + reader.values, filters, console=console, output_format=output_format, @@ -679,7 +680,7 @@ def values( # noqa: PLR0913 ) else: _render_series_values( - reader, + reader.values, filters, console=console, output_format=output_format, @@ -693,7 +694,7 @@ def values( # noqa: PLR0913 def _render_scalar_values( # noqa: PLR0913 - reader: Reader, + values: ValuesReader, filters: MetricValueFilter, *, console: Console, @@ -703,7 +704,7 @@ def _render_scalar_values( # noqa: PLR0913 offset: int, limit: int | None, ) -> None: - collection = reader.scalar_values( + collection = values.scalar_values( filters, outliers=OutlierPolicy() if outliers else None, include_unverified=include_unverified, @@ -732,7 +733,7 @@ def _render_scalar_values( # noqa: PLR0913 def _render_series_values( # noqa: PLR0913 - reader: Reader, + values: ValuesReader, filters: MetricValueFilter, *, console: Console, @@ -740,7 +741,7 @@ def _render_series_values( # noqa: PLR0913 offset: int, limit: int | None, ) -> None: - collection = reader.series_values(filters, offset=offset, limit=limit, with_facets=False) + collection = values.series_values(filters, offset=offset, limit=limit, with_facets=False) if not len(collection): console.print("No series values found.") return diff --git a/packages/climate-ref/src/climate_ref/results/__init__.py b/packages/climate-ref/src/climate_ref/results/__init__.py index 0c01e6916..d301258ef 100644 --- a/packages/climate-ref/src/climate_ref/results/__init__.py +++ b/packages/climate-ref/src/climate_ref/results/__init__.py @@ -21,46 +21,32 @@ from climate_ref.results import Reader with Database.from_config(Config.default(), read_only=True) as db: - df = Reader(db).scalar_values().to_pandas() + df = Reader(db).values.scalar_values().to_pandas() + +Public surface convention +------------------------- + +The top level exports only what a caller *names to make a call*, so the namespace stays small as +domains are added: + +* the ``Reader`` entry point, +* filter objects you construct and pass in (``MetricValueFilter``, and future + ``ExecutionGroupFilter`` / ``DatasetFilter``), +* value objects you pass in (``OutlierPolicy``, and the future ``ResultPaths``). + +Everything the package *returns* -- DTOs, collections, views -- and the sub-reader classes reached +via ``reader.values`` etc. live in their domain submodule (``climate_ref.results.values``, ...). +Import them from there on the rare occasion you need to name one, e.g. +``from climate_ref.results.values import ScalarValue``. The ``Select`` builders and other plumbing +stay in ``climate_ref.results._query`` and are not part of the public surface. """ -from climate_ref.results._query import ( - MetricValueFilter, - count_values, - latest_execution_for_group, - select_scalar_values, - select_series_values, -) -from climate_ref.results.frames import ( - collect_facets, - scalar_values_to_frame, - series_values_to_frame, -) -from climate_ref.results.outliers import OutlierPolicy, detect_scalar_outliers -from climate_ref.results.values import ( - Facet, - Reader, - ScalarValue, - ScalarValueCollection, - SeriesValue, - SeriesValueCollection, -) +from climate_ref.results._query import MetricValueFilter +from climate_ref.results.outliers import OutlierPolicy +from climate_ref.results.values import Reader __all__ = [ - "Facet", "MetricValueFilter", "OutlierPolicy", "Reader", - "ScalarValue", - "ScalarValueCollection", - "SeriesValue", - "SeriesValueCollection", - "collect_facets", - "count_values", - "detect_scalar_outliers", - "latest_execution_for_group", - "scalar_values_to_frame", - "select_scalar_values", - "select_series_values", - "series_values_to_frame", ] diff --git a/packages/climate-ref/src/climate_ref/results/values.py b/packages/climate-ref/src/climate_ref/results/values.py index f888c2499..9d9a17c12 100644 --- a/packages/climate-ref/src/climate_ref/results/values.py +++ b/packages/climate-ref/src/climate_ref/results/values.py @@ -2,16 +2,18 @@ Typed, ORM-free surface for metric-value results. [Reader][climate_ref.results.values.Reader] is the intent-named entry point that notebooks -and (eventually) the API use. Its read methods return frozen, detached value objects wrapped in -collections that offer ``to_pandas()``. The objects are fully materialised, so they remain valid -after the originating session closes -- a notebook can build a DataFrame inside a -``with Database(...)`` block and keep using it afterwards. +and (eventually) the API use, via its [values][climate_ref.results.values.Reader.values] +sub-reader. [ValuesReader][climate_ref.results.values.ValuesReader]'s read methods return frozen, +detached value objects wrapped in collections that offer ``to_pandas()``. The objects are fully +materialised, so they remain valid after the originating session closes -- a notebook can build a +DataFrame inside a ``with Database(...)`` block and keep using it afterwards. Everything here is a thin layer over [climate_ref.results._query][] (the shared ``Select`` builders) and [climate_ref.results.outliers][]; power users who need the raw ``Select`` or ORM rows use those modules directly. """ +import functools from collections.abc import Iterator, Mapping, Sequence from typing import Any @@ -183,9 +185,9 @@ def to_pandas(self, *, explode: bool = True) -> pd.DataFrame: return pd.DataFrame.from_records(records) -class Reader: +class ValuesReader: """ - Typed entry point to REF query results. + Metric-value read domain: scalar/series values and their facets. Constructed from a [Database][climate_ref.database.Database], which owns the session and the read-only story. All read methods return detached collections that outlive the session. @@ -384,3 +386,26 @@ def _to_series_dto(self, row: SeriesMetricValue, include_context: bool) -> Serie diagnostic_slug=diagnostic_slug, provider_slug=provider_slug, ) + + +class Reader: + """ + Typed entry point to REF query results. + + Constructed from a [Database][climate_ref.database.Database], which owns the session and the + read-only story. This is a thin entry point: it exposes per-domain sub-readers as properties, + currently just [values][climate_ref.results.values.Reader.values] for metric-value reads. + """ + + def __init__(self, database: Database) -> None: + self._db = database + + @property + def session(self) -> Session: + """The underlying database session.""" + return self._db.session + + @functools.cached_property + def values(self) -> ValuesReader: + """Metric-value reads (scalar/series/facets).""" + return ValuesReader(self._db) diff --git a/packages/climate-ref/tests/unit/results/test_results.py b/packages/climate-ref/tests/unit/results/test_results.py index b790dd9c1..bcec2064f 100644 --- a/packages/climate-ref/tests/unit/results/test_results.py +++ b/packages/climate-ref/tests/unit/results/test_results.py @@ -11,8 +11,8 @@ MetricValueFilter, OutlierPolicy, Reader, - latest_execution_for_group, ) +from climate_ref.results._query import latest_execution_for_group @pytest.fixture @@ -106,7 +106,7 @@ def dal_db(db_seeded): class TestScalarValues: def test_returns_all_without_outlier_detection(self, dal_db): ref = Reader(dal_db) - coll = ref.scalar_values(MetricValueFilter(execution_group_ids=[dal_db.execution_group_id])) + coll = ref.values.scalar_values(MetricValueFilter(execution_group_ids=[dal_db.execution_group_id])) assert coll.total_count == 8 # 5 base + wild + nan + reference assert len(coll) == 8 assert coll.had_outliers is False @@ -116,7 +116,7 @@ def test_returns_all_without_outlier_detection(self, dal_db): def test_outlier_detection_removes_flagged(self, dal_db): ref = Reader(dal_db) - coll = ref.scalar_values( + coll = ref.values.scalar_values( MetricValueFilter(execution_group_ids=[dal_db.execution_group_id]), outliers=OutlierPolicy(), include_unverified=False, @@ -132,7 +132,7 @@ def test_outlier_detection_removes_flagged(self, dal_db): def test_include_unverified_keeps_flagged_with_annotation(self, dal_db): ref = Reader(dal_db) - coll = ref.scalar_values( + coll = ref.values.scalar_values( MetricValueFilter(execution_group_ids=[dal_db.execution_group_id]), outliers=OutlierPolicy(), include_unverified=True, @@ -146,7 +146,7 @@ def test_include_unverified_keeps_flagged_with_annotation(self, dal_db): def test_to_pandas_columns(self, dal_db): ref = Reader(dal_db) - coll = ref.scalar_values(MetricValueFilter(execution_group_ids=[dal_db.execution_group_id])) + coll = ref.values.scalar_values(MetricValueFilter(execution_group_ids=[dal_db.execution_group_id])) df = coll.to_pandas() expected_cols = ( "source_id", @@ -164,7 +164,7 @@ def test_to_pandas_columns(self, dal_db): def test_facets(self, dal_db): ref = Reader(dal_db) - coll = ref.scalar_values(MetricValueFilter(execution_group_ids=[dal_db.execution_group_id])) + coll = ref.values.scalar_values(MetricValueFilter(execution_group_ids=[dal_db.execution_group_id])) facets = coll.facets_dict() assert "source_id" in facets assert "WILD" in facets["source_id"] @@ -172,7 +172,7 @@ def test_facets(self, dal_db): def test_pagination(self, dal_db): ref = Reader(dal_db) - coll = ref.scalar_values( + coll = ref.values.scalar_values( MetricValueFilter(execution_group_ids=[dal_db.execution_group_id]), limit=3, offset=0, @@ -182,7 +182,7 @@ def test_pagination(self, dal_db): def test_dimension_filter(self, dal_db): ref = Reader(dal_db) - coll = ref.scalar_values( + coll = ref.values.scalar_values( MetricValueFilter( execution_group_ids=[dal_db.execution_group_id], dimensions={"source_id": ["ACCESS-CM2", "CESM3"]}, @@ -194,11 +194,11 @@ def test_dimension_filter(self, dal_db): def test_unknown_dimension_raises(self, dal_db): ref = Reader(dal_db) with pytest.raises(KeyError, match="not_a_dim"): - ref.scalar_values(MetricValueFilter(dimensions={"not_a_dim": "x"})) + ref.values.scalar_values(MetricValueFilter(dimensions={"not_a_dim": "x"})) def test_include_context_adds_slugs(self, dal_db): ref = Reader(dal_db) - coll = ref.scalar_values( + coll = ref.values.scalar_values( MetricValueFilter(execution_group_ids=[dal_db.execution_group_id]), include_context=True, limit=1, @@ -209,7 +209,7 @@ def test_include_context_adds_slugs(self, dal_db): def test_detached_after_session_expunge(self, dal_db): ref = Reader(dal_db) - coll = ref.scalar_values(MetricValueFilter(execution_group_ids=[dal_db.execution_group_id])) + coll = ref.values.scalar_values(MetricValueFilter(execution_group_ids=[dal_db.execution_group_id])) dal_db.session.expunge_all() # DTOs are detached; building a frame must not touch the ORM/session. df = coll.to_pandas() @@ -219,7 +219,7 @@ def test_detached_after_session_expunge(self, dal_db): class TestSeriesValues: def test_returns_series_with_resolved_index(self, dal_db): ref = Reader(dal_db) - coll = ref.series_values(MetricValueFilter(execution_group_ids=[dal_db.execution_group_id])) + coll = ref.values.series_values(MetricValueFilter(execution_group_ids=[dal_db.execution_group_id])) assert coll.total_count == 2 v = coll.items[0] assert v.index == [0, 1, 2] @@ -228,14 +228,14 @@ def test_returns_series_with_resolved_index(self, dal_db): def test_reference_only_filter(self, dal_db): ref = Reader(dal_db) - refs = ref.series_values( + refs = ref.values.series_values( MetricValueFilter(execution_group_ids=[dal_db.execution_group_id], reference_only=True) ) assert refs.total_count == 1 assert refs.items[0].is_reference assert refs.items[0].reference_id == "ref-hash-1" - models = ref.series_values( + models = ref.values.series_values( MetricValueFilter(execution_group_ids=[dal_db.execution_group_id], reference_only=False) ) assert models.total_count == 1 @@ -243,7 +243,7 @@ def test_reference_only_filter(self, dal_db): def test_to_pandas_long_form(self, dal_db): ref = Reader(dal_db) - coll = ref.series_values(MetricValueFilter(execution_group_ids=[dal_db.execution_group_id])) + coll = ref.values.series_values(MetricValueFilter(execution_group_ids=[dal_db.execution_group_id])) df = coll.to_pandas(explode=True) # 2 series x 3 points assert len(df) == 6 From 4c901bee3ee2c18c0c5e404fcec519e4d54cd2b1 Mon Sep 17 00:00:00 2001 From: Jared Lewis Date: Wed, 1 Jul 2026 22:25:00 +1000 Subject: [PATCH 03/23] feat(results): add reader.executions and migrate CLI list/stats onto it Add the executions domain to the results read layer: an ExecutionsReader sub-reader (reached via reader.executions) exposing groups(), statistics() and latest_execution(), with an ExecutionGroupFilter, detached ExecutionGroupView/ExecutionView/ExecutionStats DTOs, an ExecutionGroupCollection, and a select_execution_statistics() builder. groups() wraps the existing get_execution_group_and_latest_filtered helper rather than reimplementing it, since reingest and the versioning tests also depend on that helper; reimplementing would reintroduce the query drift this layer exists to remove. The stats aggregate, previously inlined only in the CLI, moves into the read layer as a reusable Select builder. Migrate the read-only CLI commands list-groups, stats and values onto reader.executions. delete-groups, reingest and fail-running stay on the ORM helper by design: they mutate rows or need the original (oldest) execution, which the detached read layer does not serve. Export only ExecutionGroupFilter at the package top level, keeping the public surface to the entry point plus constructible filters; views, collection and sub-reader are imported from the domain submodule. --- .../src/climate_ref/cli/executions.py | 129 ++---- .../src/climate_ref/results/__init__.py | 23 +- .../src/climate_ref/results/executions.py | 379 ++++++++++++++++ .../src/climate_ref/results/values.py | 26 +- .../tests/unit/results/test_executions.py | 406 ++++++++++++++++++ 5 files changed, 844 insertions(+), 119 deletions(-) create mode 100644 packages/climate-ref/src/climate_ref/results/executions.py create mode 100644 packages/climate-ref/tests/unit/results/test_executions.py diff --git a/packages/climate-ref/src/climate_ref/cli/executions.py b/packages/climate-ref/src/climate_ref/cli/executions.py index 335a39cfe..b9542c4bd 100644 --- a/packages/climate-ref/src/climate_ref/cli/executions.py +++ b/packages/climate-ref/src/climate_ref/cli/executions.py @@ -17,7 +17,7 @@ from rich.panel import Panel from rich.text import Text from rich.tree import Tree -from sqlalchemy import case, func, or_ +from sqlalchemy import or_ from climate_ref.cli._utils import ( OutputFormat, @@ -32,11 +32,11 @@ from climate_ref.models.execution import execution_datasets, get_execution_group_and_latest_filtered from climate_ref.models.provider import Provider from climate_ref.results import ( + ExecutionGroupFilter, MetricValueFilter, OutlierPolicy, Reader, ) -from climate_ref.results._query import latest_execution_for_group from climate_ref.results.values import ValuesReader from climate_ref_core.logging import EXECUTION_LOG_FILENAME @@ -138,10 +138,9 @@ def list_groups( # noqa: PLR0913 The output is a table by default, or machine-readable JSON with ``--format json``. """ - import pandas as pd - session = ctx.obj.database.session console = ctx.obj.console + reader = Reader(ctx.obj.database) # Parse facet filters try: @@ -163,52 +162,25 @@ def list_groups( # noqa: PLR0913 # Apply filters to query try: - all_filtered_results = get_execution_group_and_latest_filtered( - session, - diagnostic_filters=filters.diagnostic, - provider_filters=filters.provider, - facet_filters=filters.facets, - successful=successful, - dirty=dirty, + collection = reader.executions.groups( + ExecutionGroupFilter( + diagnostic_contains=diagnostic, + provider_contains=provider, + facets=facet_filters or None, + successful=successful, + dirty=dirty, + ), + limit=limit, ) - execution_groups_results = all_filtered_results[:limit] except Exception as e: # pragma: no cover logger.error(f"Error applying filters: {e}") raise typer.Exit(code=1) # Check if any results found - if not execution_groups_results: + if len(collection) == 0: emit_no_results_warning(filters, total_count) - results_df = pd.DataFrame( - columns=[ - "id", - "key", - "provider", - "diagnostic", - "dirty", - "successful", - "created_at", - "updated_at", - "selectors", - ] - ) - else: - results_df = pd.DataFrame( - [ - { - "id": eg.id, - "key": eg.key, - "provider": eg.diagnostic.provider.slug, - "diagnostic": eg.diagnostic.slug, - "dirty": eg.dirty, - "successful": result.successful if result else None, - "created_at": eg.created_at, - "updated_at": eg.updated_at, - "selectors": eg.selectors, - } - for eg, result in execution_groups_results - ] - ) + + results_df = collection.to_pandas() # Select columns to display. if column: @@ -224,7 +196,7 @@ def list_groups( # noqa: PLR0913 render_dataframe(results_df, console=console, output_format=output_format) # Show limit warning if applicable - filtered_count = len(all_filtered_results) + filtered_count = collection.total_count if filtered_count > limit: logger.warning( f"Displaying {limit} of {filtered_count} filtered results. " @@ -232,6 +204,8 @@ def list_groups( # noqa: PLR0913 ) +# Stays on `get_execution_group_and_latest_filtered` rather than `reader.executions.groups()` by design. +# Revisit once a mutable query surface exists. @app.command() def delete_groups( # noqa: PLR0912, PLR0913, PLR0915 ctx: typer.Context, @@ -640,7 +614,7 @@ def values( # noqa: PLR0913 raise typer.Exit(code=1) if execution_id is None: - latest = latest_execution_for_group(database.session, group_id) + latest = reader.executions.latest_execution(group_id) if latest is None: logger.error(f"No executions found for group {group_id}.") raise typer.Exit(code=1) @@ -891,64 +865,12 @@ def stats( """ import pandas as pd - session = ctx.obj.database.session console = ctx.obj.console + reader = Reader(ctx.obj.database) - # Subquery to get the latest execution per execution group - latest_exec_subquery = ( - session.query( - Execution.execution_group_id, - func.max(Execution.created_at).label("latest_created_at"), - ) - .group_by(Execution.execution_group_id) - .subquery() - ) - - # Build query: join ExecutionGroup -> Diagnostic -> Provider, and optionally latest Execution - query = ( - session.query( - Provider.slug.label("provider"), - Diagnostic.slug.label("diagnostic"), - func.count().label("total"), - func.sum( - case( - (Execution.successful.is_(None) & Execution.id.isnot(None), 1), - else_=0, - ) - ).label("running"), - func.sum(case((Execution.successful.is_(False), 1), else_=0)).label("failed"), - func.sum(case((Execution.successful.is_(True), 1), else_=0)).label("successful"), - func.sum(case((Execution.id.is_(None), 1), else_=0)).label("not_started"), - func.sum(case((ExecutionGroup.dirty.is_(True), 1), else_=0)).label("dirty"), - ) - .join(Diagnostic, ExecutionGroup.diagnostic_id == Diagnostic.id) - .filter(ExecutionGroup.diagnostic_version == Diagnostic.promoted_version) - .join(Provider, Diagnostic.provider_id == Provider.id) - .outerjoin(latest_exec_subquery, ExecutionGroup.id == latest_exec_subquery.c.execution_group_id) - .outerjoin( - Execution, - (Execution.execution_group_id == ExecutionGroup.id) - & (Execution.created_at == latest_exec_subquery.c.latest_created_at), - ) - .group_by(Provider.slug, Diagnostic.slug) - .order_by(Provider.slug, Diagnostic.slug) - ) + stats_rows = reader.executions.statistics(diagnostic_contains=diagnostic, provider_contains=provider) - # Apply diagnostic filter - if diagnostic: - diagnostic_conditions = [ - Diagnostic.slug.ilike(f"%{filter_value.lower()}%") for filter_value in diagnostic - ] - query = query.filter(or_(*diagnostic_conditions)) - - # Apply provider filter - if provider: - provider_conditions = [Provider.slug.ilike(f"%{filter_value.lower()}%") for filter_value in provider] - query = query.filter(or_(*provider_conditions)) - - results = query.all() - - if not results: + if not stats_rows: console.print("No execution groups found.") return @@ -963,7 +885,7 @@ def stats( "dirty": row.dirty, "total": row.total, } - for row in results + for row in stats_rows ] results_df = pd.DataFrame(rows) @@ -979,6 +901,11 @@ def stats( pretty_print_df(results_df, console=console) +# `get_executions_for_reingest` stays on the ORM helper by design. +# it passes`include_superseded=True` and selects `eg.executions[0]` +# (the oldest / original execution in the group), +# which is a different "latest" than `reader.executions` definition. +# Revisit once a mutable query surface exists. @app.command() def reingest( # noqa: PLR0913 ctx: typer.Context, diff --git a/packages/climate-ref/src/climate_ref/results/__init__.py b/packages/climate-ref/src/climate_ref/results/__init__.py index d301258ef..98e26b5ea 100644 --- a/packages/climate-ref/src/climate_ref/results/__init__.py +++ b/packages/climate-ref/src/climate_ref/results/__init__.py @@ -1,9 +1,9 @@ """ Data access layer for REF results. -This package is the single source of truth for how result data is filtered and queried. It serves -notebooks / local users (via pandas-friendly, detached value objects) and, in future, the ref-app -API (which can reuse the shared ``Select`` builders directly). +This package is the single source of truth for how result data is filtered and queried. +It serves notebooks / local users (via pandas-friendly, detached value objects) and, +in future, the ref-app API (which can reuse the shared ``Select`` builders directly). Layers: @@ -13,6 +13,8 @@ * [frames][climate_ref.results.frames] -- ``*_to_frame`` + ``collect_facets``. * [outliers][climate_ref.results.outliers] -- source-id-aware IQR outlier detection. * [values][climate_ref.results.values] -- typed DTOs + collections + the ``Reader`` facade. +* [executions][climate_ref.results.executions] -- execution-group / execution DTOs + collection + + the ``ExecutionsReader`` facade. The typical notebook entry point:: @@ -26,26 +28,29 @@ Public surface convention ------------------------- -The top level exports only what a caller *names to make a call*, so the namespace stays small as -domains are added: +The top level exports only what a caller *names to make a call*, +so the namespace stays small as domains are added: * the ``Reader`` entry point, -* filter objects you construct and pass in (``MetricValueFilter``, and future - ``ExecutionGroupFilter`` / ``DatasetFilter``), +* filter objects you construct and pass in (``MetricValueFilter``, ``ExecutionGroupFilter``, + and the future ``DatasetFilter``), * value objects you pass in (``OutlierPolicy``, and the future ``ResultPaths``). Everything the package *returns* -- DTOs, collections, views -- and the sub-reader classes reached via ``reader.values`` etc. live in their domain submodule (``climate_ref.results.values``, ...). Import them from there on the rare occasion you need to name one, e.g. -``from climate_ref.results.values import ScalarValue``. The ``Select`` builders and other plumbing -stay in ``climate_ref.results._query`` and are not part of the public surface. +``from climate_ref.results.values import ScalarValue``. +The ``Select`` builders and other plumbing stay in ``climate_ref.results._query`` +and are not part of the public surface. """ from climate_ref.results._query import MetricValueFilter +from climate_ref.results.executions import ExecutionGroupFilter from climate_ref.results.outliers import OutlierPolicy from climate_ref.results.values import Reader __all__ = [ + "ExecutionGroupFilter", "MetricValueFilter", "OutlierPolicy", "Reader", diff --git a/packages/climate-ref/src/climate_ref/results/executions.py b/packages/climate-ref/src/climate_ref/results/executions.py new file mode 100644 index 000000000..00d2f4fcc --- /dev/null +++ b/packages/climate-ref/src/climate_ref/results/executions.py @@ -0,0 +1,379 @@ +""" +Typed, ORM-free surface for execution-group and execution results. + +[ExecutionsReader][climate_ref.results.executions.ExecutionsReader] is reached via +[Reader.executions][climate_ref.results.values.Reader.executions]. +It wraps the sanctioned group+latest query +([get_execution_group_and_latest_filtered][climate_ref.models.execution.get_execution_group_and_latest_filtered]) +and the per-(provider, diagnostic) status aggregate (formerly inlined in ``cli/executions.py::stats``), +mapping ORM rows to frozen, detached DTOs that outlive the originating session. + +Two write/recovery paths deliberately stay on the ORM helper instead of this reader: + +- ``cli/executions.py::delete_groups`` + (needs live ORM objects to cascade-delete related rows and remove output directories) +- ``executor/reingest.py::get_executions_for_reingest`` + (needs ``include_superseded=True`` plus the *oldest* execution in a group, + not this reader's "latest" definition). +""" + +from collections.abc import Iterator, Mapping, Sequence +from typing import Any + +import attrs +import pandas as pd +from sqlalchemy import Select, case, func, or_, select +from sqlalchemy.orm import Session + +from climate_ref.database import Database +from climate_ref.models.diagnostic import Diagnostic +from climate_ref.models.execution import Execution, ExecutionGroup, get_execution_group_and_latest_filtered +from climate_ref.models.provider import Provider +from climate_ref.results._query import latest_execution_for_group + + +def _as_str_tuple(value: Sequence[str] | None) -> tuple[str, ...] | None: + """ + Coerce a multi-value filter field to ``tuple[str, ...]``, rejecting a bare ``str``. + + A bare ``str`` is itself a ``Sequence[str]``, so without this guard a caller passing + ``diagnostic_contains="enso"`` would have it iterated character-by-character + (``"enso"`` -> ``e``, ``n``, ``s``, ``o``) before it ever reaches ``ilike`` / + the facet matcher. + """ + if value is None: + return None + if isinstance(value, str): + raise TypeError("Expected a sequence of strings, not a bare str. Wrap it in a list/tuple.") + return tuple(value) + + +def _as_facets( + value: Mapping[str, Sequence[str]] | None, +) -> Mapping[str, tuple[str, ...]] | None: + """Copy a facets mapping into an immutable ``dict`` of immutable ``tuple`` values.""" + if value is None: + return None + return {k: tuple(v) for k, v in value.items()} + + +@attrs.frozen(kw_only=True) +class ExecutionGroupFilter: + """ + Declarative filter over execution groups. + + Every field is optional; ``None`` means "do not constrain on this axis". + This mirrors exactly what [get_execution_group_and_latest_filtered] + [climate_ref.models.execution.get_execution_group_and_latest_filtered] supports today -- + ``diagnostic_contains``/``provider_contains`` are case-insensitive substring matches (OR-combined), + ``facets`` is the selector-facet map consumed by the helper's Python-side matching. + + Exact-match ``diagnostic_slug``/``provider_slug`` are a documented follow-up, + as the underlying helper only does substring matching today, + so adding exact match means extending the helper first. + """ + + diagnostic_contains: tuple[str, ...] | None = attrs.field(default=None, converter=_as_str_tuple) + provider_contains: tuple[str, ...] | None = attrs.field(default=None, converter=_as_str_tuple) + dirty: bool | None = None + successful: bool | None = None + facets: Mapping[str, tuple[str, ...]] | None = attrs.field(default=None, converter=_as_facets) + include_superseded: bool = False + + +@attrs.frozen(kw_only=True) +class ExecutionView: + """A single execution, detached from the ORM.""" + + id: int + execution_group_id: int + successful: bool | None + dataset_hash: str + retracted: bool + output_fragment: str + path: str | None + provider_version: str | None + created_at: Any + updated_at: Any + + +@attrs.frozen(kw_only=True) +class ExecutionGroupView: + """An execution group, detached from the ORM, with its per-group latest execution (if any).""" + + id: int + key: str + diagnostic_slug: str + provider_slug: str + dirty: bool + diagnostic_version: int + selectors: Mapping[str, Any] + created_at: Any + updated_at: Any + # Reflects the helper's `max(created_at)` outer join + # (models/execution.py::get_execution_group_and_latest), which can duplicate a group on an + # exact `created_at` tie. This may differ from `ExecutionsReader.latest_execution()`, which + # tie-breaks by `created_at DESC, id DESC` (`_query.latest_execution_for_group`). Both + # behaviours are intentional for their respective consumers (`groups()` vs `latest_execution()`) + # -- do not unify them. + latest: ExecutionView | None + + @property + def successful(self) -> bool | None: + """Convenience mirror of ``latest.successful`` (matches the CLI's ``successful`` column).""" + return self.latest.successful if self.latest else None + + +@attrs.frozen(kw_only=True) +class ExecutionStats: + """Per-(provider, diagnostic) execution status counts, detached from the ORM.""" + + provider: str + diagnostic: str + running: int + failed: int + successful: int + not_started: int + dirty: int + total: int + + +@attrs.frozen(kw_only=True) +class ExecutionGroupCollection: + """An immutable page of execution groups plus collection-level metadata.""" + + items: tuple[ExecutionGroupView, ...] + total_count: int + offset: int + limit: int | None + + def __iter__(self) -> Iterator[ExecutionGroupView]: + return iter(self.items) + + def __len__(self) -> int: + return len(self.items) + + def to_pandas(self) -> pd.DataFrame: + """ + DataFrame mirroring the ``list-groups`` CLI columns. + + Columns are emitted explicitly (``id, key, provider, diagnostic, dirty, successful, + created_at, updated_at, selectors``) even when the collection is empty, so callers can + select columns / build an empty table without special-casing. + """ + columns = [ + "id", + "key", + "provider", + "diagnostic", + "dirty", + "successful", + "created_at", + "updated_at", + "selectors", + ] + records = [ + { + "id": eg.id, + "key": eg.key, + "provider": eg.provider_slug, + "diagnostic": eg.diagnostic_slug, + "dirty": eg.dirty, + "successful": eg.successful, + "created_at": eg.created_at, + "updated_at": eg.updated_at, + "selectors": dict(eg.selectors), + } + for eg in self.items + ] + return pd.DataFrame.from_records(records, columns=columns) + + +def select_execution_statistics( + *, + diagnostic_contains: Sequence[str] | None = None, + provider_contains: Sequence[str] | None = None, +) -> Select[Any]: + """ + Build the per-(provider, diagnostic) status-count aggregate ``Select``. + + Moved out of ``cli/executions.py::stats`` so the aggregate is reusable plumbing. + Only ``diagnostic_contains``/``provider_contains`` are accepted + (not the full [ExecutionGroupFilter][climate_ref.results.executions.ExecutionGroupFilter]) + so ``dirty`` / ``successful`` / ``facets`` are never silently ignored + on an aggregate that cannot honour them. + + Status definitions (unchanged from the CLI): + + * ``running`` -- latest execution exists and ``successful IS NULL``. + * ``failed`` -- latest execution exists and ``successful IS False``. + * ``successful`` -- latest execution exists and ``successful IS True``. + * ``not_started`` -- no execution exists for the group. + * ``dirty`` -- the group's ``dirty`` flag is set. + + Only execution groups whose ``diagnostic_version`` matches the diagnostic's + ``promoted_version`` are counted (same promoted-version gate as + [get_execution_group_and_latest_filtered][climate_ref.models.execution.get_execution_group_and_latest_filtered]). + """ + latest_exec_subquery = ( + select( + Execution.execution_group_id, + func.max(Execution.created_at).label("latest_created_at"), + ) + .group_by(Execution.execution_group_id) + .subquery() + ) + + stmt = ( + select( + Provider.slug.label("provider"), + Diagnostic.slug.label("diagnostic"), + func.count().label("total"), + func.sum( + case( + (Execution.successful.is_(None) & Execution.id.isnot(None), 1), + else_=0, + ) + ).label("running"), + func.sum(case((Execution.successful.is_(False), 1), else_=0)).label("failed"), + func.sum(case((Execution.successful.is_(True), 1), else_=0)).label("successful"), + func.sum(case((Execution.id.is_(None), 1), else_=0)).label("not_started"), + func.sum(case((ExecutionGroup.dirty.is_(True), 1), else_=0)).label("dirty"), + ) + .join(Diagnostic, ExecutionGroup.diagnostic_id == Diagnostic.id) + .where(ExecutionGroup.diagnostic_version == Diagnostic.promoted_version) + .join(Provider, Diagnostic.provider_id == Provider.id) + .outerjoin(latest_exec_subquery, ExecutionGroup.id == latest_exec_subquery.c.execution_group_id) + .outerjoin( + Execution, + (Execution.execution_group_id == ExecutionGroup.id) + & (Execution.created_at == latest_exec_subquery.c.latest_created_at), + ) + .group_by(Provider.slug, Diagnostic.slug) + .order_by(Provider.slug, Diagnostic.slug) + ) + + if diagnostic_contains: + stmt = stmt.where(or_(*(Diagnostic.slug.ilike(f"%{s.lower()}%") for s in diagnostic_contains))) + if provider_contains: + stmt = stmt.where(or_(*(Provider.slug.ilike(f"%{s.lower()}%") for s in provider_contains))) + + return stmt + + +class ExecutionsReader: + """ + Execution-group and execution read domain. + + Constructed from a [Database][climate_ref.database.Database], + which owns the session and the read-only story. + All read methods return detached DTOs that outlive the session. + """ + + def __init__(self, database: Database) -> None: + self._db = database + + @property + def session(self) -> Session: + """The underlying database session.""" + return self._db.session + + def _to_execution_view(self, execution: Execution) -> ExecutionView: + return ExecutionView( + id=execution.id, + execution_group_id=execution.execution_group_id, + successful=execution.successful, + dataset_hash=execution.dataset_hash, + retracted=execution.retracted, + output_fragment=execution.output_fragment, + path=execution.path, + provider_version=execution.provider_version, + created_at=execution.created_at, + updated_at=execution.updated_at, + ) + + def _to_group_view(self, eg: ExecutionGroup, latest: Execution | None) -> ExecutionGroupView: + return ExecutionGroupView( + id=eg.id, + key=eg.key, + diagnostic_slug=eg.diagnostic.slug, + provider_slug=eg.diagnostic.provider.slug, + dirty=eg.dirty, + diagnostic_version=eg.diagnostic_version, + selectors=dict(eg.selectors), + created_at=eg.created_at, + updated_at=eg.updated_at, + latest=self._to_execution_view(latest) if latest is not None else None, + ) + + def groups( + self, + filters: ExecutionGroupFilter | None = None, + *, + offset: int = 0, + limit: int | None = None, + ) -> ExecutionGroupCollection: + """ + Query execution groups with their per-group latest execution. + + Wraps [get_execution_group_and_latest_filtered] + [climate_ref.models.execution.get_execution_group_and_latest_filtered] exactly: the full + filtered list is materialised, ``total_count`` is its length, and the page is a Python + slice (``all[offset:offset + limit]``). This preserves today's ``list-groups`` pagination + and ordering exactly -- no SQL-level pagination or additional ``order_by`` is introduced + here. + """ + filters = filters or ExecutionGroupFilter() + + all_results = get_execution_group_and_latest_filtered( + self.session, + diagnostic_filters=list(filters.diagnostic_contains) if filters.diagnostic_contains else None, + provider_filters=list(filters.provider_contains) if filters.provider_contains else None, + facet_filters={k: list(v) for k, v in filters.facets.items()} if filters.facets else None, + dirty=filters.dirty, + successful=filters.successful, + include_superseded=filters.include_superseded, + ) + + total_count = len(all_results) + page = all_results[offset : offset + limit] if limit is not None else all_results[offset:] + items = tuple(self._to_group_view(eg, latest) for eg, latest in page) + + return ExecutionGroupCollection(items=items, total_count=total_count, offset=offset, limit=limit) + + def statistics( + self, + *, + diagnostic_contains: Sequence[str] | None = None, + provider_contains: Sequence[str] | None = None, + ) -> tuple[ExecutionStats, ...]: + """Execute the ``select_execution_statistics`` builder and map rows to ``ExecutionStats``.""" + stmt = select_execution_statistics( + diagnostic_contains=diagnostic_contains, provider_contains=provider_contains + ) + rows = self.session.execute(stmt).all() + return tuple( + ExecutionStats( + provider=row.provider, + diagnostic=row.diagnostic, + running=row.running, + failed=row.failed, + successful=row.successful, + not_started=row.not_started, + dirty=row.dirty, + total=row.total, + ) + for row in rows + ) + + def latest_execution(self, execution_group_id: int) -> ExecutionView | None: + """ + Return the most recent execution for a group. + + Wraps [latest_execution_for_group][climate_ref.results._query.latest_execution_for_group], + which tie-breaks by ``created_at DESC, id DESC``. See the note on + [ExecutionGroupView.latest][climate_ref.results.executions.ExecutionGroupView] for how this + differs from the per-group latest returned by ``groups()`` on an exact timestamp tie. + """ + execution = latest_execution_for_group(self.session, execution_group_id) + return self._to_execution_view(execution) if execution is not None else None diff --git a/packages/climate-ref/src/climate_ref/results/values.py b/packages/climate-ref/src/climate_ref/results/values.py index 9d9a17c12..73126efb5 100644 --- a/packages/climate-ref/src/climate_ref/results/values.py +++ b/packages/climate-ref/src/climate_ref/results/values.py @@ -3,14 +3,15 @@ [Reader][climate_ref.results.values.Reader] is the intent-named entry point that notebooks and (eventually) the API use, via its [values][climate_ref.results.values.Reader.values] -sub-reader. [ValuesReader][climate_ref.results.values.ValuesReader]'s read methods return frozen, -detached value objects wrapped in collections that offer ``to_pandas()``. The objects are fully -materialised, so they remain valid after the originating session closes -- a notebook can build a -DataFrame inside a ``with Database(...)`` block and keep using it afterwards. - -Everything here is a thin layer over [climate_ref.results._query][] (the shared ``Select`` -builders) and [climate_ref.results.outliers][]; power users who need the raw ``Select`` or ORM -rows use those modules directly. +sub-reader. +[ValuesReader][climate_ref.results.values.ValuesReader]'s read methods return frozen, +detached value objects wrapped in collections that offer ``to_pandas()``. +The objects are fully materialised, so they remain valid after the originating session closes. +A notebook can build a DataFrame inside a ``with Database(...)`` block and keep using it afterwards. + +Everything here is a thin layer over [climate_ref.results._query][] +(the shared ``Select`` builders) and [climate_ref.results.outliers][]. +power users who need the raw ``Select`` or ORM rows use those modules directly. """ import functools @@ -31,6 +32,7 @@ select_scalar_values, select_series_values, ) +from climate_ref.results.executions import ExecutionsReader from climate_ref.results.frames import collect_facets from climate_ref.results.outliers import OutlierPolicy, detect_scalar_outliers @@ -394,7 +396,8 @@ class Reader: Constructed from a [Database][climate_ref.database.Database], which owns the session and the read-only story. This is a thin entry point: it exposes per-domain sub-readers as properties, - currently just [values][climate_ref.results.values.Reader.values] for metric-value reads. + [values][climate_ref.results.values.Reader.values] for metric-value reads and + [executions][climate_ref.results.values.Reader.executions] for execution-group reads. """ def __init__(self, database: Database) -> None: @@ -409,3 +412,8 @@ def session(self) -> Session: def values(self) -> ValuesReader: """Metric-value reads (scalar/series/facets).""" return ValuesReader(self._db) + + @functools.cached_property + def executions(self) -> ExecutionsReader: + """Execution-group and execution reads.""" + return ExecutionsReader(self._db) diff --git a/packages/climate-ref/tests/unit/results/test_executions.py b/packages/climate-ref/tests/unit/results/test_executions.py new file mode 100644 index 000000000..916e68135 --- /dev/null +++ b/packages/climate-ref/tests/unit/results/test_executions.py @@ -0,0 +1,406 @@ +"""Unit tests for the execution-group / execution read layer.""" + +import datetime + +import pytest +from climate_ref_esmvaltool import provider as esmvaltool_provider +from climate_ref_pmp import provider as pmp_provider + +from climate_ref.models import Execution, ExecutionGroup +from climate_ref.models.diagnostic import Diagnostic +from climate_ref.provider_registry import _register_provider +from climate_ref.results import ExecutionGroupFilter, Reader +from climate_ref.results.executions import select_execution_statistics + + +def test_import_smoke() -> None: + """`ExecutionGroupFilter` and `Reader` import cleanly from the top-level package.""" + assert ExecutionGroupFilter is not None + assert Reader is not None + + +@pytest.fixture +def db_with_groups(db_seeded): + """A seeded DB with execution groups spanning two providers, dirty/successful/facet variety.""" + with db_seeded.session.begin(): + _register_provider(db_seeded, pmp_provider) + _register_provider(db_seeded, esmvaltool_provider) + + # Diagnostic 1, Provider 1 (pmp), Facets: source_id=GFDL-ESM4, variable_id=tas + diag_1 = db_seeded.session.query(Diagnostic).filter_by(slug="enso_tel").first() + eg1 = ExecutionGroup( + key="key1", + diagnostic_id=diag_1.id, + selectors={"cmip6": [["source_id", "GFDL-ESM4"], ["variable_id", "tas"]]}, + ) + db_seeded.session.add(eg1) + + # Diagnostic 2, Provider 1 (pmp), Facets: source_id=ACCESS-ESM1-5, variable_id=pr + diag_2 = ( + db_seeded.session.query(Diagnostic) + .filter_by(slug="extratropical-modes-of-variability-nao") + .first() + ) + eg2 = ExecutionGroup( + key="key2", + diagnostic_id=diag_2.id, + selectors={"cmip6": [["source_id", "ACCESS-ESM1-5"], ["variable_id", "pr"]]}, + ) + db_seeded.session.add(eg2) + + # Diagnostic 3, Provider 2 (esmvaltool), Facets: source_id=CNRM-CM6-1, variable_id=tas + diag_3 = db_seeded.session.query(Diagnostic).filter_by(slug="enso-characteristics").first() + eg3 = ExecutionGroup( + key="key3", + diagnostic_id=diag_3.id, + selectors={"cmip6": [["source_id", "CNRM-CM6-1"], ["variable_id", "tas"]]}, + ) + db_seeded.session.add(eg3) + + # Diagnostic 4, Provider 2 (esmvaltool) + diag_4 = db_seeded.session.query(Diagnostic).filter_by(slug="sea-ice-area-basic").first() + eg4 = ExecutionGroup( + key="key4", diagnostic_id=diag_4.id, selectors={"cmip6": [["experiment_id", "historical"]]} + ) + db_seeded.session.add(eg4) + + db_seeded.session.flush() + db_seeded.session.add( + Execution( + execution_group_id=eg1.id, successful=True, output_fragment="out1", dataset_hash="hash1" + ) + ) + db_seeded.session.add( + Execution( + execution_group_id=eg2.id, successful=True, output_fragment="out2", dataset_hash="hash2" + ) + ) + db_seeded.session.add( + Execution( + execution_group_id=eg3.id, successful=False, output_fragment="out3", dataset_hash="hash3" + ) + ) + db_seeded.session.add( + Execution( + execution_group_id=eg4.id, successful=True, output_fragment="out4", dataset_hash="hash4" + ) + ) + + # A dirty execution group (successful=True latest, but flagged dirty) + eg5 = ExecutionGroup( + key="key5", + diagnostic_id=diag_4.id, + selectors={"cmip6": [["experiment_id", "historical"]]}, + dirty=True, + ) + db_seeded.session.add(eg5) + db_seeded.session.flush() + db_seeded.session.add( + Execution( + execution_group_id=eg5.id, successful=True, output_fragment="out5", dataset_hash="hash5" + ) + ) + + # An execution group with no executions at all (not-started) + eg6 = ExecutionGroup( + key="key6", diagnostic_id=diag_4.id, selectors={"cmip6": [["experiment_id", "ssp126"]]} + ) + db_seeded.session.add(eg6) + + # A running execution group (successful=None) + eg7 = ExecutionGroup( + key="key7", diagnostic_id=diag_4.id, selectors={"cmip6": [["experiment_id", "ssp245"]]} + ) + db_seeded.session.add(eg7) + db_seeded.session.flush() + db_seeded.session.add( + Execution( + execution_group_id=eg7.id, successful=None, output_fragment="out7", dataset_hash="hash7" + ) + ) + db_seeded.session.commit() + + ids = { + "key1": eg1.id, + "key2": eg2.id, + "key3": eg3.id, + "key4": eg4.id, + "key5": eg5.id, + "key6": eg6.id, + "key7": eg7.id, + } + db_seeded.group_ids = ids + return db_seeded + + +class TestGroupsFilter: + def test_diagnostic_contains(self, db_with_groups): + reader = Reader(db_with_groups) + coll = reader.executions.groups(ExecutionGroupFilter(diagnostic_contains=["enso"])) + keys = {g.key for g in coll} + assert "key1" in keys # enso_tel + assert "key3" in keys # enso-characteristics + assert "key2" not in keys + assert "key4" not in keys + + def test_provider_contains(self, db_with_groups): + reader = Reader(db_with_groups) + coll = reader.executions.groups(ExecutionGroupFilter(provider_contains=["pmp"])) + keys = {g.key for g in coll} + assert "key1" in keys + assert "key2" in keys + assert "key3" not in keys + + def test_dirty(self, db_with_groups): + reader = Reader(db_with_groups) + coll = reader.executions.groups(ExecutionGroupFilter(dirty=True)) + keys = {g.key for g in coll} + assert keys == {"key5"} + + coll = reader.executions.groups(ExecutionGroupFilter(dirty=False)) + keys = {g.key for g in coll} + assert "key5" not in keys + + def test_successful_true_only_true(self, db_with_groups): + reader = Reader(db_with_groups) + coll = reader.executions.groups(ExecutionGroupFilter(successful=True)) + keys = {g.key for g in coll} + # key1, key2, key4, key5 have successful=True latest executions + assert keys == {"key1", "key2", "key4", "key5"} + + def test_successful_false_includes_failed_running_and_no_exec(self, db_with_groups): + reader = Reader(db_with_groups) + coll = reader.executions.groups(ExecutionGroupFilter(successful=False)) + keys = {g.key for g in coll} + # key3 (failed), key6 (no exec), key7 (running) must all be included. + assert {"key3", "key6", "key7"} <= keys + # while truly successful groups must be excluded. + assert not ({"key1", "key2", "key4", "key5"} & keys) + + def test_facet_bare_key(self, db_with_groups): + reader = Reader(db_with_groups) + coll = reader.executions.groups(ExecutionGroupFilter(facets={"source_id": ["GFDL-ESM4"]})) + keys = {g.key for g in coll} + assert keys == {"key1"} + + def test_facet_dataset_type_scoped_key(self, db_with_groups): + reader = Reader(db_with_groups) + coll = reader.executions.groups(ExecutionGroupFilter(facets={"cmip6.source_id": ["GFDL-ESM4"]})) + keys = {g.key for g in coll} + assert keys == {"key1"} + + def test_bare_string_rejected_diagnostic_contains(self): + with pytest.raises(TypeError): + ExecutionGroupFilter(diagnostic_contains="enso") + + def test_bare_string_rejected_provider_contains(self): + with pytest.raises(TypeError): + ExecutionGroupFilter(provider_contains="pmp") + + +class TestGroupsPromotedVersion: + def test_promoted_version_default_excludes_superseded(self, db_with_groups): + # Bump the diagnostic's promoted_version so eg1's diagnostic_version=1 becomes superseded. + session = db_with_groups.session + diag = session.query(ExecutionGroup).filter_by(key="key1").first().diagnostic + with session.begin_nested() if session.in_transaction() else session.begin(): + diag.promoted_version = 2 + + reader = Reader(db_with_groups) + coll = reader.executions.groups(ExecutionGroupFilter(diagnostic_contains=["enso_tel"])) + assert coll.total_count == 0 + + def test_include_superseded_true_sees_it(self, db_with_groups): + session = db_with_groups.session + diag = session.query(ExecutionGroup).filter_by(key="key1").first().diagnostic + with session.begin_nested() if session.in_transaction() else session.begin(): + diag.promoted_version = 2 + + reader = Reader(db_with_groups) + coll = reader.executions.groups( + ExecutionGroupFilter(diagnostic_contains=["enso_tel"], include_superseded=True) + ) + assert coll.total_count == 1 + + +class TestGroupsPagination: + def test_total_count_vs_page_with_offset_limit(self, db_with_groups): + reader = Reader(db_with_groups) + full = reader.executions.groups() + assert full.total_count == 7 + + page = reader.executions.groups(offset=2, limit=2) + assert page.total_count == 7 + assert len(page) == 2 + assert page.offset == 2 + assert page.limit == 2 + + +class TestGroupsToPandas: + def test_to_pandas_columns(self, db_with_groups): + reader = Reader(db_with_groups) + coll = reader.executions.groups() + df = coll.to_pandas() + expected_cols = [ + "id", + "key", + "provider", + "diagnostic", + "dirty", + "successful", + "created_at", + "updated_at", + "selectors", + ] + assert list(df.columns) == expected_cols + assert len(df) == 7 + + def test_to_pandas_columns_when_empty(self, db_with_groups): + reader = Reader(db_with_groups) + coll = reader.executions.groups(ExecutionGroupFilter(diagnostic_contains=["nonexistent"])) + df = coll.to_pandas() + expected_cols = [ + "id", + "key", + "provider", + "diagnostic", + "dirty", + "successful", + "created_at", + "updated_at", + "selectors", + ] + assert list(df.columns) == expected_cols + assert len(df) == 0 + + def test_detached_survival(self, db_with_groups): + reader = Reader(db_with_groups) + coll = reader.executions.groups() + db_with_groups.session.expunge_all() + df = coll.to_pandas() + assert len(df) == 7 + selectors = [item.selectors for item in coll] + assert all(isinstance(s, dict) for s in selectors) + # Ensure it's a plain dict, not the ORM-attached mapping. + assert type(next(s for s in selectors if s)) is dict + + +class TestGroupLatestMapping: + def test_latest_none_when_no_executions(self, db_with_groups): + reader = Reader(db_with_groups) + coll = reader.executions.groups(ExecutionGroupFilter(facets={"experiment_id": ["ssp126"]})) + assert len(coll) == 1 + assert coll.items[0].latest is None + assert coll.items[0].successful is None + + def test_latest_successful(self, db_with_groups): + reader = Reader(db_with_groups) + coll = reader.executions.groups(ExecutionGroupFilter(diagnostic_contains=["enso_tel"])) + assert len(coll) == 1 + view = coll.items[0] + assert view.latest is not None + assert view.latest.successful is True + assert view.successful is True + + def test_latest_failed(self, db_with_groups): + reader = Reader(db_with_groups) + coll = reader.executions.groups(ExecutionGroupFilter(diagnostic_contains=["enso-characteristics"])) + assert len(coll) == 1 + view = coll.items[0] + assert view.latest is not None + assert view.latest.successful is False + + def test_latest_running(self, db_with_groups): + reader = Reader(db_with_groups) + coll = reader.executions.groups(ExecutionGroupFilter(facets={"experiment_id": ["ssp245"]})) + assert len(coll) == 1 + view = coll.items[0] + assert view.latest is not None + assert view.latest.successful is None + assert view.successful is None + + +class TestSelectExecutionStatistics: + def test_raw_rows(self, db_with_groups): + stmt = select_execution_statistics() + rows = db_with_groups.session.execute(stmt).all() + by_diag = {row.diagnostic: row for row in rows} + assert "enso_tel" in by_diag + row = by_diag["enso_tel"] + assert row.total == 1 + assert row.successful == 1 + assert row.failed == 0 + assert row.running == 0 + assert row.not_started == 0 + + def test_raw_rows_provider_filter(self, db_with_groups): + stmt = select_execution_statistics(provider_contains=["pmp"]) + rows = db_with_groups.session.execute(stmt).all() + providers = {row.provider for row in rows} + assert providers == {"pmp"} + + def test_raw_rows_diagnostic_filter(self, db_with_groups): + stmt = select_execution_statistics(diagnostic_contains=["enso"]) + rows = db_with_groups.session.execute(stmt).all() + diagnostics = {row.diagnostic for row in rows} + assert diagnostics == {"enso_tel", "enso-characteristics"} + + +class TestStatistics: + def test_status_counts(self, db_with_groups): + reader = Reader(db_with_groups) + stats = reader.executions.statistics() + by_diag = {s.diagnostic: s for s in stats} + + # sea-ice-area-basic has key4 (successful), key5 (successful, dirty), + # key6 (not started), key7 (running). + sea_ice = by_diag["sea-ice-area-basic"] + assert sea_ice.total == 4 + assert sea_ice.successful == 2 + assert sea_ice.not_started == 1 + assert sea_ice.running == 1 + assert sea_ice.dirty == 1 + assert sea_ice.failed == 0 + + def test_provider_substring_filter(self, db_with_groups): + reader = Reader(db_with_groups) + stats = reader.executions.statistics(provider_contains=["esmvaltool"]) + providers = {s.provider for s in stats} + assert providers == {"esmvaltool"} + + def test_diagnostic_substring_filter(self, db_with_groups): + reader = Reader(db_with_groups) + stats = reader.executions.statistics(diagnostic_contains=["enso"]) + diagnostics = {s.diagnostic for s in stats} + assert diagnostics == {"enso_tel", "enso-characteristics"} + + +class TestLatestExecution: + def test_returns_latest_by_created_at(self, db_with_groups): + session = db_with_groups.session + reader = Reader(db_with_groups) + group_id = db_with_groups.group_ids["key1"] + + older_id = session.query(Execution).filter_by(execution_group_id=group_id).one().id + + with session.begin_nested() if session.in_transaction() else session.begin(): + newer = Execution( + execution_group_id=group_id, + output_fragment="frag-newer", + dataset_hash="hash-newer", + successful=True, + created_at=datetime.datetime.now(tz=datetime.UTC) + datetime.timedelta(hours=1), + ) + session.add(newer) + session.flush() + newer_id = newer.id + + latest = reader.executions.latest_execution(group_id) + assert latest is not None + assert latest.id == newer_id + assert latest.id != older_id + + def test_none_for_empty_group(self, db_with_groups): + reader = Reader(db_with_groups) + group_id = db_with_groups.group_ids["key6"] + assert reader.executions.latest_execution(group_id) is None From 34137c8c4b64fcf9ea649dead79597a59c7acb2c Mon Sep 17 00:00:00 2001 From: Jared Lewis Date: Thu, 2 Jul 2026 08:59:46 +1000 Subject: [PATCH 04/23] feat(results): add reader.artifacts for output path resolution Adds the files/outputs domain to the climate_ref.results read layer (RFC domain D). ArtifactsReader resolves execution output fragments (output_fragment, Execution.path, ExecutionOutput.filename) into filesystem paths, guarded against path escape via os.path.commonpath containment checks. ExecutionsReader gains outputs(execution_id) to read registered ExecutionOutput rows as detached OutputView DTOs. Reader takes an optional results root directly (Reader(database, results=...)) rather than through a wrapper value object -- a single Path field doesn't need one, and the decoupling from Config comes from not taking Config, not from wrapping the path. ref executions inspect now shows registered outputs and resolves its output directory through reader.artifacts instead of joining config.paths.results by hand. --- .../src/climate_ref/cli/executions.py | 51 +++++++- .../src/climate_ref/results/__init__.py | 4 +- .../src/climate_ref/results/artifacts.py | 77 +++++++++++ .../src/climate_ref/results/executions.py | 57 ++++++++- .../src/climate_ref/results/values.py | 31 ++++- .../tests/unit/cli/test_executions.py | 119 ++++++++++++++++- .../unit/cli/test_executions/test_inspect.txt | 3 + .../tests/unit/results/test_artifacts.py | 73 +++++++++++ .../tests/unit/results/test_executions.py | 121 +++++++++++++++++- 9 files changed, 516 insertions(+), 20 deletions(-) create mode 100644 packages/climate-ref/src/climate_ref/results/artifacts.py create mode 100644 packages/climate-ref/tests/unit/results/test_artifacts.py diff --git a/packages/climate-ref/src/climate_ref/cli/executions.py b/packages/climate-ref/src/climate_ref/cli/executions.py index b9542c4bd..5f85bd3a4 100644 --- a/packages/climate-ref/src/climate_ref/cli/executions.py +++ b/packages/climate-ref/src/climate_ref/cli/executions.py @@ -15,6 +15,7 @@ from rich.filesize import decimal from rich.markup import escape from rich.panel import Panel +from rich.table import Table from rich.text import Text from rich.tree import Tree from sqlalchemy import or_ @@ -29,7 +30,10 @@ from climate_ref.config import Config from climate_ref.models import Execution, ExecutionGroup from climate_ref.models.diagnostic import Diagnostic -from climate_ref.models.execution import execution_datasets, get_execution_group_and_latest_filtered +from climate_ref.models.execution import ( + execution_datasets, + get_execution_group_and_latest_filtered, +) from climate_ref.models.provider import Provider from climate_ref.results import ( ExecutionGroupFilter, @@ -37,6 +41,7 @@ OutlierPolicy, Reader, ) +from climate_ref.results.executions import OutputView from climate_ref.results.values import ValuesReader from climate_ref_core.logging import EXECUTION_LOG_FILENAME @@ -122,7 +127,10 @@ def list_groups( # noqa: PLR0913 ] = None, output_format: Annotated[ OutputFormat, - typer.Option("--format", help="Output format: 'table' (default) or machine-readable 'json'."), + typer.Option( + "--format", + help="Output format: 'table' (default) or machine-readable 'json'.", + ), ] = OutputFormat.table, ) -> None: """ @@ -246,7 +254,9 @@ def delete_groups( # noqa: PLR0912, PLR0913, PLR0915 ), ] = None, remove_outputs: bool = typer.Option( - False, "--remove-outputs", help="Also remove output directories from the filesystem" + False, + "--remove-outputs", + help="Also remove output directories from the filesystem", ), force: bool = typer.Option(False, help="Skip confirmation prompt"), ) -> None: @@ -437,7 +447,11 @@ def _datasets_panel(result: Execution) -> Panel: datasets_df = pd.DataFrame( [ - {"id": dataset.id, "slug": dataset.slug, "dataset_type": dataset.dataset_type} + { + "id": dataset.id, + "slug": dataset.slug, + "dataset_type": dataset.dataset_type, + } for dataset in datasets ] ) @@ -448,6 +462,27 @@ def _datasets_panel(result: Execution) -> Panel: ) +def _outputs_panel(outputs: tuple[OutputView, ...], output_fragment: str, reader: Reader) -> Panel: + if not outputs: + return Panel(Text("No registered outputs.", "bold red"), title="Outputs") + + table = Table("output_type", "short_name", "long_name", "filename") + for out in outputs: + if out.filename is None: + filename_cell: Text | str = "" + else: + output_path = reader.artifacts.output_file(output_fragment, out.filename) + filename_cell = Text(out.filename, style=f"link file://{output_path}") + table.add_row( + out.output_type, + out.short_name or "", + out.long_name or "", + filename_cell, + ) + + return Panel(table, title="Outputs") + + def _results_directory_panel(result_directory: pathlib.Path) -> Panel: if result_directory.exists(): tree = Tree( @@ -525,6 +560,7 @@ def inspect(ctx: typer.Context, execution_id: int) -> None: config: Config = ctx.obj.config session = ctx.obj.database.session console = ctx.obj.console + reader = Reader(ctx.obj.database, results=config.paths.results) execution_group = session.get(ExecutionGroup, execution_id) @@ -539,11 +575,12 @@ def inspect(ctx: typer.Context, execution_id: int) -> None: return result: Execution = execution_group.executions[-1] - result_directory = config.paths.results / result.output_fragment + output_dir = reader.artifacts.output_directory(result.output_fragment) console.print(_datasets_panel(result)) - console.print(_results_directory_panel(result_directory)) - console.print(_log_panel(result_directory)) + console.print(_outputs_panel(reader.executions.outputs(result.id), result.output_fragment, reader)) + console.print(_results_directory_panel(output_dir)) + console.print(_log_panel(output_dir)) @app.command() diff --git a/packages/climate-ref/src/climate_ref/results/__init__.py b/packages/climate-ref/src/climate_ref/results/__init__.py index 98e26b5ea..faa409fb0 100644 --- a/packages/climate-ref/src/climate_ref/results/__init__.py +++ b/packages/climate-ref/src/climate_ref/results/__init__.py @@ -15,6 +15,8 @@ * [values][climate_ref.results.values] -- typed DTOs + collections + the ``Reader`` facade. * [executions][climate_ref.results.executions] -- execution-group / execution DTOs + collection + the ``ExecutionsReader`` facade. +* [artifacts][climate_ref.results.artifacts] -- the ``ArtifactsReader`` facade, + resolving execution output fragments into filesystem paths under a results root. The typical notebook entry point:: @@ -34,7 +36,7 @@ * the ``Reader`` entry point, * filter objects you construct and pass in (``MetricValueFilter``, ``ExecutionGroupFilter``, and the future ``DatasetFilter``), -* value objects you pass in (``OutlierPolicy``, and the future ``ResultPaths``). +* value objects you pass in (``OutlierPolicy``). Everything the package *returns* -- DTOs, collections, views -- and the sub-reader classes reached via ``reader.values`` etc. live in their domain submodule (``climate_ref.results.values``, ...). diff --git a/packages/climate-ref/src/climate_ref/results/artifacts.py b/packages/climate-ref/src/climate_ref/results/artifacts.py new file mode 100644 index 000000000..ac853252d --- /dev/null +++ b/packages/climate-ref/src/climate_ref/results/artifacts.py @@ -0,0 +1,77 @@ +""" +Path resolution for execution output artifacts. + +[ArtifactsReader][climate_ref.results.artifacts.ArtifactsReader] is reached via +[Reader.artifacts][climate_ref.results.values.Reader.artifacts]. +It resolves the primitive fragments an execution already stores +(``output_fragment``, ``Execution.path``, ``ExecutionOutput.filename``) into filesystem +[Path][pathlib.Path]s under a results root directory. + +This module is deliberately narrow: no DB, no [Config][climate_ref.config.Config], and no +coupling to the ``executions`` DTOs. +It depends on a single ``results`` root path -- not the whole ``Config`` -- passed straight in; +a value object would only earn its keep once a second root (e.g. an archive root) is needed. +Resolution is purely lexical over the results root; opening/streaming the resolved paths +stays with the consumer. +""" + +import os +from pathlib import Path + +from climate_ref_core.logging import EXECUTION_LOG_FILENAME + + +class ArtifactsReader: + """ + Resolves execution output fragments into filesystem paths. + + Constructed from a single ``results`` root directory. + Every method is containment-guarded: a resolved path that would escape the results root + raises ``ValueError`` rather than returning a path outside it. + """ + + def __init__(self, results: Path) -> None: + self._results = results + + def _within(self, base: str, *parts: str) -> Path: + """ + Join ``parts`` onto ``base`` and guard against escaping ``base``. + + Lexically normalises the joined path (no filesystem access, no symlink resolution) and + checks containment with ``os.path.commonpath`` -- a real containment primitive, not a + string-prefix check (which would wrongly accept a sibling directory like ``results2``). + """ + root = os.path.normpath(base) + candidate = os.path.normpath(os.path.join(root, *parts)) + if os.path.commonpath([root, candidate]) != root: + raise ValueError(f"Path {candidate!r} escapes {root!r}") + return Path(candidate) + + def _within_results(self, *parts: str) -> Path: + """Join ``parts`` onto the results root and guard against escaping it.""" + return self._within(str(self._results), *parts) + + def output_directory(self, output_fragment: str) -> Path: + """Resolve an execution's output directory from its ``output_fragment``.""" + return self._within_results(output_fragment) + + def log_file(self, output_fragment: str) -> Path: + """Resolve the execution log file within an execution's output directory.""" + output_dir = self.output_directory(output_fragment) + return self._within(str(output_dir), EXECUTION_LOG_FILENAME) + + def bundle(self, output_fragment: str, bundle_path: str | None) -> Path | None: + """ + Resolve an execution's output bundle (``Execution.path``), if one is set. + + Returns ``None`` when ``bundle_path`` is ``None`` (no bundle recorded). + """ + if bundle_path is None: + return None + output_dir = self.output_directory(output_fragment) + return self._within(str(output_dir), bundle_path) + + def output_file(self, output_fragment: str, filename: str) -> Path: + """Resolve a registered ``ExecutionOutput.filename`` within an execution's output directory.""" + output_dir = self.output_directory(output_fragment) + return self._within(str(output_dir), filename) diff --git a/packages/climate-ref/src/climate_ref/results/executions.py b/packages/climate-ref/src/climate_ref/results/executions.py index 00d2f4fcc..9df078b68 100644 --- a/packages/climate-ref/src/climate_ref/results/executions.py +++ b/packages/climate-ref/src/climate_ref/results/executions.py @@ -3,10 +3,10 @@ [ExecutionsReader][climate_ref.results.executions.ExecutionsReader] is reached via [Reader.executions][climate_ref.results.values.Reader.executions]. -It wraps the sanctioned group+latest query +It wraps the group+latest query ([get_execution_group_and_latest_filtered][climate_ref.models.execution.get_execution_group_and_latest_filtered]) -and the per-(provider, diagnostic) status aggregate (formerly inlined in ``cli/executions.py::stats``), -mapping ORM rows to frozen, detached DTOs that outlive the originating session. +and the per-(provider, diagnostic) status aggregate mapping ORM rows to frozen DTOs (Data Transfer Objects). +These DTOs are detached from the database so outlive the database session. Two write/recovery paths deliberately stay on the ORM helper instead of this reader: @@ -27,7 +27,12 @@ from climate_ref.database import Database from climate_ref.models.diagnostic import Diagnostic -from climate_ref.models.execution import Execution, ExecutionGroup, get_execution_group_and_latest_filtered +from climate_ref.models.execution import ( + Execution, + ExecutionGroup, + ExecutionOutput, + get_execution_group_and_latest_filtered, +) from climate_ref.models.provider import Provider from climate_ref.results._query import latest_execution_for_group @@ -138,6 +143,19 @@ class ExecutionStats: total: int +@attrs.frozen(kw_only=True) +class OutputView: + """A single registered execution output, detached from the ORM.""" + + execution_id: int + output_type: str + filename: str | None + short_name: str | None + long_name: str | None + description: str | None + dimensions: Mapping[str, str] + + @attrs.frozen(kw_only=True) class ExecutionGroupCollection: """An immutable page of execution groups plus collection-level metadata.""" @@ -261,6 +279,20 @@ def select_execution_statistics( return stmt +def select_execution_outputs(execution_id: int) -> Select[Any]: + """ + Build the ``Select`` for one execution's registered ``ExecutionOutput`` rows. + + Ordered by ``(output_type, id)`` for stable, grouped output -- there is no prior consumer + ordering to preserve, since ``inspect`` did not list these rows before. + """ + return ( + select(ExecutionOutput) + .where(ExecutionOutput.execution_id == execution_id) + .order_by(ExecutionOutput.output_type, ExecutionOutput.id) + ) + + class ExecutionsReader: """ Execution-group and execution read domain. @@ -377,3 +409,20 @@ def latest_execution(self, execution_group_id: int) -> ExecutionView | None: """ execution = latest_execution_for_group(self.session, execution_group_id) return self._to_execution_view(execution) if execution is not None else None + + def outputs(self, execution_id: int) -> tuple[OutputView, ...]: + """Execute the ``select_execution_outputs`` builder and map rows to ``OutputView``.""" + stmt = select_execution_outputs(execution_id) + rows = self.session.execute(stmt).scalars().all() + return tuple( + OutputView( + execution_id=row.execution_id, + output_type=row.output_type.value, + filename=row.filename, + short_name=row.short_name, + long_name=row.long_name, + description=row.description, + dimensions=dict(row.dimensions), + ) + for row in rows + ) diff --git a/packages/climate-ref/src/climate_ref/results/values.py b/packages/climate-ref/src/climate_ref/results/values.py index 73126efb5..b94a41dce 100644 --- a/packages/climate-ref/src/climate_ref/results/values.py +++ b/packages/climate-ref/src/climate_ref/results/values.py @@ -16,7 +16,8 @@ import functools from collections.abc import Iterator, Mapping, Sequence -from typing import Any +from pathlib import Path +from typing import TYPE_CHECKING, Any import attrs import pandas as pd @@ -36,6 +37,9 @@ from climate_ref.results.frames import collect_facets from climate_ref.results.outliers import OutlierPolicy, detect_scalar_outliers +if TYPE_CHECKING: + from climate_ref.results.artifacts import ArtifactsReader + def _kind_of(dimensions: Mapping[str, str]) -> str: """Resolve the model/reference ``kind`` from the CV dimensions (defaults to ``"model"``).""" @@ -396,12 +400,15 @@ class Reader: Constructed from a [Database][climate_ref.database.Database], which owns the session and the read-only story. This is a thin entry point: it exposes per-domain sub-readers as properties, - [values][climate_ref.results.values.Reader.values] for metric-value reads and - [executions][climate_ref.results.values.Reader.executions] for execution-group reads. + [values][climate_ref.results.values.Reader.values] for metric-value reads, + [executions][climate_ref.results.values.Reader.executions] for execution-group reads, and + [artifacts][climate_ref.results.values.Reader.artifacts] for output path resolution + (only available when a ``results`` root is supplied). """ - def __init__(self, database: Database) -> None: + def __init__(self, database: Database, results: Path | None = None) -> None: self._db = database + self._results = results @property def session(self) -> Session: @@ -417,3 +424,19 @@ def values(self) -> ValuesReader: def executions(self) -> ExecutionsReader: """Execution-group and execution reads.""" return ExecutionsReader(self._db) + + @functools.cached_property + def artifacts(self) -> "ArtifactsReader": + """ + Output path resolution. + + Raises ``ValueError`` when no ``results`` root was supplied to the constructor. + """ + if self._results is None: + raise ValueError( + "reader.artifacts requires a results root; construct " + "Reader(database, results=config.paths.results)." + ) + from climate_ref.results.artifacts import ArtifactsReader # noqa: PLC0415 + + return ArtifactsReader(self._results) diff --git a/packages/climate-ref/tests/unit/cli/test_executions.py b/packages/climate-ref/tests/unit/cli/test_executions.py index ae223e0a1..802db23a5 100644 --- a/packages/climate-ref/tests/unit/cli/test_executions.py +++ b/packages/climate-ref/tests/unit/cli/test_executions.py @@ -9,13 +9,15 @@ from climate_ref_pmp import provider as pmp_provider from rich.console import Console -from climate_ref.cli.executions import _results_directory_panel +from climate_ref.cli.executions import _outputs_panel, _results_directory_panel from climate_ref.models import Execution, ExecutionGroup from climate_ref.models.dataset import CMIP6Dataset from climate_ref.models.diagnostic import Diagnostic from climate_ref.models.execution import ExecutionOutput, ResultOutputType, execution_datasets from climate_ref.models.metric_value import ScalarMetricValue, SeriesIndex, SeriesMetricValue from climate_ref.provider_registry import _register_provider +from climate_ref.results import Reader +from climate_ref.results.executions import OutputView from climate_ref_core.datasets import SourceDatasetType @@ -857,6 +859,121 @@ def test_inspect_missing(self, invoke_cli): result = invoke_cli(["executions", "inspect", "999"], expected_exit_code=1) assert "Execution not found: 999" in result.stderr + def test_inspect_outputs_panel(self, sample_data_dir, db_seeded, invoke_cli, config): + # Real on-disk results root so the directory-tree / log panels keep their existing behaviour. + results_path = config.paths.results + results_path.mkdir(parents=True, exist_ok=True) + + execution_group = ExecutionGroup( + key="key1", + diagnostic_id=1, + created_at=datetime.datetime(2021, 1, 1), + updated_at=datetime.datetime(2021, 2, 1), + ) + with db_seeded.session.begin(): + db_seeded.session.add(execution_group) + db_seeded.session.flush() + + execution = Execution( + execution_group_id=execution_group.id, + successful=True, + output_fragment="output", + dataset_hash="hash", + ) + db_seeded.session.add(execution) + db_seeded.session.flush() + + output_dir = results_path / execution.output_fragment + output_dir.mkdir(parents=True, exist_ok=True) + (output_dir / "plot.png").write_text("fake plot") + + db_seeded.session.add( + ExecutionOutput( + execution_id=execution.id, + output_type=ResultOutputType.Plot, + filename="plot.png", + short_name="plot-short", + long_name="Plot long name", + ) + ) + # A metadata-only row with no filename must not emit a bogus file link/path. + db_seeded.session.add( + ExecutionOutput( + execution_id=execution.id, + output_type=ResultOutputType.HTML, + filename=None, + short_name="html-short", + long_name="HTML long name", + ) + ) + + result = invoke_cli(["executions", "inspect", str(execution_group.id)]) + + assert "Outputs" in result.stdout + assert "plot-short" in result.stdout + assert "plot.png" in result.stdout + assert "html-short" in result.stdout + + # Directory-tree / log panels still render (regression). + assert "File Tree" in result.stdout + assert "Execution Logs" in result.stdout + assert "plot.png (9 bytes)" in result.stdout + + def test_inspect_no_results_skips_outputs_panel(self, sample_data_dir, db_seeded, invoke_cli): + metric_execution_group = ExecutionGroup(key="key1", diagnostic_id=1) + with db_seeded.session.begin(): + db_seeded.session.add(metric_execution_group) + + result = invoke_cli(["executions", "inspect", str(metric_execution_group.id)]) + + assert result.exit_code == 0 + assert "not-started" in result.stdout + assert "Outputs" not in result.stdout + + def test_outputs_panel_no_outputs(self, db_seeded, tmp_path): + reader = Reader(db_seeded, results=tmp_path) + panel = _outputs_panel((), "frag", reader) + + console = Console() + with console.capture() as capture: + console.print(panel) + + assert "No registered outputs." in capture.get() + + def test_outputs_panel_links_filename_rows_only(self, db_seeded, tmp_path): + outputs = ( + OutputView( + execution_id=1, + output_type="plot", + filename="plot.png", + short_name="plot-short", + long_name="Plot long name", + description=None, + dimensions={}, + ), + OutputView( + execution_id=1, + output_type="html", + filename=None, + short_name="html-short", + long_name="HTML long name", + description=None, + dimensions={}, + ), + ) + reader = Reader(db_seeded, results=tmp_path) + panel = _outputs_panel(outputs, "frag", reader) + + # `force_terminal=True` so rich emits the OSC 8 hyperlink escape sequence to inspect. + console = Console(force_terminal=True) + with console.capture() as capture: + console.print(panel) + rendered = capture.get() + + assert f"file://{tmp_path / 'frag' / 'plot.png'}" in rendered + assert "html-short" in rendered + assert "file://" not in rendered.split("html-short")[1] + def test_results_directory_panel(self, tmp_path): tmp_path = tmp_path / "inner" tmp_path.mkdir() diff --git a/packages/climate-ref/tests/unit/cli/test_executions/test_inspect.txt b/packages/climate-ref/tests/unit/cli/test_executions/test_inspect.txt index 023b69364..1a36e14ef 100644 --- a/packages/climate-ref/tests/unit/cli/test_executions/test_inspect.txt +++ b/packages/climate-ref/tests/unit/cli/test_executions/test_inspect.txt @@ -16,6 +16,9 @@ │ 2 CMIP6.C4MIP CDRMIP.CSIRO.ACCESS-ESM1-5.esm-1pct-brch-1000PgC.r1i1p1f1.fx.areacella.gn.v20191206 SourceDatasetType.CMIP6 │ │ │ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭────────────────────────────────────────────────────────────────────────────────────────────── Outputs ───────────────────────────────────────────────────────────────────────────────────────────────╮ +│ No registered outputs. │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ╭───────────────────────────────────────────────────────────────────────────────────────────── File Tree ──────────────────────────────────────────────────────────────────────────────────────────────╮ │ Result directory not found. │ │ 📂/results/output │ diff --git a/packages/climate-ref/tests/unit/results/test_artifacts.py b/packages/climate-ref/tests/unit/results/test_artifacts.py new file mode 100644 index 000000000..8d2ce30dd --- /dev/null +++ b/packages/climate-ref/tests/unit/results/test_artifacts.py @@ -0,0 +1,73 @@ +"""Unit tests for `climate_ref.results.artifacts`.""" + +import pytest + +from climate_ref.results.artifacts import ArtifactsReader +from climate_ref_core.logging import EXECUTION_LOG_FILENAME + + +class TestOutputDirectory: + def test_resolves_under_results_root(self, tmp_path): + reader = ArtifactsReader(tmp_path) + assert reader.output_directory("frag") == tmp_path / "frag" + + def test_nested_fragment(self, tmp_path): + reader = ArtifactsReader(tmp_path) + assert reader.output_directory("a/b/c") == tmp_path / "a" / "b" / "c" + + def test_escape_raises(self, tmp_path): + reader = ArtifactsReader(tmp_path) + with pytest.raises(ValueError): + reader.output_directory("../escape") + + def test_absolute_fragment_raises(self, tmp_path): + reader = ArtifactsReader(tmp_path) + with pytest.raises(ValueError): + reader.output_directory("/etc/passwd") + + def test_sibling_prefix_raises(self, tmp_path): + # Guards against a naive string-prefix containment check: `results2` starts with + # the same characters as `results` but is a sibling directory, not a subdirectory. + results_root = tmp_path / "results" + reader = ArtifactsReader(results_root) + with pytest.raises(ValueError): + reader.output_directory("../results2/x") + + +class TestLogFile: + def test_resolves_under_output_directory(self, tmp_path): + reader = ArtifactsReader(tmp_path) + log_file = reader.log_file("frag") + assert log_file == tmp_path / "frag" / EXECUTION_LOG_FILENAME + assert log_file.name == "out.log" + + +class TestBundle: + def test_none_bundle_path_returns_none(self, tmp_path): + reader = ArtifactsReader(tmp_path) + assert reader.bundle("frag", None) is None + + def test_resolves_under_output_directory(self, tmp_path): + reader = ArtifactsReader(tmp_path) + assert reader.bundle("frag", "cmec.json") == tmp_path / "frag" / "cmec.json" + + def test_escape_raises(self, tmp_path): + reader = ArtifactsReader(tmp_path) + with pytest.raises(ValueError): + reader.bundle("frag", "../escape") + + +class TestOutputFile: + def test_resolves_under_output_directory(self, tmp_path): + reader = ArtifactsReader(tmp_path) + assert reader.output_file("frag", "plot.png") == tmp_path / "frag" / "plot.png" + + def test_relative_escape_raises(self, tmp_path): + reader = ArtifactsReader(tmp_path) + with pytest.raises(ValueError): + reader.output_file("frag", "../escape") + + def test_absolute_filename_raises(self, tmp_path): + reader = ArtifactsReader(tmp_path) + with pytest.raises(ValueError): + reader.output_file("frag", "/etc/passwd") diff --git a/packages/climate-ref/tests/unit/results/test_executions.py b/packages/climate-ref/tests/unit/results/test_executions.py index 916e68135..651f9de83 100644 --- a/packages/climate-ref/tests/unit/results/test_executions.py +++ b/packages/climate-ref/tests/unit/results/test_executions.py @@ -8,14 +8,17 @@ from climate_ref.models import Execution, ExecutionGroup from climate_ref.models.diagnostic import Diagnostic +from climate_ref.models.execution import ExecutionOutput, ResultOutputType from climate_ref.provider_registry import _register_provider -from climate_ref.results import ExecutionGroupFilter, Reader -from climate_ref.results.executions import select_execution_statistics +from climate_ref.results import ExecutionGroupFilter, MetricValueFilter, OutlierPolicy, Reader +from climate_ref.results.executions import select_execution_outputs, select_execution_statistics def test_import_smoke() -> None: - """`ExecutionGroupFilter` and `Reader` import cleanly from the top-level package.""" + """Public surface imports cleanly (guards the `values`<->`artifacts`<->`executions` wiring).""" assert ExecutionGroupFilter is not None + assert MetricValueFilter is not None + assert OutlierPolicy is not None assert Reader is not None @@ -404,3 +407,115 @@ def test_none_for_empty_group(self, db_with_groups): reader = Reader(db_with_groups) group_id = db_with_groups.group_ids["key6"] assert reader.executions.latest_execution(group_id) is None + + +@pytest.fixture +def execution_with_outputs(db_with_groups): + """The `key1` execution, with a mix of `ExecutionOutput` rows attached.""" + session = db_with_groups.session + group_id = db_with_groups.group_ids["key1"] + execution = session.query(Execution).filter_by(execution_group_id=group_id).one() + + with session.begin_nested() if session.in_transaction() else session.begin(): + session.add( + ExecutionOutput( + execution_id=execution.id, + output_type=ResultOutputType.Data, + filename="data.csv", + short_name="data-short", + long_name="Data long name", + description="A data output", + ) + ) + session.add( + ExecutionOutput( + execution_id=execution.id, + output_type=ResultOutputType.Plot, + filename="plot.png", + short_name="plot-short", + long_name="Plot long name", + ) + ) + # A metadata-only output with no filename (e.g. an HTML summary row without a file). + session.add( + ExecutionOutput( + execution_id=execution.id, + output_type=ResultOutputType.HTML, + filename=None, + short_name="html-short", + ) + ) + + return db_with_groups, execution.id + + +class TestOutputs: + def test_dto_fields_and_order(self, execution_with_outputs): + db, execution_id = execution_with_outputs + reader = Reader(db) + + outs = reader.executions.outputs(execution_id) + + assert len(outs) == 3 + # Ordered by (output_type, id): HTML < Data < Plot alphabetically by enum value. + assert [o.output_type for o in outs] == ["data", "html", "plot"] + + data_out = next(o for o in outs if o.output_type == "data") + assert data_out.execution_id == execution_id + assert data_out.filename == "data.csv" + assert data_out.short_name == "data-short" + assert data_out.long_name == "Data long name" + assert data_out.description == "A data output" + assert isinstance(data_out.dimensions, dict) + + html_out = next(o for o in outs if o.output_type == "html") + assert html_out.filename is None + assert html_out.long_name is None + + def test_empty_for_execution_with_no_outputs(self, db_with_groups): + reader = Reader(db_with_groups) + group_id = db_with_groups.group_ids["key2"] + execution = db_with_groups.session.query(Execution).filter_by(execution_group_id=group_id).one() + + assert reader.executions.outputs(execution.id) == () + + def test_detached_survival(self, execution_with_outputs): + db, execution_id = execution_with_outputs + reader = Reader(db) + + outs = reader.executions.outputs(execution_id) + db.session.expunge_all() + + assert len(outs) == 3 + for out in outs: + assert type(out.dimensions) is dict + + +class TestSelectExecutionOutputs: + def test_raw_rows_count_and_ordering(self, execution_with_outputs): + db, execution_id = execution_with_outputs + stmt = select_execution_outputs(execution_id) + rows = db.session.execute(stmt).scalars().all() + + assert len(rows) == 3 + output_types = [row.output_type.value for row in rows] + assert output_types == sorted(output_types) + + def test_raw_rows_scoped_to_execution(self, db_with_groups): + group_id = db_with_groups.group_ids["key2"] + execution = db_with_groups.session.query(Execution).filter_by(execution_group_id=group_id).one() + + stmt = select_execution_outputs(execution.id) + rows = db_with_groups.session.execute(stmt).scalars().all() + assert rows == [] + + +class TestReaderArtifactsGating: + def test_raises_without_paths(self, db_with_groups): + reader = Reader(db_with_groups) + with pytest.raises(ValueError): + reader.artifacts + + def test_works_with_paths(self, db_with_groups, tmp_path): + reader = Reader(db_with_groups, results=tmp_path) + assert reader.artifacts is not None From b0358a3f7f650971fbb86d74fc0bc0abb178dab9 Mon Sep 17 00:00:00 2001 From: Jared Lewis Date: Thu, 2 Jul 2026 12:20:18 +1000 Subject: [PATCH 05/23] feat(results): add reader.datasets dataset read layer Add a first-class ORM query path for datasets: DatasetFilter and select_datasets in models/dataset_query.py, covering facet, finalised, and execution/diagnostic relationship filters, plus DatasetsReader exposing detached DatasetView and DatasetFileView DTOs via reader.datasets. Latest-version selection reuses the adapter's filter_latest_versions so there is a single definition of that policy. --- .../src/climate_ref/models/dataset_query.py | 114 ++++++ .../src/climate_ref/results/__init__.py | 6 +- .../src/climate_ref/results/datasets.py | 162 ++++++++ .../src/climate_ref/results/values.py | 11 +- .../tests/unit/results/test_datasets.py | 379 ++++++++++++++++++ 5 files changed, 670 insertions(+), 2 deletions(-) create mode 100644 packages/climate-ref/src/climate_ref/models/dataset_query.py create mode 100644 packages/climate-ref/src/climate_ref/results/datasets.py create mode 100644 packages/climate-ref/tests/unit/results/test_datasets.py diff --git a/packages/climate-ref/src/climate_ref/models/dataset_query.py b/packages/climate-ref/src/climate_ref/models/dataset_query.py new file mode 100644 index 000000000..d0cf1e75b --- /dev/null +++ b/packages/climate-ref/src/climate_ref/models/dataset_query.py @@ -0,0 +1,114 @@ +""" +Query builder for the polymorphic ``Dataset`` hierarchy. + +[select_datasets][climate_ref.models.dataset_query.select_datasets] is the single definition of +"the datasets query": it backs both ``climate_ref.datasets`` (``DatasetAdapter.load_catalog``) and +the ``climate_ref.results`` read layer (``reader.datasets``), so the two cannot drift apart. + +It lives in the models layer -- below both of those -- and depends only on the model classes. +``source_type`` is resolved to its concrete subclass through SQLAlchemy's polymorphic map rather +than the adapter registry, so this module needs no import from ``climate_ref.datasets`` and the +import graph stays strictly top-down (no cycle). +""" + +from collections.abc import Mapping, Sequence +from typing import Any, cast + +import attrs +from sqlalchemy import Select, select + +from climate_ref.models.dataset import Dataset +from climate_ref.models.diagnostic import Diagnostic +from climate_ref.models.execution import Execution, ExecutionGroup, execution_datasets +from climate_ref_core.source_types import SourceDatasetType + + +def _as_facets( + value: Mapping[str, Sequence[str]] | None, +) -> Mapping[str, tuple[str, ...]] | None: + """Copy a facets mapping into an immutable ``dict`` of immutable ``tuple`` values.""" + if value is None: + return None + return {k: tuple(v) for k, v in value.items()} + + +@attrs.frozen(kw_only=True) +class DatasetFilter: + """ + Declarative filter over datasets. + + Every field is optional. + ``None`` means "do not constrain on this axis". + + ``source_type`` selects which concrete ``Dataset`` subclass + (and therefore which facet columns) the query targets. + When ``source_type=None``, the query stays on the base ``Dataset`` database table, + so only base columns are filterable via ``facets`` + (``slug``, ``finalised``, ``dataset_type``, ``created_at``, ``updated_at``), + and ``latest_only`` is a no-op as there is no ``dataset_id_metadata`` to group by. + """ + + source_type: SourceDatasetType | None = None + facets: Mapping[str, tuple[str, ...]] | None = attrs.field(default=None, converter=_as_facets) + finalised: bool | None = None + execution_id: int | None = None + diagnostic_slug: str | None = None + latest_only: bool = True + + +def _entity_for(source_type: SourceDatasetType | None) -> type[Dataset]: + """Resolve a source type to its concrete ``Dataset`` subclass via the polymorphic map.""" + if source_type is None: + return Dataset + return cast(type[Dataset], Dataset.__mapper__.polymorphic_map[source_type].class_) + + +def select_datasets(filter: DatasetFilter) -> Select[Any]: # noqa: A002 + """ + Build the ``Select`` over the (optionally concrete) ``Dataset`` entity for the given filter. + + The latest-version filter and any limit are deliberately not applied here; callers apply them + so a numeric limit is not spent on superseded versions. + + Raises + ------ + ValueError + If a key in ``filter.facets`` is not a mapped column on the target entity. + """ + entity = _entity_for(filter.source_type) + + stmt = select(entity) + if filter.source_type is not None: + stmt = stmt.where(entity.dataset_type == filter.source_type) + + for facet, values in (filter.facets or {}).items(): + column = getattr(entity, facet, None) + if column is None or facet not in entity.__mapper__.columns: + raise ValueError(f"Unknown facet {facet!r} for {entity.__name__}") + stmt = stmt.where(column.in_(values)) + + if filter.finalised is not None: + stmt = stmt.where(entity.finalised.is_(filter.finalised)) + + # Both relationship axes reach through ``execution_datasets`` + # join it at most once so that setting ``execution_id`` and ``diagnostic_slug`` together + # does not emit a duplicate, unaliased self-join (invalid SQL). + needs_execution_join = filter.execution_id is not None or filter.diagnostic_slug is not None + if needs_execution_join: + stmt = stmt.join(execution_datasets, entity.id == execution_datasets.c.dataset_id) + + if filter.execution_id is not None: + stmt = stmt.where(execution_datasets.c.execution_id == filter.execution_id) + + if filter.diagnostic_slug is not None: + stmt = ( + stmt.join(Execution, Execution.id == execution_datasets.c.execution_id) + .join(ExecutionGroup, ExecutionGroup.id == Execution.execution_group_id) + .join(Diagnostic, Diagnostic.id == ExecutionGroup.diagnostic_id) + .where(Diagnostic.slug == filter.diagnostic_slug) + ) + + if needs_execution_join: + stmt = stmt.distinct() + + return stmt.order_by(entity.updated_at.desc()) diff --git a/packages/climate-ref/src/climate_ref/results/__init__.py b/packages/climate-ref/src/climate_ref/results/__init__.py index faa409fb0..1fe5068ef 100644 --- a/packages/climate-ref/src/climate_ref/results/__init__.py +++ b/packages/climate-ref/src/climate_ref/results/__init__.py @@ -15,6 +15,8 @@ * [values][climate_ref.results.values] -- typed DTOs + collections + the ``Reader`` facade. * [executions][climate_ref.results.executions] -- execution-group / execution DTOs + collection + the ``ExecutionsReader`` facade. +* [datasets][climate_ref.results.datasets] -- dataset DTOs + collection + the ``DatasetsReader`` + facade, querying the polymorphic ``Dataset`` hierarchy directly. * [artifacts][climate_ref.results.artifacts] -- the ``ArtifactsReader`` facade, resolving execution output fragments into filesystem paths under a results root. @@ -35,7 +37,7 @@ * the ``Reader`` entry point, * filter objects you construct and pass in (``MetricValueFilter``, ``ExecutionGroupFilter``, - and the future ``DatasetFilter``), + ``DatasetFilter``), * value objects you pass in (``OutlierPolicy``). Everything the package *returns* -- DTOs, collections, views -- and the sub-reader classes reached @@ -47,11 +49,13 @@ """ from climate_ref.results._query import MetricValueFilter +from climate_ref.results.datasets import DatasetFilter from climate_ref.results.executions import ExecutionGroupFilter from climate_ref.results.outliers import OutlierPolicy from climate_ref.results.values import Reader __all__ = [ + "DatasetFilter", "ExecutionGroupFilter", "MetricValueFilter", "OutlierPolicy", diff --git a/packages/climate-ref/src/climate_ref/results/datasets.py b/packages/climate-ref/src/climate_ref/results/datasets.py new file mode 100644 index 000000000..1fc9baef3 --- /dev/null +++ b/packages/climate-ref/src/climate_ref/results/datasets.py @@ -0,0 +1,162 @@ +""" +Typed, detached read surface for datasets. + +[DatasetsReader][climate_ref.results.datasets.DatasetsReader] is reached via +[Reader.datasets][climate_ref.results.values.Reader.datasets]. It executes the shared +[select_datasets][climate_ref.models.dataset_query.select_datasets] query (also used by +``adapter.load_catalog``) and maps the rows into detached DTOs that outlive the session. + +The filter and query builder live in the models layer +([climate_ref.models.dataset_query][climate_ref.models.dataset_query]); ``DatasetFilter`` is +re-exported here so callers construct it from ``climate_ref.results``. +""" + +from collections.abc import Iterator, Mapping +from typing import Any + +import attrs +import pandas as pd +from sqlalchemy.orm import selectinload + +from climate_ref.database import Database +from climate_ref.datasets import get_dataset_adapter +from climate_ref.models.dataset import Dataset +from climate_ref.models.dataset_query import DatasetFilter, select_datasets +from climate_ref_core.source_types import SourceDatasetType + +__all__ = ["DatasetFilter", "select_datasets"] + + +@attrs.frozen(kw_only=True) +class DatasetFileView: + """A single dataset file, detached from the ORM. Times are raw stored strings (no cftime).""" + + path: str + start_time: str | None + end_time: str | None + tracking_id: str | None + + +@attrs.frozen(kw_only=True) +class DatasetView: + """A single dataset, detached from the ORM.""" + + id: int + slug: str + dataset_type: SourceDatasetType + finalised: bool + created_at: Any + updated_at: Any + facets: Mapping[str, object] + files: tuple[DatasetFileView, ...] + + +@attrs.frozen(kw_only=True) +class DatasetCollection: + """An immutable collection of datasets.""" + + datasets: tuple[DatasetView, ...] + + def __iter__(self) -> Iterator[DatasetView]: + return iter(self.datasets) + + def __len__(self) -> int: + return len(self.datasets) + + def to_pandas(self) -> pd.DataFrame: + """DataFrame with one row per dataset; columns are base fields plus the facet dict expanded.""" + records = [] + for ds in self.datasets: + rec: dict[str, Any] = { + "id": ds.id, + "slug": ds.slug, + "dataset_type": ds.dataset_type.value, + "finalised": ds.finalised, + "created_at": ds.created_at, + "updated_at": ds.updated_at, + } + rec.update(ds.facets) + records.append(rec) + return pd.DataFrame.from_records(records) + + +class DatasetsReader: + """ + Dataset read domain. + + Constructed from a [Database][climate_ref.database.Database], which owns the session. + + All read methods return detached DTOs that outlive the session. + """ + + def __init__(self, database: Database) -> None: + self._db = database + + def _to_view(self, dataset: Dataset, *, include_files: bool) -> DatasetView: + adapter = get_dataset_adapter(dataset.dataset_type.value) + facets = {k: getattr(dataset, k) for k in adapter.dataset_specific_metadata if hasattr(dataset, k)} + files = ( + tuple( + DatasetFileView( + path=f.path, start_time=f.start_time, end_time=f.end_time, tracking_id=f.tracking_id + ) + for f in dataset.files # type: ignore[attr-defined] + ) + if include_files + else () + ) + return DatasetView( + id=dataset.id, + slug=dataset.slug, + dataset_type=dataset.dataset_type, + finalised=dataset.finalised, + created_at=dataset.created_at, + updated_at=dataset.updated_at, + facets=facets, + files=files, + ) + + def datasets( + self, + filter: DatasetFilter | None = None, # noqa: A002 + *, + limit: int | None = None, + include_files: bool = False, + ) -> DatasetCollection: + """ + Query datasets, optionally scoped to a source type, execution or diagnostic. + + ``limit`` (when given) is applied after ``latest_only`` filtering, over the ordered, latest + datasets, so it caps returned datasets -- matching ``adapter.load_catalog``'s dedup-then-limit + ordering. + """ + filter = filter or DatasetFilter() # noqa: A001 + + entity: type[Dataset] = ( + get_dataset_adapter(filter.source_type.value).dataset_cls if filter.source_type else Dataset + ) + stmt = select_datasets(filter) + if include_files: + stmt = stmt.options(selectinload(entity.files)) # type: ignore[attr-defined] + + session = self._db.session + rows = list(session.execute(stmt).scalars().unique().all()) + + adapter = get_dataset_adapter(filter.source_type.value) if filter.source_type else None + if filter.latest_only and adapter is not None and adapter.dataset_id_metadata: + id_cols = adapter.dataset_id_metadata + version_col = adapter.version_metadata + # Reuse the adapter's own latest-version policy (its short-circuits + numeric compare) so + # there is a single definition of "latest version": feed it a minimal id/version frame + # indexed by dataset id and keep the survivors. + versions = pd.DataFrame( + [{**{c: getattr(r, c) for c in id_cols}, version_col: getattr(r, version_col)} for r in rows], + index=[r.id for r in rows], + ) + surviving_ids = set(adapter.filter_latest_versions(versions).index) + rows = [r for r in rows if r.id in surviving_ids] + + if limit is not None: + rows = rows[:limit] + + return DatasetCollection(datasets=tuple(self._to_view(r, include_files=include_files) for r in rows)) diff --git a/packages/climate-ref/src/climate_ref/results/values.py b/packages/climate-ref/src/climate_ref/results/values.py index b94a41dce..a5938c105 100644 --- a/packages/climate-ref/src/climate_ref/results/values.py +++ b/packages/climate-ref/src/climate_ref/results/values.py @@ -39,6 +39,7 @@ if TYPE_CHECKING: from climate_ref.results.artifacts import ArtifactsReader + from climate_ref.results.datasets import DatasetsReader def _kind_of(dimensions: Mapping[str, str]) -> str: @@ -401,7 +402,8 @@ class Reader: Constructed from a [Database][climate_ref.database.Database], which owns the session and the read-only story. This is a thin entry point: it exposes per-domain sub-readers as properties, [values][climate_ref.results.values.Reader.values] for metric-value reads, - [executions][climate_ref.results.values.Reader.executions] for execution-group reads, and + [executions][climate_ref.results.values.Reader.executions] for execution-group reads, + [datasets][climate_ref.results.values.Reader.datasets] for dataset reads, and [artifacts][climate_ref.results.values.Reader.artifacts] for output path resolution (only available when a ``results`` root is supplied). """ @@ -425,6 +427,13 @@ def executions(self) -> ExecutionsReader: """Execution-group and execution reads.""" return ExecutionsReader(self._db) + @functools.cached_property + def datasets(self) -> "DatasetsReader": + """Dataset reads.""" + from climate_ref.results.datasets import DatasetsReader # noqa: PLC0415 + + return DatasetsReader(self._db) + @functools.cached_property def artifacts(self) -> "ArtifactsReader": """ diff --git a/packages/climate-ref/tests/unit/results/test_datasets.py b/packages/climate-ref/tests/unit/results/test_datasets.py new file mode 100644 index 000000000..b86d91cf2 --- /dev/null +++ b/packages/climate-ref/tests/unit/results/test_datasets.py @@ -0,0 +1,379 @@ +"""Unit tests for the dataset read layer.""" + +import pytest + +from climate_ref.models.dataset import CMIP6Dataset, DatasetFile, Obs4MIPsDataset +from climate_ref.models.diagnostic import Diagnostic +from climate_ref.models.execution import Execution, ExecutionGroup, execution_datasets +from climate_ref.results import DatasetFilter, Reader +from climate_ref_core.source_types import SourceDatasetType + + +def test_import_smoke() -> None: + """`DatasetFilter` is importable from the curated top-level surface.""" + assert DatasetFilter is not None + + +def _make_cmip6_dataset( + *, + slug: str, + source_id: str = "TEST-MODEL", + experiment_id: str = "historical", + variable_id: str = "tas", + member_id: str = "r1i1p1f1", + table_id: str = "Amon", + grid_label: str = "gn", + version: str = "v20200101", + finalised: bool = True, +) -> CMIP6Dataset: + return CMIP6Dataset( + slug=slug, + dataset_type=SourceDatasetType.CMIP6, + activity_id="CMIP", + experiment_id=experiment_id, + institution_id="TEST", + source_id=source_id, + member_id=member_id, + table_id=table_id, + variable_id=variable_id, + grid_label=grid_label, + version=version, + instance_id=slug, + variant_label=member_id, + finalised=finalised, + ) + + +@pytest.fixture +def db_with_datasets(db_seeded): + """A seeded DB (CMIP6 + obs4MIPs already ingested) plus controlled version/facet variety.""" + with db_seeded.session.begin(): + v2 = _make_cmip6_dataset( + slug="CMIP.TEST.TEST-MODEL.historical.Amon.tas.gn.v2", + source_id="TEST-MODEL", + variable_id="tas", + version="v2", + ) + v10 = _make_cmip6_dataset( + slug="CMIP.TEST.TEST-MODEL.historical.Amon.tas.gn.v10", + source_id="TEST-MODEL", + variable_id="tas", + version="v10", + ) + unfinalised = _make_cmip6_dataset( + slug="CMIP.TEST.OTHER-MODEL.historical.Amon.pr.gn.v1", + source_id="OTHER-MODEL", + variable_id="pr", + version="v1", + finalised=False, + ) + db_seeded.session.add_all([v2, v10, unfinalised]) + db_seeded.session.flush() + + db_seeded.session.add( + DatasetFile( + dataset_id=v10.id, + path="tas_v10.nc", + start_time="2000-01-01", + end_time="2000-12-31", + tracking_id="hdl:test-tracking-id", + ) + ) + + obs_dataset = Obs4MIPsDataset( + slug="obs4mips-test-dataset", + dataset_type=SourceDatasetType.obs4MIPs, + activity_id="obs4MIPs", + frequency="mon", + grid="native", + grid_label="gn", + institution_id="TEST", + long_name="Test obs4MIPs dataset", + nominal_resolution="100 km", + realm="atmos", + product="observations", + source_id="TEST-OBS", + source_type="satellite", + units="K", + variable_id="tas", + variant_label="v1", + version="v1", + vertical_levels=1, + source_version_number="1", + instance_id="obs4mips-test-dataset", + ) + db_seeded.session.add(obs_dataset) + db_seeded.session.flush() + + diag = db_seeded.session.query(Diagnostic).first() + eg = ExecutionGroup(key="key1", diagnostic_id=diag.id, selectors={}) + db_seeded.session.add(eg) + db_seeded.session.flush() + execution = Execution( + execution_group_id=eg.id, successful=True, output_fragment="out1", dataset_hash="hash1" + ) + db_seeded.session.add(execution) + db_seeded.session.flush() + + db_seeded.session.execute( + execution_datasets.insert(), + [{"execution_id": execution.id, "dataset_id": v10.id}], + ) + + db_seeded.session.commit() + + db_seeded.dataset_ids = { + "v2": v2.id, + "v10": v10.id, + "unfinalised": unfinalised.id, + "obs4mips": obs_dataset.id, + } + db_seeded.execution_id = execution.id + db_seeded.diagnostic_slug = diag.slug + return db_seeded + + +class TestFacetFiltering: + def test_or_within_facet(self, db_with_datasets): + reader = Reader(db_with_datasets) + coll = reader.datasets.datasets( + DatasetFilter( + source_type=SourceDatasetType.CMIP6, + facets={"variable_id": ("tas", "pr")}, + latest_only=False, + ) + ) + variable_ids = {d.facets["variable_id"] for d in coll} + assert variable_ids <= {"tas", "pr"} + assert len(coll) >= 2 + + def test_and_across_facets(self, db_with_datasets): + reader = Reader(db_with_datasets) + coll = reader.datasets.datasets( + DatasetFilter( + source_type=SourceDatasetType.CMIP6, + facets={"variable_id": ("pr",), "source_id": ("OTHER-MODEL",)}, + latest_only=False, + ) + ) + assert len(coll) == 1 + assert coll.datasets[0].facets["variable_id"] == "pr" + assert coll.datasets[0].facets["source_id"] == "OTHER-MODEL" + + def test_and_across_facets_no_match(self, db_with_datasets): + reader = Reader(db_with_datasets) + coll = reader.datasets.datasets( + DatasetFilter( + source_type=SourceDatasetType.CMIP6, + facets={"variable_id": ("pr",), "source_id": ("TEST-MODEL",)}, + latest_only=False, + ) + ) + assert len(coll) == 0 + + def test_unknown_facet_raises(self, db_with_datasets): + reader = Reader(db_with_datasets) + with pytest.raises(ValueError, match="Unknown facet"): + reader.datasets.datasets( + DatasetFilter(source_type=SourceDatasetType.CMIP6, facets={"nonexistent_facet": ("x",)}) + ) + + def test_unknown_facet_on_base_entity_raises(self, db_with_datasets): + reader = Reader(db_with_datasets) + with pytest.raises(ValueError, match="Unknown facet"): + reader.datasets.datasets(DatasetFilter(facets={"variable_id": ("tas",)})) + + def test_facet_on_base_entity_matches_base_column(self, db_with_datasets): + reader = Reader(db_with_datasets) + coll = reader.datasets.datasets(DatasetFilter(facets={"dataset_type": (SourceDatasetType.obs4MIPs,)})) + assert len(coll) > 0 + assert all(d.dataset_type == SourceDatasetType.obs4MIPs for d in coll) + assert db_with_datasets.dataset_ids["obs4mips"] in {d.id for d in coll} + + +class TestFinalisedFilter: + def test_finalised_true(self, db_with_datasets): + reader = Reader(db_with_datasets) + coll = reader.datasets.datasets( + DatasetFilter(source_type=SourceDatasetType.CMIP6, finalised=True, latest_only=False) + ) + assert all(d.finalised for d in coll) + slugs = {d.slug for d in coll} + assert "CMIP.TEST.OTHER-MODEL.historical.Amon.pr.gn.v1" not in slugs + + def test_finalised_false(self, db_with_datasets): + reader = Reader(db_with_datasets) + coll = reader.datasets.datasets( + DatasetFilter(source_type=SourceDatasetType.CMIP6, finalised=False, latest_only=False) + ) + assert len(coll) == 1 + assert coll.datasets[0].slug == "CMIP.TEST.OTHER-MODEL.historical.Amon.pr.gn.v1" + + +class TestLatestOnly: + def test_keeps_newest_numeric_version(self, db_with_datasets): + reader = Reader(db_with_datasets) + coll = reader.datasets.datasets( + DatasetFilter( + source_type=SourceDatasetType.CMIP6, + facets={"source_id": ("TEST-MODEL",), "variable_id": ("tas",)}, + ) + ) + # v10 must win over v2 numerically, not lexically ("v10" < "v2" as strings). + assert len(coll) == 1 + assert coll.datasets[0].id == db_with_datasets.dataset_ids["v10"] + + def test_latest_only_false_keeps_both_versions(self, db_with_datasets): + reader = Reader(db_with_datasets) + coll = reader.datasets.datasets( + DatasetFilter( + source_type=SourceDatasetType.CMIP6, + facets={"source_id": ("TEST-MODEL",), "variable_id": ("tas",)}, + latest_only=False, + ) + ) + assert len(coll) == 2 + + def test_latest_only_noop_when_source_type_none(self, db_with_datasets): + reader = Reader(db_with_datasets) + coll = reader.datasets.datasets(DatasetFilter(latest_only=True)) + # All datasets (across types) present -- no dataset_id_metadata to group by. + all_coll = reader.datasets.datasets(DatasetFilter(latest_only=False)) + assert len(coll) == len(all_coll) + + +class TestExecutionIdJoin: + def test_execution_id_scopes_to_linked_datasets(self, db_with_datasets): + reader = Reader(db_with_datasets) + coll = reader.datasets.datasets( + DatasetFilter( + source_type=SourceDatasetType.CMIP6, + execution_id=db_with_datasets.execution_id, + latest_only=False, + ) + ) + assert len(coll) == 1 + assert coll.datasets[0].id == db_with_datasets.dataset_ids["v10"] + + def test_execution_id_no_match(self, db_with_datasets): + reader = Reader(db_with_datasets) + coll = reader.datasets.datasets( + DatasetFilter(source_type=SourceDatasetType.CMIP6, execution_id=999999, latest_only=False) + ) + assert len(coll) == 0 + + +class TestDiagnosticSlugJoin: + def test_diagnostic_slug_scopes_to_linked_datasets(self, db_with_datasets): + reader = Reader(db_with_datasets) + coll = reader.datasets.datasets( + DatasetFilter( + source_type=SourceDatasetType.CMIP6, + diagnostic_slug=db_with_datasets.diagnostic_slug, + latest_only=False, + ) + ) + assert len(coll) == 1 + assert coll.datasets[0].id == db_with_datasets.dataset_ids["v10"] + + def test_diagnostic_slug_no_match(self, db_with_datasets): + reader = Reader(db_with_datasets) + coll = reader.datasets.datasets( + DatasetFilter( + source_type=SourceDatasetType.CMIP6, + diagnostic_slug="nonexistent-diagnostic", + latest_only=False, + ) + ) + assert len(coll) == 0 + + def test_execution_id_and_diagnostic_slug_together(self, db_with_datasets): + # Both axes reach through ``execution_datasets``; setting both must not emit a duplicate, + # unaliased self-join. This executes the query, so invalid SQL would raise here. + reader = Reader(db_with_datasets) + coll = reader.datasets.datasets( + DatasetFilter( + source_type=SourceDatasetType.CMIP6, + execution_id=db_with_datasets.execution_id, + diagnostic_slug=db_with_datasets.diagnostic_slug, + latest_only=False, + ) + ) + assert len(coll) == 1 + assert coll.datasets[0].id == db_with_datasets.dataset_ids["v10"] + + +class TestLimit: + def test_limit_applied_after_latest_only(self, db_with_datasets): + reader = Reader(db_with_datasets) + coll = reader.datasets.datasets(DatasetFilter(source_type=SourceDatasetType.CMIP6), limit=1) + assert len(coll) == 1 + + +class TestIncludeFiles: + def test_include_files_true_populates_files(self, db_with_datasets): + reader = Reader(db_with_datasets) + coll = reader.datasets.datasets( + DatasetFilter( + source_type=SourceDatasetType.CMIP6, + facets={"source_id": ("TEST-MODEL",), "variable_id": ("tas",)}, + ), + include_files=True, + ) + assert len(coll) == 1 + ds = coll.datasets[0] + assert len(ds.files) == 1 + assert ds.files[0].path == "tas_v10.nc" + assert ds.files[0].start_time == "2000-01-01" + assert ds.files[0].end_time == "2000-12-31" + assert ds.files[0].tracking_id == "hdl:test-tracking-id" + + def test_include_files_false_leaves_files_empty(self, db_with_datasets): + reader = Reader(db_with_datasets) + coll = reader.datasets.datasets( + DatasetFilter( + source_type=SourceDatasetType.CMIP6, + facets={"source_id": ("TEST-MODEL",), "variable_id": ("tas",)}, + ), + include_files=False, + ) + assert len(coll) == 1 + assert coll.datasets[0].files == () + + +class TestSourceTypeNonePolymorphic: + def test_base_query_returns_all_types(self, db_with_datasets): + reader = Reader(db_with_datasets) + coll = reader.datasets.datasets(DatasetFilter(latest_only=False)) + types = {d.dataset_type for d in coll} + assert SourceDatasetType.CMIP6 in types + assert SourceDatasetType.obs4MIPs in types + + +class TestDetachment: + def test_dto_fields_survive_session_close(self, db_with_datasets): + reader = Reader(db_with_datasets) + coll = reader.datasets.datasets( + DatasetFilter( + source_type=SourceDatasetType.CMIP6, + facets={"source_id": ("TEST-MODEL",), "variable_id": ("tas",)}, + ), + include_files=True, + ) + db_with_datasets.session.expunge_all() + + assert len(coll) == 1 + ds = coll.datasets[0] + assert ds.slug == "CMIP.TEST.TEST-MODEL.historical.Amon.tas.gn.v10" + assert ds.facets["variable_id"] == "tas" + assert len(ds.files) == 1 + + +class TestToPandas: + def test_to_pandas_columns(self, db_with_datasets): + reader = Reader(db_with_datasets) + coll = reader.datasets.datasets(DatasetFilter(source_type=SourceDatasetType.CMIP6, latest_only=False)) + df = coll.to_pandas() + for col in ("id", "slug", "dataset_type", "finalised", "created_at", "updated_at"): + assert col in df.columns + assert len(df) == len(coll) From d5630f138c8008b728709838651aab14b50e6b2d Mon Sep 17 00:00:00 2001 From: Jared Lewis Date: Thu, 2 Jul 2026 12:20:28 +1000 Subject: [PATCH 06/23] refactor(datasets): extract coerce_catalog_times DatasetAdapter.load_catalog and solve_helpers.load_local_catalogs coerced a catalog's start_time/end_time strings to cftime with identical inline code. Move it to a shared datasets.utils.coerce_catalog_times helper. --- .../climate-ref/src/climate_ref/datasets/utils.py | 14 ++++++++++++++ .../climate-ref/src/climate_ref/solve_helpers.py | 12 ++++-------- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/packages/climate-ref/src/climate_ref/datasets/utils.py b/packages/climate-ref/src/climate_ref/datasets/utils.py index a832afa11..6651ce210 100644 --- a/packages/climate-ref/src/climate_ref/datasets/utils.py +++ b/packages/climate-ref/src/climate_ref/datasets/utils.py @@ -140,6 +140,20 @@ def _inner(date_value: object, cal_value: object) -> cftime.datetime | None: ) +def coerce_catalog_times(catalog: pd.DataFrame) -> pd.DataFrame: + """ + Coerce a catalog's stored ``start_time``/``end_time`` strings to cftime objects. + + A no-op when the catalog has no ``start_time`` column (dataset-level catalogs). Each row's + ``calendar`` is used when present, otherwise ``"standard"``. Mutates and returns ``catalog``. + """ + if "start_time" in catalog.columns: + cal = catalog["calendar"] if "calendar" in catalog.columns else "standard" + catalog["start_time"] = parse_cftime_dates(catalog["start_time"], cal) + catalog["end_time"] = parse_cftime_dates(catalog["end_time"], cal) + return catalog + + def clean_branch_time(branch_time: pd.Series[str]) -> pd.Series[float]: """ Clean branch time values, handling missing values and EC-Earth3 suffixes. diff --git a/packages/climate-ref/src/climate_ref/solve_helpers.py b/packages/climate-ref/src/climate_ref/solve_helpers.py index c590046bf..c2fd2fc0f 100644 --- a/packages/climate-ref/src/climate_ref/solve_helpers.py +++ b/packages/climate-ref/src/climate_ref/solve_helpers.py @@ -19,7 +19,7 @@ from climate_ref.data_catalog import DataCatalog from climate_ref.datasets import get_dataset_adapter -from climate_ref.datasets.utils import parse_cftime_dates +from climate_ref.datasets.utils import coerce_catalog_times from climate_ref.provider_registry import ProviderRegistry from climate_ref.solver import ExecutionSolver, SolveFilterOptions from climate_ref_core.datasets import SourceDatasetType @@ -136,13 +136,9 @@ def load_solve_catalog(catalog_dir: Path) -> dict[SourceDatasetType, pd.DataFram if path.exists(): catalog = pd.read_parquet(path) - # Convert start_time/end_time strings back to cftime objects - if "start_time" in catalog.columns: - cal = catalog["calendar"] if "calendar" in catalog.columns else "standard" - catalog["start_time"] = parse_cftime_dates(catalog["start_time"], cal) - catalog["end_time"] = parse_cftime_dates(catalog["end_time"], cal) - - # Apply the same version deduplication as DatasetAdapter.load_catalog() + # Normalise the parquet catalog the same way DatasetAdapter.load_catalog() does: + # coerce time columns to cftime, then keep only the latest version of each dataset. + catalog = coerce_catalog_times(catalog) adapter = get_dataset_adapter(source_type.value) result[source_type] = adapter.filter_latest_versions(catalog) From b28c842aacf5e93e3fda1b1b922662111c4c097a Mon Sep 17 00:00:00 2001 From: Jared Lewis Date: Thu, 2 Jul 2026 12:20:44 +1000 Subject: [PATCH 07/23] refactor(datasets): back load_catalog with the shared query builder Route load_catalog's dataset and file queries through select_datasets so load_catalog and reader.datasets share one definition of the datasets query. Apply the row limit after deduplicating to the latest version. Previously the limit was pushed into the query before deduplication, so it could be spent on superseded versions that were then dropped, returning fewer datasets than requested and occasionally omitting datasets entirely. --- .../src/climate_ref/datasets/base.py | 77 +++++++++++-------- .../tests/unit/datasets/test_cmip6.py | 9 +++ 2 files changed, 52 insertions(+), 34 deletions(-) diff --git a/packages/climate-ref/src/climate_ref/datasets/base.py b/packages/climate-ref/src/climate_ref/datasets/base.py index 151dc8f53..a114d7e80 100644 --- a/packages/climate-ref/src/climate_ref/datasets/base.py +++ b/packages/climate-ref/src/climate_ref/datasets/base.py @@ -4,11 +4,13 @@ import pandas as pd from attrs import define from loguru import logger -from sqlalchemy.orm import joinedload +from sqlalchemy import Select +from sqlalchemy.orm import selectinload from climate_ref.database import Database, ModelState -from climate_ref.datasets.utils import _is_na, _to_db_str, parse_cftime_dates, validate_path +from climate_ref.datasets.utils import _is_na, _to_db_str, coerce_catalog_times, validate_path from climate_ref.models.dataset import Dataset, DatasetFile +from climate_ref.models.dataset_query import DatasetFilter, select_datasets from climate_ref_core.datasets import select_latest_version from climate_ref_core.exceptions import RefException @@ -445,42 +447,44 @@ def filter_latest_versions(self, catalog: pd.DataFrame) -> pd.DataFrame: group_by=self.dataset_id_metadata, ) - def _get_dataset_files(self, db: Database, limit: int | None = None) -> pd.DataFrame: - dataset_type = self.dataset_cls.__mapper_args__["polymorphic_identity"] - - result = ( - db.session.query(DatasetFile) - # The join is necessary to be able to order by the dataset columns - .join(DatasetFile.dataset) - .where(Dataset.dataset_type == dataset_type) - # The joinedload is necessary to avoid N+1 queries (one for each dataset) - # https://docs.sqlalchemy.org/en/14/orm/loading_relationships.html#the-zen-of-joined-eager-loading - .options(joinedload(DatasetFile.dataset.of_type(self.dataset_cls))) - .order_by(Dataset.updated_at.desc()) - .limit(limit) - .all() - ) + def _dataset_query(self) -> Select[Any]: + """ + Build the ``Select`` for this adapter's datasets via the shared query builder. + + Routing through ``climate_ref.models.dataset_query.select_datasets`` keeps ONE definition of + "the datasets query" behind both ``load_catalog`` and ``climate_ref.results``'s + ``reader.datasets``, so the two cannot drift apart. + """ + source_type = self.dataset_cls.__mapper_args__["polymorphic_identity"] + # ``select_datasets`` does not apply the latest-version filter; + # ``load_catalog`` still owns deduplication (``filter_latest_versions``) and the limit, + # so behaviour is unchanged. + return select_datasets(DatasetFilter(source_type=source_type)) + + def _get_dataset_files(self, db: Database) -> pd.DataFrame: + # Eager-load files to avoid N+1 (one query per dataset), then explode to one row per file. + stmt = self._dataset_query().options(selectinload(self.dataset_cls.files)) # type: ignore[attr-defined] + datasets = db.session.execute(stmt).scalars().unique().all() return pd.DataFrame( [ { **{k: getattr(file, k) for k in self.file_specific_metadata}, - **{k: getattr(file.dataset, k) for k in self.dataset_specific_metadata}, - "finalised": file.dataset.finalised, + **{k: getattr(dataset, k) for k in self.dataset_specific_metadata}, + "finalised": dataset.finalised, } - for file in result + for dataset in datasets + for file in dataset.files ], - index=[file.dataset.id for file in result], + index=[dataset.id for dataset in datasets for _file in dataset.files], ) - def _get_datasets(self, db: Database, limit: int | None = None) -> pd.DataFrame: - result_datasets = ( - db.session.query(self.dataset_cls).order_by(Dataset.updated_at.desc()).limit(limit).all() - ) + def _get_datasets(self, db: Database) -> pd.DataFrame: + result_datasets = db.session.execute(self._dataset_query()).scalars().unique().all() return pd.DataFrame( [{k: getattr(dataset, k) for k in self.dataset_specific_metadata} for dataset in result_datasets], - index=[file.id for file in result_datasets], + index=[dataset.id for dataset in result_datasets], ) def load_catalog( @@ -497,6 +501,9 @@ def load_catalog( The index of the data catalog is the primary key of the dataset. This should be maintained during any processing. + ``limit`` (when given) bounds the returned rows after deduplicating to the latest version, + so up to ``limit`` datasets are returned. + Returns ------- : @@ -505,9 +512,9 @@ def load_catalog( with db.session.begin(): # TODO: Paginate this query to avoid loading all the data at once if include_files: - catalog = self._get_dataset_files(db, limit) + catalog = self._get_dataset_files(db) else: - catalog = self._get_datasets(db, limit) + catalog = self._get_datasets(db) # If there are no datasets, return an empty DataFrame if catalog.empty: @@ -515,12 +522,14 @@ def load_catalog( return self._finalise_loaded_catalog(empty) # Convert start_time/end_time strings from DB to cftime objects - if "start_time" in catalog.columns: - cal = catalog["calendar"] if "calendar" in catalog.columns else "standard" - catalog["start_time"] = parse_cftime_dates(catalog["start_time"], cal) - catalog["end_time"] = parse_cftime_dates(catalog["end_time"], cal) - - return self._finalise_loaded_catalog(self.filter_latest_versions(catalog)) + catalog = coerce_catalog_times(catalog) + + # Deduplicate to the latest version BEFORE applying the limit, so the limit is not spent on + # superseded versions (which would otherwise be dropped, returning fewer rows than requested). + catalog = self.filter_latest_versions(catalog) + if limit is not None: + catalog = catalog.head(limit) + return self._finalise_loaded_catalog(catalog) def _finalise_loaded_catalog(self, catalog: pd.DataFrame) -> pd.DataFrame: """ diff --git a/packages/climate-ref/tests/unit/datasets/test_cmip6.py b/packages/climate-ref/tests/unit/datasets/test_cmip6.py index 386ef22d5..fea180b16 100644 --- a/packages/climate-ref/tests/unit/datasets/test_cmip6.py +++ b/packages/climate-ref/tests/unit/datasets/test_cmip6.py @@ -52,6 +52,15 @@ def test_load_catalog_multiple_versions(self, config, db_seeded, catalog_regress assert target_ds not in latest_instance_ids assert new_instance_id in latest_instance_ids + # The limit is applied AFTER deduplicating to the latest version, so it bounds the number of + # datasets actually returned. The target now has three versions in the DB; a limit equal to the + # deduplicated dataset count must still return every dataset (a limit applied before dedup would + # spend slots on the target's superseded versions and silently drop other datasets). + n_datasets = len(adapter.load_catalog(db_seeded, include_files=False)) + limited = adapter.load_catalog(db_seeded, include_files=False, limit=n_datasets) + assert len(limited) == n_datasets + assert new_instance_id in limited.instance_id.tolist() + class TestCMIP6IterLocalDatasets: def test_streaming_matches_whole_tree(self, sample_data, sample_data_dir): From ed6d586b8879d04abb4da79837858f636aa599ef Mon Sep 17 00:00:00 2001 From: Jared Lewis Date: Thu, 2 Jul 2026 14:42:14 +1000 Subject: [PATCH 08/23] feat(datasets): add version_key column synced by a mapper event MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Persist a numeric version_key on the base dataset table, computed by version_sort_key, so the latest version of a dataset can be selected in SQL (a following change adds the window-function dedup that orders by it). A before_insert/before_update mapper event on Dataset keeps version_key in sync for every write path — including direct-ORM inserts — rather than relying on register_dataset, and reads the final attribute value so a stripped-metadata upsert cannot leave a stale key. version_key is BigInteger to hold any version string's digits without overflow on PostgreSQL. The migration backfills existing rows in Python via version_sort_key over the four subclass tables; version strings are not parsed in SQL because that mapping has no portable expression across SQLite and PostgreSQL. --- ...07-02T0338_f6a7b8c9d0e1_add_version_key.py | 74 +++++++ .../src/climate_ref/models/dataset.py | 27 ++- .../datasets/test_migrations_version_key.py | 189 ++++++++++++++++++ 3 files changed, 289 insertions(+), 1 deletion(-) create mode 100644 packages/climate-ref/src/climate_ref/migrations/versions/2026-07-02T0338_f6a7b8c9d0e1_add_version_key.py create mode 100644 packages/climate-ref/tests/unit/datasets/test_migrations_version_key.py diff --git a/packages/climate-ref/src/climate_ref/migrations/versions/2026-07-02T0338_f6a7b8c9d0e1_add_version_key.py b/packages/climate-ref/src/climate_ref/migrations/versions/2026-07-02T0338_f6a7b8c9d0e1_add_version_key.py new file mode 100644 index 000000000..0a7119908 --- /dev/null +++ b/packages/climate-ref/src/climate_ref/migrations/versions/2026-07-02T0338_f6a7b8c9d0e1_add_version_key.py @@ -0,0 +1,74 @@ +"""add version_key to base dataset table + +Adds ``dataset.version_key``, a numeric ordering key for each dataset's ``version`` string, +computed by ``climate_ref_core.datasets.version_sort_key``. It powers the SQL-side +latest-version window function (``RANK() OVER (PARTITION BY ORDER BY +version_key DESC)``) used by ``select_datasets(..., latest_group_by=...)``. + +``version`` itself lives on the four subclass tables (``cmip6_dataset``, ``cmip7_dataset``, +``obs4mips_dataset``, ``pmp_climatology_dataset``), not the base ``dataset`` table, so the +backfill below reads each subclass table's ``version`` column and writes the computed key onto +the corresponding base-table row. Parsing versions in SQL was rejected -- ``version_sort_key``'s +regex/int-cast logic has no portable SQL expression across SQLite and PostgreSQL -- so the +backfill runs in Python. + +After this migration, a SQLAlchemy ``before_insert``/``before_update`` mapper event on the +``Dataset`` model keeps ``version_key`` in sync for every future write. + +Revision ID: f6a7b8c9d0e1 +Revises: e5f6a7b8c9d0 +Create Date: 2026-07-02 +""" + +from collections.abc import Sequence + +import sqlalchemy as sa +from alembic import op + +from climate_ref_core.datasets import version_sort_key + +revision: str = "f6a7b8c9d0e1" +down_revision: str | None = "e5f6a7b8c9d0" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + +# The four Dataset subclass tables that own a ``version`` column. +_SUBCLASS_TABLES = ("cmip6_dataset", "cmip7_dataset", "obs4mips_dataset", "pmp_climatology_dataset") + + +def _backfill() -> None: + """Compute ``version_key`` in Python from each subclass table's ``version`` column. + + Batches the ``UPDATE`` by distinct version value (there are typically few distinct versions + relative to the number of datasets), rather than issuing one ``UPDATE`` per row. + """ + bind = op.get_bind() + metadata = sa.MetaData() + dataset = sa.Table("dataset", metadata, autoload_with=bind) + + for table_name in _SUBCLASS_TABLES: + sub = sa.Table(table_name, metadata, autoload_with=bind) + distinct_versions = bind.execute(sa.select(sub.c.version).distinct()).scalars().all() + + for version in distinct_versions: + key = version_sort_key(version) + # ``version`` is NOT NULL on every subclass table, so ``==`` is correct and portable. + # ``col.is_(value)`` would compile to ``version IS 'v10'``, which SQLite tolerates but + # PostgreSQL rejects (``IS`` only accepts NULL/boolean) -- and the unit tests are + # SQLite-only, so that would surface only on a populated Postgres upgrade. + id_subquery = sa.select(sub.c.id).where(sub.c.version == version) + bind.execute(dataset.update().where(dataset.c.id.in_(id_subquery)).values(version_key=key)) + + +def upgrade() -> None: + with op.batch_alter_table("dataset", schema=None) as batch_op: + batch_op.add_column( + sa.Column("version_key", sa.BigInteger(), nullable=False, server_default=sa.text("-1")) + ) + + _backfill() + + +def downgrade() -> None: + with op.batch_alter_table("dataset", schema=None) as batch_op: + batch_op.drop_column("version_key") diff --git a/packages/climate-ref/src/climate_ref/models/dataset.py b/packages/climate-ref/src/climate_ref/models/dataset.py index b4d0a90ba..179f226f5 100644 --- a/packages/climate-ref/src/climate_ref/models/dataset.py +++ b/packages/climate-ref/src/climate_ref/models/dataset.py @@ -1,11 +1,12 @@ import datetime from typing import Any, ClassVar -from sqlalchemy import ColumnElement, ForeignKey, func +from sqlalchemy import BigInteger, ColumnElement, ForeignKey, event, func from sqlalchemy.ext.hybrid import hybrid_property from sqlalchemy.orm import Mapped, mapped_column, relationship, validates from climate_ref.models.base import Base +from climate_ref_core.datasets import version_sort_key from climate_ref_core.source_types import SourceDatasetType @@ -56,12 +57,36 @@ class Dataset(Base): For other dataset types (e.g., obs4MIPs, PMP climatology), this should be True upon creation. """ + version_key: Mapped[int] = mapped_column(BigInteger, default=-1, server_default="-1", nullable=False) + """ + Numeric ordering key for the subclass's ``version`` column, + computed by :func:`climate_ref_core.datasets.version_sort_key`. + + Kept in sync by the ``_sync_version_key`` mapper event. + Rows with no ``version`` attribute (base-table-only inserts) keep the ``-1`` backstop. + + Lives on the base table (not a subclass) so the SQL latest-version window function + (``select_datasets(..., latest_group_by=...)``) can read it off any polymorphic row while + partitioning on the subclass's ``dataset_id_metadata`` columns. + """ + def __repr__(self) -> str: return f"" __mapper_args__: ClassVar[Any] = {"polymorphic_on": dataset_type} # type: ignore +@event.listens_for(Dataset, "before_insert", propagate=True) +@event.listens_for(Dataset, "before_update", propagate=True) +def _sync_version_key(mapper: Any, connection: Any, target: Dataset) -> None: + """Keep ``version_key`` numerically in sync with the subclass's ``version`` column. + + ``propagate=True`` fires this for every ``Dataset`` subclass, + reading the final attribute value on the instance so it is write path independent. + """ + target.version_key = version_sort_key(getattr(target, "version", None)) + + class DatasetFile(Base): """ Capture the metadata for a file in a dataset diff --git a/packages/climate-ref/tests/unit/datasets/test_migrations_version_key.py b/packages/climate-ref/tests/unit/datasets/test_migrations_version_key.py new file mode 100644 index 000000000..0ad58c325 --- /dev/null +++ b/packages/climate-ref/tests/unit/datasets/test_migrations_version_key.py @@ -0,0 +1,189 @@ +"""Tests for the ``version_key`` migration (SQL-side latest-version deduplication). + +Covers: +1. Backfill of pre-existing rows across all four ``Dataset`` subclasses, keyed via + ``version_sort_key`` (numeric, non-conforming, and non-numeric versions). +2. Downgrade dropping the column. + +Driven in-test via ``database.alembic_config(...)`` + ``command.upgrade``/``downgrade``. +""" + +import sqlalchemy as sa +from alembic import command + +from climate_ref.database import Database +from climate_ref_core.source_types import SourceDatasetType + +# The revision immediately before "add version_key to base dataset table". +_PREVIOUS_REVISION = "e5f6a7b8c9d0" +_THIS_REVISION = "f6a7b8c9d0e1" + + +class TestVersionKeyBackfill: + """The migration backfills ``version_key`` for rows that pre-date the column.""" + + def test_backfill_computes_correct_keys(self, db: Database, config) -> None: + alembic_cfg = db.alembic_config(config) + + # Start from just before this revision so the column doesn't exist yet. + command.downgrade(alembic_cfg, _PREVIOUS_REVISION) + + # Insert rows directly via Core (the ORM model no longer matches the downgraded schema) + # covering: numeric version, multi-digit numeric version, and a non-conforming version, + # across two different subclass tables. + bind = db._engine + metadata = sa.MetaData() + dataset = sa.Table("dataset", metadata, autoload_with=bind) + cmip6_dataset = sa.Table("cmip6_dataset", metadata, autoload_with=bind) + obs4mips_dataset = sa.Table("obs4mips_dataset", metadata, autoload_with=bind) + cmip7_dataset = sa.Table("cmip7_dataset", metadata, autoload_with=bind) + pmp_climatology_dataset = sa.Table("pmp_climatology_dataset", metadata, autoload_with=bind) + + with bind.begin() as conn: + base_common = {"finalised": True} + cmip6_common = { + "activity_id": "CMIP", + "experiment_id": "historical", + "institution_id": "TEST", + "source_id": "TEST-MODEL", + "member_id": "r1i1p1f1", + "table_id": "Amon", + "variable_id": "tas", + "grid_label": "gn", + "variant_label": "r1i1p1f1", + } + + def _insert_cmip6(slug: str, version: str) -> int: + base_id = conn.execute( + dataset.insert().values( + slug=slug, dataset_type=SourceDatasetType.CMIP6.value, **base_common + ) + ).inserted_primary_key[0] + conn.execute( + cmip6_dataset.insert().values( + id=base_id, version=version, instance_id=slug, **cmip6_common + ) + ) + return base_id + + id_v2 = _insert_cmip6("backfill-v2", "v2") + id_v10 = _insert_cmip6("backfill-v10", "v10") + id_nonconforming = _insert_cmip6("backfill-nonconforming", "latest") + + obs_base_id = conn.execute( + dataset.insert().values( + slug="backfill-obs4mips-v3", dataset_type=SourceDatasetType.obs4MIPs.value, **base_common + ) + ).inserted_primary_key[0] + conn.execute( + obs4mips_dataset.insert().values( + id=obs_base_id, + version="v3", + instance_id="backfill-obs4mips-v3", + activity_id="obs4MIPs", + frequency="mon", + grid="native", + grid_label="gn", + institution_id="TEST", + long_name="Test", + nominal_resolution="100 km", + realm="atmos", + product="observations", + source_id="TEST-OBS", + source_type="satellite", + units="K", + variable_id="tas", + variant_label="v1", + vertical_levels=1, + source_version_number="1", + ) + ) + + # CMIP7 with a vYYYYMMDD version -- the real production format (key = 20250622). + cmip7_base_id = conn.execute( + dataset.insert().values( + slug="backfill-cmip7-date", dataset_type=SourceDatasetType.CMIP7.value, **base_common + ) + ).inserted_primary_key[0] + conn.execute( + cmip7_dataset.insert().values( + id=cmip7_base_id, + version="v20250622", + instance_id="backfill-cmip7-date", + activity_id="CMIP", + institution_id="TEST", + source_id="TEST-MODEL", + experiment_id="historical", + variant_label="r1i1p1f1", + variable_id="tas", + grid_label="gn", + frequency="mon", + region="glb", + branding_suffix="tavg-h2m-hxy-x", + mip_era="CMIP7", + ) + ) + + pmp_base_id = conn.execute( + dataset.insert().values( + slug="backfill-pmp-v5", dataset_type=SourceDatasetType.PMPClimatology.value, **base_common + ) + ).inserted_primary_key[0] + conn.execute( + pmp_climatology_dataset.insert().values( + id=pmp_base_id, + version="v5", + instance_id="backfill-pmp-v5", + activity_id="PMP", + frequency="mon", + grid="native", + grid_label="gn", + institution_id="TEST", + long_name="Test", + nominal_resolution="100 km", + realm="atmos", + product="observations", + source_id="TEST-OBS", + source_type="satellite", + units="K", + variable_id="tas", + variant_label="v1", + vertical_levels=1, + source_version_number="1", + ) + ) + + # Upgrade to this revision -- adds the column and runs the Python backfill. + command.upgrade(alembic_cfg, _THIS_REVISION) + + with bind.connect() as conn: + dataset_reflected = sa.Table("dataset", sa.MetaData(), autoload_with=bind) + rows = { + row.id: row.version_key + for row in conn.execute(sa.select(dataset_reflected.c.id, dataset_reflected.c.version_key)) + } + + assert rows[id_v2] == 2 + assert rows[id_v10] == 10 + assert rows[id_nonconforming] == -1 + assert rows[obs_base_id] == 3 + assert rows[cmip7_base_id] == 20250622 + assert rows[pmp_base_id] == 5 + + # Re-apply head so the fixture teardown (which may run further migrations) doesn't fail. + command.upgrade(alembic_cfg, "head") + + def test_downgrade_removes_column(self, db: Database, config) -> None: + alembic_cfg = db.alembic_config(config) + + command.downgrade(alembic_cfg, _PREVIOUS_REVISION) + + insp = sa.inspect(db._engine) + cols = {c["name"] for c in insp.get_columns("dataset")} + assert "version_key" not in cols + + command.upgrade(alembic_cfg, "head") + + insp = sa.inspect(db._engine) + cols = {c["name"] for c in insp.get_columns("dataset")} + assert "version_key" in cols From 9e409b31280c6dfdc1fd943dafd085ac406b5465 Mon Sep 17 00:00:00 2001 From: Jared Lewis Date: Thu, 2 Jul 2026 14:42:48 +1000 Subject: [PATCH 09/23] feat(datasets): deduplicate datasets to the latest version in SQL Add an optional latest-version dedup to select_datasets: when latest_only is set and the caller supplies the partition columns (the adapter's dataset_id_metadata), keep only the rows at RANK 1 of a RANK() OVER (PARTITION BY ORDER BY version_key DESC) window. RANK (not ROW_NUMBER) preserves ties, matching the pandas select_latest_version it mirrors. The dedup uses an id-membership subquery rather than an aliased entity, so callers' class-bound loader options (selectinload) keep working. reader.datasets and load_catalog now dedup in SQL and, for the dataset-level result, push the limit into the same statement instead of fetching the whole table and trimming in pandas. include_files still bounds files, so its limit is applied after exploding rows. The pandas filter_latest_versions path stays for parquet catalogs, which have no database rows to window over. --- .../src/climate_ref/datasets/base.py | 38 ++- .../src/climate_ref/models/dataset_query.py | 54 +++- .../src/climate_ref/results/datasets.py | 36 +-- .../tests/unit/cli/test_datasets.py | 25 ++ .../tests/unit/datasets/test_dataset_query.py | 252 ++++++++++++++++++ .../tests/unit/datasets/test_datasets.py | 111 +++++++- 6 files changed, 468 insertions(+), 48 deletions(-) create mode 100644 packages/climate-ref/tests/unit/datasets/test_dataset_query.py diff --git a/packages/climate-ref/src/climate_ref/datasets/base.py b/packages/climate-ref/src/climate_ref/datasets/base.py index a114d7e80..b105ce2c6 100644 --- a/packages/climate-ref/src/climate_ref/datasets/base.py +++ b/packages/climate-ref/src/climate_ref/datasets/base.py @@ -449,20 +449,24 @@ def filter_latest_versions(self, catalog: pd.DataFrame) -> pd.DataFrame: def _dataset_query(self) -> Select[Any]: """ - Build the ``Select`` for this adapter's datasets via the shared query builder. + Build the ``Select`` for this adapter's latest-version datasets via the shared query builder. Routing through ``climate_ref.models.dataset_query.select_datasets`` keeps ONE definition of "the datasets query" behind both ``load_catalog`` and ``climate_ref.results``'s ``reader.datasets``, so the two cannot drift apart. + + Passes this adapter's ``dataset_id_metadata`` as ``latest_group_by`` so deduplication to the + latest version happens in SQL using a ``RANK`` window keyed on ``version_key``. """ source_type = self.dataset_cls.__mapper_args__["polymorphic_identity"] - # ``select_datasets`` does not apply the latest-version filter; - # ``load_catalog`` still owns deduplication (``filter_latest_versions``) and the limit, - # so behaviour is unchanged. - return select_datasets(DatasetFilter(source_type=source_type)) + return select_datasets( + DatasetFilter(source_type=source_type), latest_group_by=self.dataset_id_metadata + ) def _get_dataset_files(self, db: Database) -> pd.DataFrame: # Eager-load files to avoid N+1 (one query per dataset), then explode to one row per file. + # No SQL limit here: ``limit`` bounds *files* for this path (applied after exploding, in + # ``load_catalog``), not datasets, so it cannot be pushed into this dataset-level query. stmt = self._dataset_query().options(selectinload(self.dataset_cls.files)) # type: ignore[attr-defined] datasets = db.session.execute(stmt).scalars().unique().all() @@ -479,8 +483,11 @@ def _get_dataset_files(self, db: Database) -> pd.DataFrame: index=[dataset.id for dataset in datasets for _file in dataset.files], ) - def _get_datasets(self, db: Database) -> pd.DataFrame: - result_datasets = db.session.execute(self._dataset_query()).scalars().unique().all() + def _get_datasets(self, db: Database, limit: int | None = None) -> pd.DataFrame: + stmt = self._dataset_query() + if limit is not None: + stmt = stmt.limit(limit) + result_datasets = db.session.execute(stmt).scalars().unique().all() return pd.DataFrame( [{k: getattr(dataset, k) for k in self.dataset_specific_metadata} for dataset in result_datasets], @@ -497,12 +504,16 @@ def load_catalog( operation for the `instance_id` column. Only the latest version of each dataset is returned. + Deduplication happens in SQL (``_dataset_query``), not in pandas. The index of the data catalog is the primary key of the dataset. This should be maintained during any processing. ``limit`` (when given) bounds the returned rows after deduplicating to the latest version, - so up to ``limit`` datasets are returned. + so up to ``limit`` datasets are returned when ``include_files=False`` (the limit is pushed + into the SQL query). When ``include_files=True``, ``limit`` bounds *files* instead (per the + CLI help text), so it is applied in Python after exploding datasets to one row per file -- + it is not pushed into the dataset-level query in that case. Returns ------- @@ -514,7 +525,7 @@ def load_catalog( if include_files: catalog = self._get_dataset_files(db) else: - catalog = self._get_datasets(db) + catalog = self._get_datasets(db, limit=limit) # If there are no datasets, return an empty DataFrame if catalog.empty: @@ -524,10 +535,11 @@ def load_catalog( # Convert start_time/end_time strings from DB to cftime objects catalog = coerce_catalog_times(catalog) - # Deduplicate to the latest version BEFORE applying the limit, so the limit is not spent on - # superseded versions (which would otherwise be dropped, returning fewer rows than requested). - catalog = self.filter_latest_versions(catalog) - if limit is not None: + # include_files=True: the SQL query already deduplicated to the latest version; the file-count + # limit is applied here, in Python, after exploding to one row per file (it bounds files, not + # datasets -- see the docstring). include_files=False: dedup and limit both already happened + # in SQL (``_get_datasets``), so this is a no-op pass-through. + if include_files and limit is not None: catalog = catalog.head(limit) return self._finalise_loaded_catalog(catalog) diff --git a/packages/climate-ref/src/climate_ref/models/dataset_query.py b/packages/climate-ref/src/climate_ref/models/dataset_query.py index d0cf1e75b..479eb7019 100644 --- a/packages/climate-ref/src/climate_ref/models/dataset_query.py +++ b/packages/climate-ref/src/climate_ref/models/dataset_query.py @@ -1,20 +1,17 @@ """ Query builder for the polymorphic ``Dataset`` hierarchy. -[select_datasets][climate_ref.models.dataset_query.select_datasets] is the single definition of -"the datasets query": it backs both ``climate_ref.datasets`` (``DatasetAdapter.load_catalog``) and +[select_datasets][climate_ref.models.dataset_query.select_datasets] +is the cononical definition for selecting datasets consistently. +It backs both ``climate_ref.datasets`` (``DatasetAdapter.load_catalog``) and the ``climate_ref.results`` read layer (``reader.datasets``), so the two cannot drift apart. - -It lives in the models layer -- below both of those -- and depends only on the model classes. -``source_type`` is resolved to its concrete subclass through SQLAlchemy's polymorphic map rather -than the adapter registry, so this module needs no import from ``climate_ref.datasets`` and the -import graph stays strictly top-down (no cycle). """ from collections.abc import Mapping, Sequence from typing import Any, cast import attrs +import sqlalchemy as sa from sqlalchemy import Select, select from climate_ref.models.dataset import Dataset @@ -63,12 +60,26 @@ def _entity_for(source_type: SourceDatasetType | None) -> type[Dataset]: return cast(type[Dataset], Dataset.__mapper__.polymorphic_map[source_type].class_) -def select_datasets(filter: DatasetFilter) -> Select[Any]: # noqa: A002 +def select_datasets( + filter: DatasetFilter, # noqa: A002 + *, + latest_group_by: Sequence[str] | None = None, +) -> Select[Any]: """ Build the ``Select`` over the (optionally concrete) ``Dataset`` entity for the given filter. - The latest-version filter and any limit are deliberately not applied here; callers apply them - so a numeric limit is not spent on superseded versions. + Any limit is deliberately not applied here; callers apply it so a numeric limit is not spent + on superseded versions. + + ``latest_group_by`` is the adapter's ``dataset_id_metadata`` -- the partition columns for the + latest-version window. It is optional because ``select_datasets`` lives in the models layer and + must not import the adapter registry, so it cannot look this up itself; callers pass it through. + + ``filter.latest_only`` is INERT unless ``latest_group_by`` is also given (non-empty): passing + ``latest_only=True`` alone does NOT dedup. Both must be set together for SQL-side deduplication + to apply. When both are set, rows are deduplicated with a ``RANK() OVER (PARTITION BY + ORDER BY version_key DESC)`` window (applied after all other filters/joins), + keeping every row tied at the maximum ``version_key`` -- so ties are not silently dropped. Raises ------ @@ -111,4 +122,27 @@ def select_datasets(filter: DatasetFilter) -> Select[Any]: # noqa: A002 if needs_execution_join: stmt = stmt.distinct() + if filter.latest_only and latest_group_by: + # Rank by version_key within each partition, then keep only ids at rank 1. Applied after all + # the where-filters/joins above so "latest" is chosen among the already-filtered set. + # + # This uses the ``entity.id.in_()`` shape rather than + # ``aliased(entity, subquery)``: making the outer entity an alias would break the class-bound + # ``selectinload(entity.files)`` loader option that callers attach to the returned ``Select`` + # (``ArgumentError`` at execution), since a bare ``Select`` gives callers no handle on the + # alias to rewrite their options against. Keeping the outer entity a plain class means all + # existing ``.options(...)`` calls keep working unchanged. + # + # RANK (not ROW_NUMBER) keeps every row tied at the max version_key in a partition. + rank = sa.func.rank().over( + partition_by=[getattr(entity, c) for c in latest_group_by], + order_by=entity.version_key.desc(), + ) + inner = stmt.add_columns(rank.label("_rank")).subquery() + latest_ids = select(inner.c.id).where(inner.c._rank == 1) + # Plain filtered select of the entity (not the ``stmt`` built above, which now carries the + # window's subquery machinery) -- the id membership already encodes every filter/join from + # above, so the outer select only needs the entity and the id predicate. + stmt = select(entity).where(entity.id.in_(latest_ids)) + return stmt.order_by(entity.updated_at.desc()) diff --git a/packages/climate-ref/src/climate_ref/results/datasets.py b/packages/climate-ref/src/climate_ref/results/datasets.py index 1fc9baef3..a848ded0b 100644 --- a/packages/climate-ref/src/climate_ref/results/datasets.py +++ b/packages/climate-ref/src/climate_ref/results/datasets.py @@ -126,37 +126,25 @@ def datasets( """ Query datasets, optionally scoped to a source type, execution or diagnostic. - ``limit`` (when given) is applied after ``latest_only`` filtering, over the ordered, latest - datasets, so it caps returned datasets -- matching ``adapter.load_catalog``'s dedup-then-limit - ordering. + Deduplication to the latest version (when ``filter.latest_only``) happens in SQL via a + ``RANK`` window, keyed off the adapter's ``dataset_id_metadata``. ``limit`` is pushed into + the same statement, so it is applied after dedup, over the ordered, latest datasets -- + matching ``adapter.load_catalog``'s dedup-then-limit ordering, and only fetching the rows + actually returned instead of the whole table. """ filter = filter or DatasetFilter() # noqa: A001 - entity: type[Dataset] = ( - get_dataset_adapter(filter.source_type.value).dataset_cls if filter.source_type else Dataset - ) - stmt = select_datasets(filter) + adapter = get_dataset_adapter(filter.source_type.value) if filter.source_type else None + entity: type[Dataset] = adapter.dataset_cls if adapter else Dataset + + latest_group_by = adapter.dataset_id_metadata if adapter is not None else None + stmt = select_datasets(filter, latest_group_by=latest_group_by) if include_files: stmt = stmt.options(selectinload(entity.files)) # type: ignore[attr-defined] + if limit is not None: + stmt = stmt.limit(limit) session = self._db.session rows = list(session.execute(stmt).scalars().unique().all()) - adapter = get_dataset_adapter(filter.source_type.value) if filter.source_type else None - if filter.latest_only and adapter is not None and adapter.dataset_id_metadata: - id_cols = adapter.dataset_id_metadata - version_col = adapter.version_metadata - # Reuse the adapter's own latest-version policy (its short-circuits + numeric compare) so - # there is a single definition of "latest version": feed it a minimal id/version frame - # indexed by dataset id and keep the survivors. - versions = pd.DataFrame( - [{**{c: getattr(r, c) for c in id_cols}, version_col: getattr(r, version_col)} for r in rows], - index=[r.id for r in rows], - ) - surviving_ids = set(adapter.filter_latest_versions(versions).index) - rows = [r for r in rows if r.id in surviving_ids] - - if limit is not None: - rows = rows[:limit] - return DatasetCollection(datasets=tuple(self._to_view(r, include_files=include_files) for r in rows)) diff --git a/packages/climate-ref/tests/unit/cli/test_datasets.py b/packages/climate-ref/tests/unit/cli/test_datasets.py index 3e73e3f12..12903f62d 100644 --- a/packages/climate-ref/tests/unit/cli/test_datasets.py +++ b/packages/climate-ref/tests/unit/cli/test_datasets.py @@ -82,6 +82,31 @@ def test_list_dataset_filter_invalid_facet(self, db_seeded, invoke_cli): expected_exit_code=1, ) + def test_list_include_files_limit_bounds_files_not_datasets(self, db_seeded, invoke_cli): + """``--limit`` bounds *files* (not datasets) when ``--include-files`` is set. + + Add a second file to one of the seeded datasets so at least one dataset has >1 file, + then confirm ``--limit`` set to fewer than the total file count still returns that many file rows + (i.e. the limit was not spent solely on datasets). + """ + dataset = db_seeded.session.query(CMIP6Dataset).first() + db_seeded.session.add( + DatasetFile( + dataset_id=dataset.id, + path="extra-file-for-limit-test.nc", + start_time="2000-01-01", + end_time="2000-12-31", + ) + ) + db_seeded.session.commit() + + total_files = db_seeded.session.query(DatasetFile).count() + assert total_files >= 2, "need at least 2 files in the DB for this test to be meaningful" + + result = invoke_cli(["datasets", "list", "--include-files", "--limit", "1", "--column", "path"]) + data_lines = [line for line in result.stdout.strip().split("\n")[2:] if line.strip()] + assert len(data_lines) == 1 + class TestDatasetsStats: def test_stats_basic(self, db_seeded, invoke_cli): diff --git a/packages/climate-ref/tests/unit/datasets/test_dataset_query.py b/packages/climate-ref/tests/unit/datasets/test_dataset_query.py new file mode 100644 index 000000000..aa0b79ae9 --- /dev/null +++ b/packages/climate-ref/tests/unit/datasets/test_dataset_query.py @@ -0,0 +1,252 @@ +"""Tests for the SQL-side latest-version dedup in ``select_datasets``. + +Covers: +1. ``select_datasets(..., latest_group_by=...)`` returns the same survivor set as the old pandas + ``filter_latest_versions`` path (numeric ties, non-conforming versions, combined with + finalised/facet/relationship filters -- including ``diagnostic_slug`` + ``latest_only`` together). +2. The base-entity (``source_type=None``) case stays a no-op. +3. The non-nullability invariant that the SQL dedup depends on: every adapter's + ``dataset_id_metadata`` columns are non-nullable and absent from ``columns_requiring_finalisation``. +""" + +import pytest +from sqlalchemy.orm import selectinload + +from climate_ref.datasets import get_dataset_adapter +from climate_ref.datasets.cmip6 import CMIP6DatasetAdapter +from climate_ref.datasets.cmip7 import CMIP7DatasetAdapter +from climate_ref.datasets.obs4mips import Obs4MIPsDatasetAdapter +from climate_ref.datasets.pmp_climatology import PMPClimatologyDatasetAdapter +from climate_ref.models.dataset import CMIP6Dataset, Dataset, DatasetFile +from climate_ref.models.dataset_query import DatasetFilter, select_datasets +from climate_ref.models.diagnostic import Diagnostic +from climate_ref.models.execution import Execution, ExecutionGroup, execution_datasets +from climate_ref_core.source_types import SourceDatasetType + + +def _make_cmip6( + *, + slug: str, + source_id: str = "TEST-MODEL", + variable_id: str = "tas", + version: str = "v1", + finalised: bool = True, +) -> CMIP6Dataset: + return CMIP6Dataset( + slug=slug, + dataset_type=SourceDatasetType.CMIP6, + activity_id="CMIP", + experiment_id="historical", + institution_id="TEST", + source_id=source_id, + member_id="r1i1p1f1", + table_id="Amon", + variable_id=variable_id, + grid_label="gn", + version=version, + instance_id=slug, + variant_label="r1i1p1f1", + finalised=finalised, + ) + + +@pytest.fixture +def db_with_versions(db_seeded): + """Add controlled version variety on top of the seeded DB: v2/v10 ties and a non-conforming version.""" + with db_seeded.session.begin(): + v2 = _make_cmip6(slug="dq.TEST-MODEL.tas.v2", version="v2") + v10 = _make_cmip6(slug="dq.TEST-MODEL.tas.v10", version="v10") + nonconforming = _make_cmip6(slug="dq.TEST-MODEL.pr.latest", variable_id="pr", version="latest") + unfinalised_v1 = _make_cmip6( + slug="dq.OTHER-MODEL.tas.v1", source_id="OTHER-MODEL", version="v1", finalised=False + ) + db_seeded.session.add_all([v2, v10, nonconforming, unfinalised_v1]) + db_seeded.session.flush() + + db_seeded.session.add( + DatasetFile(dataset_id=v10.id, path="dq_v10.nc", start_time="2000-01-01", end_time="2000-12-31") + ) + + diag = db_seeded.session.query(Diagnostic).first() + eg = ExecutionGroup(key="dq-key", diagnostic_id=diag.id, selectors={}) + db_seeded.session.add(eg) + db_seeded.session.flush() + execution = Execution( + execution_group_id=eg.id, successful=True, output_fragment="dq-out", dataset_hash="dq-hash" + ) + db_seeded.session.add(execution) + db_seeded.session.flush() + db_seeded.session.execute( + execution_datasets.insert(), [{"execution_id": execution.id, "dataset_id": v10.id}] + ) + + db_seeded.session.commit() + db_seeded.dq_ids = { + "v2": v2.id, + "v10": v10.id, + "nonconforming": nonconforming.id, + "unfinalised_v1": unfinalised_v1.id, + } + db_seeded.dq_execution_id = execution.id + db_seeded.dq_diagnostic_slug = diag.slug + return db_seeded + + +ID_COLS = CMIP6DatasetAdapter.dataset_id_metadata + + +class TestSelectDatasetsLatestVersionDedup: + def test_numeric_tie_keeps_v10_over_v2(self, db_with_versions): + stmt = select_datasets( + DatasetFilter(source_type=SourceDatasetType.CMIP6, facets={"source_id": ("TEST-MODEL",)}), + latest_group_by=ID_COLS, + ) + rows = db_with_versions.session.execute(stmt).scalars().unique().all() + ids = {r.id for r in rows} + assert db_with_versions.dq_ids["v10"] in ids + assert db_with_versions.dq_ids["v2"] not in ids + + def test_ties_at_max_version_both_survive(self, db_with_versions): + """RANK (not ROW_NUMBER): two rows tied at the max ``version_key`` in a partition both survive. + + Pins the tie behaviour at the SQL layer so a future ``rank()`` -> ``row_number()`` regression is + caught in CI, not only by the pandas parity tests. + """ + tie_a = _make_cmip6(slug="dq.TIE-MODEL.tas.v10.a", source_id="TIE-MODEL", version="v10") + tie_b = _make_cmip6(slug="dq.TIE-MODEL.tas.v10.b", source_id="TIE-MODEL", version="v10") + older = _make_cmip6(slug="dq.TIE-MODEL.tas.v2", source_id="TIE-MODEL", version="v2") + db_with_versions.session.add_all([tie_a, tie_b, older]) + db_with_versions.session.flush() + tie_ids = {tie_a.id, tie_b.id} + + stmt = select_datasets( + DatasetFilter(source_type=SourceDatasetType.CMIP6, facets={"source_id": ("TIE-MODEL",)}), + latest_group_by=ID_COLS, + ) + rows = db_with_versions.session.execute(stmt).scalars().unique().all() + assert {r.id for r in rows} == tie_ids + + def test_non_conforming_version_survives_alone_in_its_group(self, db_with_versions): + stmt = select_datasets( + DatasetFilter( + source_type=SourceDatasetType.CMIP6, + facets={"source_id": ("TEST-MODEL",), "variable_id": ("pr",)}, + ), + latest_group_by=ID_COLS, + ) + rows = db_with_versions.session.execute(stmt).scalars().unique().all() + assert {r.id for r in rows} == {db_with_versions.dq_ids["nonconforming"]} + + def test_combined_with_finalised_filter(self, db_with_versions): + stmt = select_datasets( + DatasetFilter(source_type=SourceDatasetType.CMIP6, finalised=False), + latest_group_by=ID_COLS, + ) + rows = db_with_versions.session.execute(stmt).scalars().unique().all() + assert {r.id for r in rows} == {db_with_versions.dq_ids["unfinalised_v1"]} + + def test_combined_with_execution_id_and_diagnostic_slug(self, db_with_versions): + """diagnostic_slug + latest_only together must not emit invalid SQL (duplicate self-join).""" + stmt = select_datasets( + DatasetFilter( + source_type=SourceDatasetType.CMIP6, + execution_id=db_with_versions.dq_execution_id, + diagnostic_slug=db_with_versions.dq_diagnostic_slug, + ), + latest_group_by=ID_COLS, + ) + rows = db_with_versions.session.execute(stmt).scalars().unique().all() + assert {r.id for r in rows} == {db_with_versions.dq_ids["v10"]} + + def test_selectinload_files_still_works(self, db_with_versions): + """The ``id IN (subquery)`` shape keeps the outer entity a plain class, so class-bound + loader options attached by callers (e.g. ``load_catalog``, ``reader.datasets``) still work.""" + stmt = select_datasets( + DatasetFilter(source_type=SourceDatasetType.CMIP6, facets={"source_id": ("TEST-MODEL",)}), + latest_group_by=ID_COLS, + ).options(selectinload(CMIP6Dataset.files)) + rows = db_with_versions.session.execute(stmt).scalars().unique().all() + v10 = next(r for r in rows if r.id == db_with_versions.dq_ids["v10"]) + assert [f.path for f in v10.files] == ["dq_v10.nc"] + + def test_latest_only_false_ignores_latest_group_by(self, db_with_versions): + stmt = select_datasets( + DatasetFilter( + source_type=SourceDatasetType.CMIP6, + facets={"source_id": ("TEST-MODEL",)}, + latest_only=False, + ), + latest_group_by=ID_COLS, + ) + rows = db_with_versions.session.execute(stmt).scalars().unique().all() + ids = {r.id for r in rows} + assert db_with_versions.dq_ids["v2"] in ids + assert db_with_versions.dq_ids["v10"] in ids + + def test_latest_group_by_none_is_inert(self, db_with_versions): + """``latest_only=True`` alone (no ``latest_group_by``) must not dedup -- the two-flag API.""" + stmt = select_datasets( + DatasetFilter(source_type=SourceDatasetType.CMIP6, facets={"source_id": ("TEST-MODEL",)}) + ) + rows = db_with_versions.session.execute(stmt).scalars().unique().all() + ids = {r.id for r in rows} + assert db_with_versions.dq_ids["v2"] in ids + assert db_with_versions.dq_ids["v10"] in ids + + def test_base_entity_no_op(self, db_with_versions): + """``source_type=None`` stays a no-op even if a ``latest_group_by`` were (incorrectly) passed, + because there are no ``dataset_id_metadata`` columns on the base ``Dataset`` entity.""" + stmt_no_dedup = select_datasets(DatasetFilter()) + stmt_with_group_by = select_datasets(DatasetFilter(), latest_group_by=()) + rows_a = db_with_versions.session.execute(stmt_no_dedup).scalars().unique().all() + rows_b = db_with_versions.session.execute(stmt_with_group_by).scalars().unique().all() + assert len(rows_a) == len(rows_b) + + +class TestDatasetIdMetadataNonNullabilityInvariant: + """Pin the assumption the SQL dedup depends on: every adapter's ``dataset_id_metadata`` column + is non-nullable on the model AND absent from ``columns_requiring_finalisation``. + + An actual NULL insert would raise ``IntegrityError`` (NOT NULL constraint), so this is pinned + structurally instead of by attempting a NULL insert. + """ + + @pytest.mark.parametrize( + "adapter_cls", + [CMIP6DatasetAdapter, CMIP7DatasetAdapter, Obs4MIPsDatasetAdapter, PMPClimatologyDatasetAdapter], + ) + def test_dataset_id_metadata_columns_are_non_nullable(self, adapter_cls): + adapter = adapter_cls() + entity: type[Dataset] = adapter.dataset_cls + for column_name in adapter.dataset_id_metadata: + column = entity.__mapper__.columns[column_name] + assert not column.nullable, ( + f"{entity.__name__}.{column_name} is nullable but is a dataset_id_metadata " + "partition column for the SQL latest-version window; a NULL value would silently " + "break RANK partitioning." + ) + + @pytest.mark.parametrize( + "adapter_cls", + [CMIP6DatasetAdapter, CMIP7DatasetAdapter, Obs4MIPsDatasetAdapter, PMPClimatologyDatasetAdapter], + ) + def test_dataset_id_metadata_disjoint_from_finalisation_columns(self, adapter_cls): + adapter = adapter_cls() + overlap = set(adapter.dataset_id_metadata) & adapter.columns_requiring_finalisation + assert not overlap, ( + f"{adapter_cls.__name__}: dataset_id_metadata columns {overlap} require finalisation, " + "so they may be NA pre-finalisation, which would break the SQL latest-version window." + ) + + +def test_get_dataset_adapter_covers_all_four_subclasses(): + """Sanity check that all four Dataset subclasses are reachable via the adapter registry + (keeps the parametrised tests above from silently under-covering a fifth subclass added later).""" + for source_type in ( + SourceDatasetType.CMIP6, + SourceDatasetType.CMIP7, + SourceDatasetType.obs4MIPs, + SourceDatasetType.PMPClimatology, + ): + adapter = get_dataset_adapter(source_type.value) + assert adapter.dataset_id_metadata diff --git a/packages/climate-ref/tests/unit/datasets/test_datasets.py b/packages/climate-ref/tests/unit/datasets/test_datasets.py index b6c6cf692..6af8e1ade 100644 --- a/packages/climate-ref/tests/unit/datasets/test_datasets.py +++ b/packages/climate-ref/tests/unit/datasets/test_datasets.py @@ -10,7 +10,7 @@ from climate_ref.datasets import base as base_module from climate_ref.datasets.base import DatasetAdapter, _is_na from climate_ref.datasets.cmip6 import CMIP6DatasetAdapter -from climate_ref.models.dataset import CMIP6Dataset, DatasetFile +from climate_ref.models.dataset import CMIP6Dataset, Dataset, DatasetFile, _sync_version_key from climate_ref_core.datasets import SourceDatasetType from climate_ref_core.exceptions import RefException @@ -435,6 +435,115 @@ def test_register_dataset_updates_dataset_metadata(monkeypatch, test_db): assert dataset.grid_label == "gr2" +class TestVersionKeySync: + """The ``before_insert``/``before_update`` mapper event keeps ``version_key`` in sync.""" + + def test_register_dataset_sets_version_key(self, monkeypatch, test_db): + """``register_dataset`` (the normal ingest path) picks up the event too.""" + adapter, db = test_db + + df = _mk_df(rows=[{"path": "f1.nc", "start_time": "2001-01-01", "end_time": "2001-12-31"}]) + df.loc[:, "version"] = "v10" + + with db.session.begin(): + adapter.register_dataset(db=db, data_catalog_dataset=df) + + dataset = db.session.query(CMIP6Dataset).filter_by(slug="CESM2.tas.gn").first() + assert dataset.version_key == 10 + + def test_direct_orm_insert_sets_version_key(self, test_db): + """Direct-ORM inserts (e.g. test fixtures) get a correct key without calling register_dataset.""" + _, db = test_db + + ds = CMIP6Dataset( + slug="direct-orm-v2", + dataset_type=SourceDatasetType.CMIP6, + activity_id="CMIP", + experiment_id="historical", + institution_id="TEST", + source_id="TEST-MODEL", + member_id="r1i1p1f1", + table_id="Amon", + variable_id="tas", + grid_label="gn", + version="v2", + instance_id="direct-orm-v2", + variant_label="r1i1p1f1", + ) + with db.session.begin(): + db.session.add(ds) + + assert ds.version_key == 2 + + def test_direct_orm_insert_numeric_v10_beats_v2(self, test_db): + """Regression guard for the review blocker: v2/v10 fixtures must not tie at version_key=-1.""" + _, db = test_db + + def _make(slug: str, version: str) -> CMIP6Dataset: + return CMIP6Dataset( + slug=slug, + dataset_type=SourceDatasetType.CMIP6, + activity_id="CMIP", + experiment_id="historical", + institution_id="TEST", + source_id="TEST-MODEL", + member_id="r1i1p1f1", + table_id="Amon", + variable_id="tas", + grid_label="gn", + version=version, + instance_id=slug, + variant_label="r1i1p1f1", + ) + + v2 = _make("regression-v2", "v2") + v10 = _make("regression-v10", "v10") + with db.session.begin(): + db.session.add_all([v2, v10]) + + assert v2.version_key == 2 + assert v10.version_key == 10 + assert v10.version_key > v2.version_key + + def test_no_version_attribute_backstops_to_minus_one(self): + """A target with no ``version`` attribute keeps the ``-1`` backstop. + + Exercises the event function directly (rather than a full DB round-trip) since ``Dataset`` + itself has no ``polymorphic_identity`` and is not meant to be inserted standalone. + """ + target = Dataset(slug="base-only") + _sync_version_key(mapper=None, connection=None, target=target) + + assert target.version_key == -1 + + def test_update_resyncs_version_key(self, test_db): + """Updating ``version`` on an existing row recomputes ``version_key`` via before_update.""" + _, db = test_db + + ds = CMIP6Dataset( + slug="update-me", + dataset_type=SourceDatasetType.CMIP6, + activity_id="CMIP", + experiment_id="historical", + institution_id="TEST", + source_id="TEST-MODEL", + member_id="r1i1p1f1", + table_id="Amon", + variable_id="tas", + grid_label="gn", + version="v2", + instance_id="update-me", + variant_label="r1i1p1f1", + ) + with db.session.begin(): + db.session.add(ds) + assert ds.version_key == 2 + + ds.version = "v10" + db.session.commit() + assert ds.version_key == 10 + + class TestFilterLatestVersions: """Tests for DatasetAdapter.filter_latest_versions.""" From 0817825e84884f8a6726b279814272f598e738e6 Mon Sep 17 00:00:00 2001 From: Jared Lewis Date: Thu, 2 Jul 2026 15:48:36 +1000 Subject: [PATCH 10/23] feat(results): add reader.diagnostics and ref diagnostics list Add a diagnostics read domain reachable via reader.diagnostics, with a list() returning provider/slug/name/promoted_version, an all-versions execution-group count, and promoted-version successful/inflight/total counts merged in from the existing execution-statistics builder so the status-count logic stays defined once. stats() delegates to that same builder rather than duplicating the aggregate SQL. Expose the domain through a new ref diagnostics list command that renders the collection and documents every column in its help text. Extract the shared bare-str-rejecting _as_str_tuple converter into results/_converters so the filter DTOs no longer copy it per module. --- .../src/climate_ref/cli/__init__.py | 3 + .../src/climate_ref/cli/diagnostics.py | 84 ++++++ .../src/climate_ref/results/__init__.py | 6 +- .../src/climate_ref/results/_converters.py | 19 ++ .../src/climate_ref/results/diagnostics.py | 281 ++++++++++++++++++ .../src/climate_ref/results/executions.py | 17 +- .../src/climate_ref/results/values.py | 11 +- .../tests/unit/cli/test_diagnostics.py | 101 +++++++ .../climate-ref/tests/unit/cli/test_root.py | 2 +- .../tests/unit/results/test_diagnostics.py | 216 ++++++++++++++ 10 files changed, 721 insertions(+), 19 deletions(-) create mode 100644 packages/climate-ref/src/climate_ref/cli/diagnostics.py create mode 100644 packages/climate-ref/src/climate_ref/results/_converters.py create mode 100644 packages/climate-ref/src/climate_ref/results/diagnostics.py create mode 100644 packages/climate-ref/tests/unit/cli/test_diagnostics.py create mode 100644 packages/climate-ref/tests/unit/results/test_diagnostics.py diff --git a/packages/climate-ref/src/climate_ref/cli/__init__.py b/packages/climate-ref/src/climate_ref/cli/__init__.py index dbbe2aaaa..6bb838815 100644 --- a/packages/climate-ref/src/climate_ref/cli/__init__.py +++ b/packages/climate-ref/src/climate_ref/cli/__init__.py @@ -32,6 +32,7 @@ ("db", "heads"), ("db", "history"), ("db", "tables"), + ("diagnostics", "list"), ("executions", "list-groups"), ("executions", "inspect"), ("executions", "stats"), @@ -221,6 +222,7 @@ def build_app() -> typer.Typer: config, datasets, db, + diagnostics, executions, providers, solve, @@ -233,6 +235,7 @@ def build_app() -> typer.Typer: app.add_typer(config.app, name="config") app.add_typer(datasets.app, name="datasets") app.add_typer(db.app, name="db") + app.add_typer(diagnostics.app, name="diagnostics") app.add_typer(executions.app, name="executions") app.add_typer(providers.app, name="providers") app.add_typer(test_cases.app, name="test-cases") diff --git a/packages/climate-ref/src/climate_ref/cli/diagnostics.py b/packages/climate-ref/src/climate_ref/cli/diagnostics.py new file mode 100644 index 000000000..b5fa2dfe0 --- /dev/null +++ b/packages/climate-ref/src/climate_ref/cli/diagnostics.py @@ -0,0 +1,84 @@ +""" +View diagnostic metadata +""" + +from typing import Annotated + +import typer +from loguru import logger + +from climate_ref.cli._utils import OutputFormat, render_dataframe +from climate_ref.results import DiagnosticFilter, Reader + +app = typer.Typer(help=__doc__) + + +@app.command(name="list") +def list_( # noqa: PLR0913 + ctx: typer.Context, + provider: Annotated[ + list[str] | None, + typer.Option( + help="Filter by provider slug (substring match, case-insensitive)." + "Multiple values can be provided." + ), + ] = None, + diagnostic: Annotated[ + list[str] | None, + typer.Option( + help="Filter by diagnostic slug (substring match, case-insensitive)." + "Multiple values can be provided." + ), + ] = None, + column: Annotated[ + list[str] | None, + typer.Option(help="Only include specified columns in the output"), + ] = None, + limit: int = typer.Option(100, help="Limit the number of rows to display"), + output_format: Annotated[ + OutputFormat, + typer.Option( + "--format", + help="Output format: 'table' (default) or machine-readable 'json'.", + ), + ] = OutputFormat.table, +) -> None: + """ + List the registered diagnostics. + + Columns: + + - provider: the provider slug. + - diagnostic: the diagnostic slug. + - name: the human-readable diagnostic name. + - promoted_version: the currently promoted diagnostic version. + - execution_group_count: execution groups created for it across all versions. + - successful: promoted-version execution groups whose latest execution succeeded. + - inflight: promoted-version execution groups whose latest execution is still running. + - total: execution groups at the promoted version. + """ + console = ctx.obj.console + reader = Reader(ctx.obj.database) + + collection = reader.diagnostics.list( + DiagnosticFilter(provider_contains=provider, diagnostic_contains=diagnostic), + limit=limit, + ) + + results_df = collection.to_pandas() + + if column: + if not results_df.empty: + if not all(col in results_df.columns for col in column): + logger.error(f"Column not found in data catalog: {column}") + raise typer.Exit(code=1) + results_df = results_df[column] + + render_dataframe(results_df, console=console, output_format=output_format) + + filtered_count = collection.total_count + if filtered_count > limit: + logger.warning( + f"Displaying {limit} of {filtered_count} filtered results. " + f"Use the `--limit` option to display more." + ) diff --git a/packages/climate-ref/src/climate_ref/results/__init__.py b/packages/climate-ref/src/climate_ref/results/__init__.py index 1fe5068ef..3ef0cf02b 100644 --- a/packages/climate-ref/src/climate_ref/results/__init__.py +++ b/packages/climate-ref/src/climate_ref/results/__init__.py @@ -17,6 +17,8 @@ the ``ExecutionsReader`` facade. * [datasets][climate_ref.results.datasets] -- dataset DTOs + collection + the ``DatasetsReader`` facade, querying the polymorphic ``Dataset`` hierarchy directly. +* [diagnostics][climate_ref.results.diagnostics] -- diagnostic DTOs + collection + the + ``DiagnosticsReader`` facade, querying ``Diagnostic -> Provider`` directly. * [artifacts][climate_ref.results.artifacts] -- the ``ArtifactsReader`` facade, resolving execution output fragments into filesystem paths under a results root. @@ -37,7 +39,7 @@ * the ``Reader`` entry point, * filter objects you construct and pass in (``MetricValueFilter``, ``ExecutionGroupFilter``, - ``DatasetFilter``), + ``DatasetFilter``, ``DiagnosticFilter``), * value objects you pass in (``OutlierPolicy``). Everything the package *returns* -- DTOs, collections, views -- and the sub-reader classes reached @@ -50,12 +52,14 @@ from climate_ref.results._query import MetricValueFilter from climate_ref.results.datasets import DatasetFilter +from climate_ref.results.diagnostics import DiagnosticFilter from climate_ref.results.executions import ExecutionGroupFilter from climate_ref.results.outliers import OutlierPolicy from climate_ref.results.values import Reader __all__ = [ "DatasetFilter", + "DiagnosticFilter", "ExecutionGroupFilter", "MetricValueFilter", "OutlierPolicy", diff --git a/packages/climate-ref/src/climate_ref/results/_converters.py b/packages/climate-ref/src/climate_ref/results/_converters.py new file mode 100644 index 000000000..9feb876ec --- /dev/null +++ b/packages/climate-ref/src/climate_ref/results/_converters.py @@ -0,0 +1,19 @@ +"""Shared ``attrs`` field converters for the results filter DTOs.""" + +from collections.abc import Sequence + + +def _as_str_tuple(value: Sequence[str] | None) -> tuple[str, ...] | None: + """ + Coerce a multi-value filter field to ``tuple[str, ...]``, rejecting a bare ``str``. + + A bare ``str`` is itself a ``Sequence[str]``, so without this guard a caller passing + ``diagnostic_contains="enso"`` would have it iterated character-by-character + (``"enso"`` -> ``e``, ``n``, ``s``, ``o``) before it ever reaches ``ilike`` / + the facet matcher. + """ + if value is None: + return None + if isinstance(value, str): + raise TypeError("Expected a sequence of strings, not a bare str. Wrap it in a list/tuple.") + return tuple(value) diff --git a/packages/climate-ref/src/climate_ref/results/diagnostics.py b/packages/climate-ref/src/climate_ref/results/diagnostics.py new file mode 100644 index 000000000..3ad944d63 --- /dev/null +++ b/packages/climate-ref/src/climate_ref/results/diagnostics.py @@ -0,0 +1,281 @@ +""" +Typed, detached read surface for diagnostic metadata. + +[DiagnosticsReader][climate_ref.results.diagnostics.DiagnosticsReader] is reached via +[Reader.diagnostics][climate_ref.results.values.Reader.diagnostics]. +It executes the [select_diagnostics][climate_ref.results.diagnostics.select_diagnostics] query +(joining ``Diagnostic -> Provider`` and counting execution groups per diagnostic) +and maps the rows into detached DTOs that outlive the session. +""" + +from collections.abc import Iterator, Mapping, Sequence +from typing import Any + +import attrs +import pandas as pd +from sqlalchemy import Select, func, or_, select +from sqlalchemy.orm import Session + +from climate_ref.database import Database +from climate_ref.models.diagnostic import Diagnostic +from climate_ref.models.execution import ExecutionGroup +from climate_ref.models.provider import Provider +from climate_ref.results._converters import _as_str_tuple +from climate_ref.results.executions import ExecutionStats, select_execution_statistics + + +@attrs.frozen(kw_only=True) +class DiagnosticFilter: + """ + Declarative filter over diagnostics. + + Every field is optional; ``None`` means "do not constrain on this axis". + ``diagnostic_contains``/``provider_contains`` are case-insensitive substring matches + (OR-combined within each field), matching the semantics used by + [ExecutionGroupFilter][climate_ref.results.executions.ExecutionGroupFilter]. + """ + + provider_contains: tuple[str, ...] | None = attrs.field(default=None, converter=_as_str_tuple) + diagnostic_contains: tuple[str, ...] | None = attrs.field(default=None, converter=_as_str_tuple) + + +@attrs.frozen(kw_only=True) +class DiagnosticView: + """ + A single diagnostic with information about the currently promoted version and execution-group counts. + + ``execution_group_count`` counts execution groups across every diagnostic version, + while ``successful``/``inflight``/``total`` are scoped to the currently ``promoted_version``. + """ + + provider_slug: str + """Slug of the provider that owns the diagnostic.""" + + slug: str + """The diagnostic's own slug, unique within its provider.""" + + name: str + """Human-readable diagnostic name.""" + + promoted_version: int + """Diagnostic version currently promoted for production.""" + + execution_group_count: int + """Execution groups for the diagnostic across every version.""" + + successful: int + """Promoted-version execution groups whose latest execution succeeded.""" + + inflight: int + """Promoted-version execution groups whose latest execution is still running (outcome not yet known).""" + + total: int + """Execution groups at the promoted version.""" + + +@attrs.frozen(kw_only=True) +class DiagnosticCollection: + """ + An immutable page of diagnostics plus collection-level metadata. + + ``total_count`` is the number of diagnostics matching the filter *before* pagination, so a + caller can tell there are more rows than the returned page. ``offset``/``limit`` echo back the + pagination applied to produce ``items`` (``limit`` is ``None`` when the whole result was + returned). + """ + + items: tuple[DiagnosticView, ...] + """The diagnostics on this page.""" + + total_count: int + """Total diagnostics matching the filter before ``offset``/``limit``.""" + + offset: int + """Rows skipped before this page.""" + + limit: int | None + """Page size requested, or ``None`` when the whole result was returned.""" + + def __iter__(self) -> Iterator[DiagnosticView]: + return iter(self.items) + + def __len__(self) -> int: + return len(self.items) + + def to_pandas(self) -> pd.DataFrame: + """ + DataFrame mirroring the ``diagnostics list`` CLI columns. + + Columns are emitted explicitly (``provider, diagnostic, name, promoted_version, + execution_group_count, successful, inflight, total``) even when the collection is empty, so + callers can select columns / build an empty table without special-casing. + """ + columns = [ + "provider", + "diagnostic", + "name", + "promoted_version", + "execution_group_count", + "successful", + "inflight", + "total", + ] + records = [ + { + "provider": d.provider_slug, + "diagnostic": d.slug, + "name": d.name, + "promoted_version": d.promoted_version, + "execution_group_count": d.execution_group_count, + "successful": d.successful, + "inflight": d.inflight, + "total": d.total, + } + for d in self.items + ] + return pd.DataFrame.from_records(records, columns=columns) + + +def select_diagnostics(filter: DiagnosticFilter | None = None) -> Select[Any]: # noqa: A002 + """ + Build the ``Select`` for diagnostics joined to their provider, with an execution-group count. + + ``execution_group_count`` counts every ``ExecutionGroup`` row for the diagnostic + (all versions, not scoped to ``promoted_version``), + so it reflects the diagnostic's full execution history. + Ordered by ``(Provider.slug, Diagnostic.slug)`` for stable output. + """ + filter = filter or DiagnosticFilter() # noqa: A001 + + group_count_subquery = ( + select( + ExecutionGroup.diagnostic_id, + func.count(ExecutionGroup.id).label("execution_group_count"), + ) + .group_by(ExecutionGroup.diagnostic_id) + .subquery() + ) + + stmt = ( + select( + Provider.slug.label("provider_slug"), + Diagnostic.slug.label("slug"), + Diagnostic.name.label("name"), + Diagnostic.promoted_version.label("promoted_version"), + func.coalesce(group_count_subquery.c.execution_group_count, 0).label("execution_group_count"), + ) + .join(Provider, Diagnostic.provider_id == Provider.id) + .outerjoin(group_count_subquery, Diagnostic.id == group_count_subquery.c.diagnostic_id) + .order_by(Provider.slug, Diagnostic.slug) + ) + + if filter.provider_contains: + stmt = stmt.where(or_(*(Provider.slug.ilike(f"%{s.lower()}%") for s in filter.provider_contains))) + if filter.diagnostic_contains: + stmt = stmt.where(or_(*(Diagnostic.slug.ilike(f"%{s.lower()}%") for s in filter.diagnostic_contains))) + + return stmt + + +class DiagnosticsReader: + """ + Diagnostic read domain. + + Constructed from a [Database][climate_ref.database.Database], which owns the session. + All read methods return detached DTOs that outlive the session. + """ + + def __init__(self, database: Database) -> None: + self._db = database + + @property + def session(self) -> Session: + """The underlying database session.""" + return self._db.session + + def _to_view(self, row: Any, stats_by_key: Mapping[tuple[str, str], ExecutionStats]) -> DiagnosticView: + stat = stats_by_key.get((row.provider_slug, row.slug)) + return DiagnosticView( + provider_slug=row.provider_slug, + slug=row.slug, + name=row.name, + promoted_version=row.promoted_version, + execution_group_count=row.execution_group_count, + successful=stat.successful if stat is not None else 0, + inflight=stat.running if stat is not None else 0, + total=stat.total if stat is not None else 0, + ) + + def list( + self, + filter: DiagnosticFilter | None = None, # noqa: A002 + *, + offset: int = 0, + limit: int | None = None, + ) -> DiagnosticCollection: + """ + Query diagnostics, with their provider, execution-group count, and promoted-version stats. + + Pagination is applied in SQL (``offset``/``limit``), with ``total_count`` computed from a + separate unpaged count query over the same filtered statement. + Promoted-version status counts (``successful``/``inflight``/``total``) are merged in from + [stats][climate_ref.results.diagnostics.DiagnosticsReader.stats], + and keyed by ``(provider, diagnostic)`` so status-count logic stays defined once. + + Diagnostics with no execution groups at the promoted version get zeros. + """ + filter = filter or DiagnosticFilter() # noqa: A001 + + base_stmt = select_diagnostics(filter) + count_stmt = select(func.count()).select_from(base_stmt.subquery()) + total_count = self.session.execute(count_stmt).scalar_one() + + stmt = base_stmt + if limit is not None: + stmt = stmt.offset(offset).limit(limit) + elif offset: + stmt = stmt.offset(offset) + + rows = self.session.execute(stmt).all() + + stats = self.stats( + provider_contains=filter.provider_contains, + diagnostic_contains=filter.diagnostic_contains, + ) + stats_by_key = {(s.provider, s.diagnostic): s for s in stats} + items = tuple(self._to_view(row, stats_by_key) for row in rows) + + return DiagnosticCollection(items=items, total_count=total_count, offset=offset, limit=limit) + + def stats( + self, + *, + diagnostic_contains: Sequence[str] | None = None, + provider_contains: Sequence[str] | None = None, + ) -> tuple[ExecutionStats, ...]: + """ + Per-(provider, diagnostic) execution status counts. + + Reuses [select_execution_statistics][climate_ref.results.executions.select_execution_statistics] + rather than duplicating the aggregate SQL, so status-count logic stays defined once. + This has the same signature and behaviour as + [ExecutionsReader.statistics][climate_ref.results.executions.ExecutionsReader.statistics], + just reachable under the diagnostics domain. + """ + stmt = select_execution_statistics( + diagnostic_contains=diagnostic_contains, provider_contains=provider_contains + ) + rows = self.session.execute(stmt).all() + return tuple( + ExecutionStats( + provider=row.provider, + diagnostic=row.diagnostic, + running=row.running, + failed=row.failed, + successful=row.successful, + not_started=row.not_started, + dirty=row.dirty, + total=row.total, + ) + for row in rows + ) diff --git a/packages/climate-ref/src/climate_ref/results/executions.py b/packages/climate-ref/src/climate_ref/results/executions.py index 9df078b68..34f03970b 100644 --- a/packages/climate-ref/src/climate_ref/results/executions.py +++ b/packages/climate-ref/src/climate_ref/results/executions.py @@ -34,25 +34,10 @@ get_execution_group_and_latest_filtered, ) from climate_ref.models.provider import Provider +from climate_ref.results._converters import _as_str_tuple from climate_ref.results._query import latest_execution_for_group -def _as_str_tuple(value: Sequence[str] | None) -> tuple[str, ...] | None: - """ - Coerce a multi-value filter field to ``tuple[str, ...]``, rejecting a bare ``str``. - - A bare ``str`` is itself a ``Sequence[str]``, so without this guard a caller passing - ``diagnostic_contains="enso"`` would have it iterated character-by-character - (``"enso"`` -> ``e``, ``n``, ``s``, ``o``) before it ever reaches ``ilike`` / - the facet matcher. - """ - if value is None: - return None - if isinstance(value, str): - raise TypeError("Expected a sequence of strings, not a bare str. Wrap it in a list/tuple.") - return tuple(value) - - def _as_facets( value: Mapping[str, Sequence[str]] | None, ) -> Mapping[str, tuple[str, ...]] | None: diff --git a/packages/climate-ref/src/climate_ref/results/values.py b/packages/climate-ref/src/climate_ref/results/values.py index a5938c105..051f8f64e 100644 --- a/packages/climate-ref/src/climate_ref/results/values.py +++ b/packages/climate-ref/src/climate_ref/results/values.py @@ -40,6 +40,7 @@ if TYPE_CHECKING: from climate_ref.results.artifacts import ArtifactsReader from climate_ref.results.datasets import DatasetsReader + from climate_ref.results.diagnostics import DiagnosticsReader def _kind_of(dimensions: Mapping[str, str]) -> str: @@ -403,7 +404,8 @@ class Reader: read-only story. This is a thin entry point: it exposes per-domain sub-readers as properties, [values][climate_ref.results.values.Reader.values] for metric-value reads, [executions][climate_ref.results.values.Reader.executions] for execution-group reads, - [datasets][climate_ref.results.values.Reader.datasets] for dataset reads, and + [datasets][climate_ref.results.values.Reader.datasets] for dataset reads, + [diagnostics][climate_ref.results.values.Reader.diagnostics] for diagnostic reads, and [artifacts][climate_ref.results.values.Reader.artifacts] for output path resolution (only available when a ``results`` root is supplied). """ @@ -434,6 +436,13 @@ def datasets(self) -> "DatasetsReader": return DatasetsReader(self._db) + @functools.cached_property + def diagnostics(self) -> "DiagnosticsReader": + """Diagnostic reads.""" + from climate_ref.results.diagnostics import DiagnosticsReader # noqa: PLC0415 + + return DiagnosticsReader(self._db) + @functools.cached_property def artifacts(self) -> "ArtifactsReader": """ diff --git a/packages/climate-ref/tests/unit/cli/test_diagnostics.py b/packages/climate-ref/tests/unit/cli/test_diagnostics.py new file mode 100644 index 000000000..4f1f1c1f5 --- /dev/null +++ b/packages/climate-ref/tests/unit/cli/test_diagnostics.py @@ -0,0 +1,101 @@ +import json + +import pytest +from climate_ref_esmvaltool import provider as esmvaltool_provider +from climate_ref_pmp import provider as pmp_provider + +from climate_ref.models.diagnostic import Diagnostic +from climate_ref.models.execution import ExecutionGroup +from climate_ref.provider_registry import _register_provider + + +@pytest.fixture +def db_with_diagnostics(db_seeded): + """A seeded DB with two providers and execution groups spread unevenly across diagnostics.""" + with db_seeded.session.begin(): + _register_provider(db_seeded, pmp_provider) + _register_provider(db_seeded, esmvaltool_provider) + + diag_1 = db_seeded.session.query(Diagnostic).filter_by(slug="enso_tel").first() + diag_2 = ( + db_seeded.session.query(Diagnostic) + .filter_by(slug="extratropical-modes-of-variability-nao") + .first() + ) + + db_seeded.session.add(ExecutionGroup(key="key1", diagnostic_id=diag_1.id, selectors={})) + db_seeded.session.add(ExecutionGroup(key="key2", diagnostic_id=diag_1.id, selectors={})) + db_seeded.session.add(ExecutionGroup(key="key3", diagnostic_id=diag_2.id, selectors={})) + + db_seeded.session.commit() + return db_seeded + + +def test_diagnostics_help(invoke_cli): + result = invoke_cli(["diagnostics", "--help"]) + + assert "View diagnostic metadata" in result.stdout + + +class TestDiagnosticsList: + def test_list(self, db_with_diagnostics, invoke_cli): + result = invoke_cli(["diagnostics", "list"]) + + assert "enso_tel" in result.stdout + assert "extratropical-modes-of-variability-nao" in result.stdout + assert "pmp" in result.stdout + assert "execution_group_count" in result.stdout + + def test_list_shows_counts(self, db_with_diagnostics, invoke_cli): + result = invoke_cli(["diagnostics", "list", "--format", "json"]) + + payload = json.loads(result.stdout) + by_slug = {row["diagnostic"]: row for row in payload} + assert by_slug["enso_tel"]["execution_group_count"] == 2 + assert by_slug["extratropical-modes-of-variability-nao"]["execution_group_count"] == 1 + # No executions seeded, so promoted-version groups are counted in `total` only. + assert by_slug["enso_tel"]["total"] == 2 + assert by_slug["enso_tel"]["successful"] == 0 + assert by_slug["enso_tel"]["inflight"] == 0 + + def test_list_help_documents_columns(self, invoke_cli): + result = invoke_cli(["diagnostics", "list", "--help"]) + + for column in ("execution_group_count", "successful", "inflight", "total"): + assert column in result.stdout + + def test_filter_by_provider(self, db_with_diagnostics, invoke_cli): + result = invoke_cli(["diagnostics", "list", "--provider", "pmp"]) + + assert "pmp" in result.stdout + assert "esmvaltool" not in result.stdout + + def test_filter_by_diagnostic(self, db_with_diagnostics, invoke_cli): + result = invoke_cli(["diagnostics", "list", "--diagnostic", "enso"]) + + assert "enso_tel" in result.stdout + assert "extratropical-modes-of-variability-nao" not in result.stdout + + def test_list_limit(self, db_with_diagnostics, invoke_cli): + result = invoke_cli(["diagnostics", "list", "--limit", "1", "--format", "json"]) + + payload = json.loads(result.stdout) + assert len(payload) == 1 + + def test_list_columns(self, db_with_diagnostics, invoke_cli): + result = invoke_cli(["diagnostics", "list", "--column", "diagnostic", "--column", "provider"]) + + assert "enso_tel" in result.stdout + assert "provider" in result.stdout + assert "promoted_version" not in result.stdout + + def test_list_columns_missing(self, db_with_diagnostics, invoke_cli): + invoke_cli( + ["diagnostics", "list", "--column", "diagnostic", "--column", "missing"], + expected_exit_code=1, + ) + + def test_list_json_empty(self, db_seeded, invoke_cli): + result = invoke_cli(["diagnostics", "list", "--diagnostic", "nonexistent", "--format", "json"]) + + assert json.loads(result.stdout) == [] diff --git a/packages/climate-ref/tests/unit/cli/test_root.py b/packages/climate-ref/tests/unit/cli/test_root.py index 53d3091c4..7eea0628f 100644 --- a/packages/climate-ref/tests/unit/cli/test_root.py +++ b/packages/climate-ref/tests/unit/cli/test_root.py @@ -128,7 +128,7 @@ def test_import_emits_no_debug_logs_without_celery(): @pytest.fixture() def expected_groups() -> set[str]: - return {"config", "datasets", "db", "executions", "providers", "celery", "test-cases"} + return {"config", "datasets", "db", "diagnostics", "executions", "providers", "celery", "test-cases"} def test_build_app(expected_groups): diff --git a/packages/climate-ref/tests/unit/results/test_diagnostics.py b/packages/climate-ref/tests/unit/results/test_diagnostics.py new file mode 100644 index 000000000..4b44238b5 --- /dev/null +++ b/packages/climate-ref/tests/unit/results/test_diagnostics.py @@ -0,0 +1,216 @@ +"""Unit tests for the diagnostic read layer.""" + +import pytest +from climate_ref_esmvaltool import provider as esmvaltool_provider +from climate_ref_pmp import provider as pmp_provider + +from climate_ref.models.diagnostic import Diagnostic +from climate_ref.models.execution import Execution, ExecutionGroup +from climate_ref.provider_registry import _register_provider +from climate_ref.results import DiagnosticFilter, Reader +from climate_ref.results.diagnostics import select_diagnostics + + +def test_import_smoke() -> None: + """Public surface imports cleanly (guards the `values`<->`diagnostics` wiring).""" + assert DiagnosticFilter is not None + assert Reader is not None + + +@pytest.fixture +def db_with_diagnostics(db_seeded): + """A seeded DB with two providers and execution groups spread unevenly across diagnostics.""" + with db_seeded.session.begin(): + _register_provider(db_seeded, pmp_provider) + _register_provider(db_seeded, esmvaltool_provider) + + diag_1 = db_seeded.session.query(Diagnostic).filter_by(slug="enso_tel").first() + diag_2 = ( + db_seeded.session.query(Diagnostic) + .filter_by(slug="extratropical-modes-of-variability-nao") + .first() + ) + # diag_1 gets two execution groups, diag_2 gets one, enso-characteristics gets none. + db_seeded.session.add(ExecutionGroup(key="key1", diagnostic_id=diag_1.id, selectors={})) + db_seeded.session.add(ExecutionGroup(key="key2", diagnostic_id=diag_1.id, selectors={})) + db_seeded.session.add(ExecutionGroup(key="key3", diagnostic_id=diag_2.id, selectors={})) + + db_seeded.session.commit() + + return db_seeded + + +class TestSelectDiagnostics: + def test_raw_rows_include_all_diagnostics(self, db_with_diagnostics): + stmt = select_diagnostics() + rows = db_with_diagnostics.session.execute(stmt).all() + slugs = {row.slug for row in rows} + assert "enso_tel" in slugs + assert "enso-characteristics" in slugs + + def test_execution_group_counts(self, db_with_diagnostics): + stmt = select_diagnostics() + rows = db_with_diagnostics.session.execute(stmt).all() + by_slug = {row.slug: row for row in rows} + + assert by_slug["enso_tel"].execution_group_count == 2 + assert by_slug["extratropical-modes-of-variability-nao"].execution_group_count == 1 + # A diagnostic with zero execution groups still appears, with a count of 0. + assert by_slug["enso-characteristics"].execution_group_count == 0 + + def test_provider_contains_filter(self, db_with_diagnostics): + stmt = select_diagnostics(DiagnosticFilter(provider_contains=["pmp"])) + rows = db_with_diagnostics.session.execute(stmt).all() + providers = {row.provider_slug for row in rows} + assert providers == {"pmp"} + + def test_diagnostic_contains_filter(self, db_with_diagnostics): + stmt = select_diagnostics(DiagnosticFilter(diagnostic_contains=["enso_tel"])) + rows = db_with_diagnostics.session.execute(stmt).all() + slugs = {row.slug for row in rows} + assert slugs == {"enso_tel"} + + def test_ordered_by_provider_then_diagnostic(self, db_with_diagnostics): + stmt = select_diagnostics() + rows = db_with_diagnostics.session.execute(stmt).all() + pairs = [(row.provider_slug, row.slug) for row in rows] + assert pairs == sorted(pairs) + + def test_bare_string_rejected_diagnostic_contains(self): + with pytest.raises(TypeError): + DiagnosticFilter(diagnostic_contains="enso") + + def test_bare_string_rejected_provider_contains(self): + with pytest.raises(TypeError): + DiagnosticFilter(provider_contains="pmp") + + +class TestDiagnosticsReaderList: + def test_dto_mapping(self, db_with_diagnostics): + reader = Reader(db_with_diagnostics) + coll = reader.diagnostics.list(DiagnosticFilter(diagnostic_contains=["enso_tel"])) + assert len(coll) == 1 + view = coll.items[0] + assert view.provider_slug == "pmp" + assert view.slug == "enso_tel" + assert view.name + assert view.promoted_version == 1 + assert view.execution_group_count == 2 + # Both groups are at the promoted version with no executions yet. + assert view.total == 2 + assert view.successful == 0 + assert view.inflight == 0 + + def test_promoted_version_status_counts(self, db_with_diagnostics): + # enso_tel has two promoted-version groups (key1, key2); give key1 a successful latest + # execution and key2 a running (successful IS NULL) one. + with db_with_diagnostics.session.begin(): + group_1 = db_with_diagnostics.session.query(ExecutionGroup).filter_by(key="key1").one() + group_2 = db_with_diagnostics.session.query(ExecutionGroup).filter_by(key="key2").one() + db_with_diagnostics.session.add( + Execution( + execution_group_id=group_1.id, + output_fragment="frag1", + dataset_hash="hash1", + successful=True, + ) + ) + db_with_diagnostics.session.add( + Execution( + execution_group_id=group_2.id, + output_fragment="frag2", + dataset_hash="hash2", + successful=None, + ) + ) + + reader = Reader(db_with_diagnostics) + view = reader.diagnostics.list(DiagnosticFilter(diagnostic_contains=["enso_tel"])).items[0] + assert view.successful == 1 + assert view.inflight == 1 + assert view.total == 2 + + def test_filter_by_provider(self, db_with_diagnostics): + reader = Reader(db_with_diagnostics) + coll = reader.diagnostics.list(DiagnosticFilter(provider_contains=["esmvaltool"])) + providers = {d.provider_slug for d in coll} + assert providers == {"esmvaltool"} + + def test_pagination_offset_limit(self, db_with_diagnostics): + reader = Reader(db_with_diagnostics) + full = reader.diagnostics.list() + total = full.total_count + assert total > 2 + + page = reader.diagnostics.list(offset=1, limit=1) + assert page.total_count == total + assert len(page) == 1 + assert page.offset == 1 + assert page.limit == 1 + + def test_detached_survival(self, db_with_diagnostics): + reader = Reader(db_with_diagnostics) + coll = reader.diagnostics.list() + db_with_diagnostics.session.expunge_all() + df = coll.to_pandas() + assert len(df) == coll.total_count + + def test_to_pandas_columns(self, db_with_diagnostics): + reader = Reader(db_with_diagnostics) + coll = reader.diagnostics.list() + df = coll.to_pandas() + assert list(df.columns) == [ + "provider", + "diagnostic", + "name", + "promoted_version", + "execution_group_count", + "successful", + "inflight", + "total", + ] + + def test_to_pandas_columns_when_empty(self, db_with_diagnostics): + reader = Reader(db_with_diagnostics) + coll = reader.diagnostics.list(DiagnosticFilter(diagnostic_contains=["nonexistent"])) + df = coll.to_pandas() + assert list(df.columns) == [ + "provider", + "diagnostic", + "name", + "promoted_version", + "execution_group_count", + "successful", + "inflight", + "total", + ] + assert len(df) == 0 + + +class TestDiagnosticsReaderStats: + def test_delegates_to_execution_statistics(self, db_with_diagnostics): + reader = Reader(db_with_diagnostics) + stats = reader.diagnostics.stats() + by_diag = {s.diagnostic: s for s in stats} + + assert "enso_tel" in by_diag + assert by_diag["enso_tel"].total == 2 + assert by_diag["enso_tel"].not_started == 2 + + def test_matches_executions_statistics(self, db_with_diagnostics): + reader = Reader(db_with_diagnostics) + via_diagnostics = reader.diagnostics.stats() + via_executions = reader.executions.statistics() + assert via_diagnostics == via_executions + + def test_provider_filter(self, db_with_diagnostics): + reader = Reader(db_with_diagnostics) + stats = reader.diagnostics.stats(provider_contains=["pmp"]) + providers = {s.provider for s in stats} + assert providers == {"pmp"} + + def test_diagnostic_filter(self, db_with_diagnostics): + reader = Reader(db_with_diagnostics) + stats = reader.diagnostics.stats(diagnostic_contains=["enso_tel"]) + diagnostics = {s.diagnostic for s in stats} + assert diagnostics == {"enso_tel"} From 4dfc544a846354d8fb66995c62c6bb325fb914d6 Mon Sep 17 00:00:00 2001 From: Jared Lewis Date: Thu, 2 Jul 2026 16:33:48 +1000 Subject: [PATCH 11/23] docs(results): add attribute docstrings to the read-layer DTOs Document each field on the execution, dataset, diagnostic, outlier, and metric-value filter DTOs as griffe/mkdocstrings attribute docstrings so they render per-attribute, replacing the stray inline comments that the docs tooling would not pick up, and unify wording across parallel fields. --- .../src/climate_ref/results/_query.py | 24 +++-- .../src/climate_ref/results/datasets.py | 23 +++++ .../src/climate_ref/results/diagnostics.py | 5 +- .../src/climate_ref/results/executions.py | 99 +++++++++++++++++-- .../src/climate_ref/results/outliers.py | 7 +- 5 files changed, 144 insertions(+), 14 deletions(-) diff --git a/packages/climate-ref/src/climate_ref/results/_query.py b/packages/climate-ref/src/climate_ref/results/_query.py index 92f71fa35..9c1da8d96 100644 --- a/packages/climate-ref/src/climate_ref/results/_query.py +++ b/packages/climate-ref/src/climate_ref/results/_query.py @@ -48,29 +48,41 @@ class MetricValueFilter: exists today between the CLI and the ref-app API. """ - # scoping to executions / groups execution_ids: Sequence[int] | None = None + """Restrict to values produced by these executions.""" + execution_group_ids: Sequence[int] | None = None + """Restrict to values produced by executions belonging to these groups.""" - # provenance filters -- exact (API) vs substring (CLI search) diagnostic_slug: str | None = None + """Exact-match diagnostic slug, for API path-scoped queries.""" + provider_slug: str | None = None + """Exact-match provider slug, for API path-scoped queries.""" + diagnostic_contains: Sequence[str] | None = None + """Case-insensitive substring matches on diagnostic slug (OR-combined), for CLI-style search.""" + provider_contains: Sequence[str] | None = None + """Case-insensitive substring matches on provider slug (OR-combined), for CLI-style search.""" - # CV dimension filters, keyed by registered dimension name; str -> equality, sequence -> IN dimensions: Mapping[str, str | Sequence[str]] | None = None + """CV dimension filters keyed by registered dimension name; a string is equality, a sequence is IN.""" - # id isolate / exclude (isolate takes precedence, matching the API) isolate_ids: Sequence[int] | None = None + """Restrict to exactly these value ids; takes precedence over ``exclude_ids``, matching the API.""" + exclude_ids: Sequence[int] | None = None + """Exclude these value ids; ignored when ``isolate_ids`` is set.""" - # series-only: reference_id IS NOT NULL (observations) / IS NULL (model) reference_only: bool | None = None + """Series-only: ``True`` for observation/reference series, ``False`` for model series.""" - # version / retraction gating promoted_only: bool = True + """Restrict to execution groups at the diagnostic's currently promoted version.""" + include_retracted: bool = False + """Include values produced by retracted executions.""" def dimension_clauses(self, entity: type[MetricValue]) -> list[Any]: """ diff --git a/packages/climate-ref/src/climate_ref/results/datasets.py b/packages/climate-ref/src/climate_ref/results/datasets.py index a848ded0b..79ace1685 100644 --- a/packages/climate-ref/src/climate_ref/results/datasets.py +++ b/packages/climate-ref/src/climate_ref/results/datasets.py @@ -32,9 +32,16 @@ class DatasetFileView: """A single dataset file, detached from the ORM. Times are raw stored strings (no cftime).""" path: str + """Path to the file.""" + start_time: str | None + """Start of the file's time range, as a raw stored string.""" + end_time: str | None + """End of the file's time range, as a raw stored string.""" + tracking_id: str | None + """The file's tracking ID, when present.""" @attrs.frozen(kw_only=True) @@ -42,13 +49,28 @@ class DatasetView: """A single dataset, detached from the ORM.""" id: int + """Primary key of the underlying ``Dataset`` row.""" + slug: str + """The dataset's slug.""" + dataset_type: SourceDatasetType + """The dataset's source type (e.g. CMIP6, obs4MIPs).""" + finalised: bool + """Whether the dataset was registered via the complete (netCDF-opening) parser.""" + created_at: Any + """Timestamp the dataset was created.""" + updated_at: Any + """Timestamp the dataset was last updated.""" + facets: Mapping[str, object] + """Dataset-specific metadata, keyed by the adapter's ``dataset_specific_metadata`` fields.""" + files: tuple[DatasetFileView, ...] + """The dataset's files; empty unless requested with ``include_files=True``.""" @attrs.frozen(kw_only=True) @@ -56,6 +78,7 @@ class DatasetCollection: """An immutable collection of datasets.""" datasets: tuple[DatasetView, ...] + """The datasets in this collection.""" def __iter__(self) -> Iterator[DatasetView]: return iter(self.datasets) diff --git a/packages/climate-ref/src/climate_ref/results/diagnostics.py b/packages/climate-ref/src/climate_ref/results/diagnostics.py index 3ad944d63..1b8e1b838 100644 --- a/packages/climate-ref/src/climate_ref/results/diagnostics.py +++ b/packages/climate-ref/src/climate_ref/results/diagnostics.py @@ -36,7 +36,10 @@ class DiagnosticFilter: """ provider_contains: tuple[str, ...] | None = attrs.field(default=None, converter=_as_str_tuple) + """Case-insensitive substring matches on provider slug (OR-combined).""" + diagnostic_contains: tuple[str, ...] | None = attrs.field(default=None, converter=_as_str_tuple) + """Case-insensitive substring matches on diagnostic slug (OR-combined).""" @attrs.frozen(kw_only=True) @@ -49,7 +52,7 @@ class DiagnosticView: """ provider_slug: str - """Slug of the provider that owns the diagnostic.""" + """Owning provider's slug.""" slug: str """The diagnostic's own slug, unique within its provider.""" diff --git a/packages/climate-ref/src/climate_ref/results/executions.py b/packages/climate-ref/src/climate_ref/results/executions.py index 34f03970b..5a5b3978c 100644 --- a/packages/climate-ref/src/climate_ref/results/executions.py +++ b/packages/climate-ref/src/climate_ref/results/executions.py @@ -64,11 +64,22 @@ class ExecutionGroupFilter: """ diagnostic_contains: tuple[str, ...] | None = attrs.field(default=None, converter=_as_str_tuple) + """Case-insensitive substring matches on diagnostic slug (OR-combined).""" + provider_contains: tuple[str, ...] | None = attrs.field(default=None, converter=_as_str_tuple) + """Case-insensitive substring matches on provider slug (OR-combined).""" + dirty: bool | None = None + """Constrain on the group's ``dirty`` flag.""" + successful: bool | None = None + """Constrain on the latest execution's ``successful`` status.""" + facets: Mapping[str, tuple[str, ...]] | None = attrs.field(default=None, converter=_as_facets) + """Selector-facet map matched against each group's ``selectors``.""" + include_superseded: bool = False + """Include execution groups whose ``diagnostic_version`` is not the diagnostic's promoted version.""" @attrs.frozen(kw_only=True) @@ -76,15 +87,34 @@ class ExecutionView: """A single execution, detached from the ORM.""" id: int + """Primary key of the underlying ``Execution`` row.""" + execution_group_id: int + """The execution group this execution belongs to.""" + successful: bool | None + """``True``/``False`` once the execution has finished, ``None`` while still running.""" + dataset_hash: str + """Hash of the input datasets used for this execution.""" + retracted: bool + """Whether this execution has been retracted.""" + output_fragment: str + """Relative directory storing this execution's output, once moved to the final output directory.""" + path: str | None + """Path to the output bundle, relative to the diagnostic execution result output directory.""" + provider_version: str | None + """Version of the diagnostic provider that produced this execution.""" + created_at: Any + """Timestamp the execution was created.""" + updated_at: Any + """Timestamp the execution was last updated.""" @attrs.frozen(kw_only=True) @@ -92,21 +122,43 @@ class ExecutionGroupView: """An execution group, detached from the ORM, with its per-group latest execution (if any).""" id: int + """Primary key of the underlying ``ExecutionGroup`` row.""" + key: str + """Stable key identifying the group's input-dataset selection.""" + diagnostic_slug: str + """Owning diagnostic's slug.""" + provider_slug: str + """Owning provider's slug.""" + dirty: bool + """Whether the group is flagged for re-execution (e.g. new data has arrived).""" + diagnostic_version: int + """Diagnostic version this group was executed against.""" + selectors: Mapping[str, Any] + """The selector-facet values used to build this group's input-dataset selection.""" + created_at: Any + """Timestamp the group was created.""" + updated_at: Any - # Reflects the helper's `max(created_at)` outer join - # (models/execution.py::get_execution_group_and_latest), which can duplicate a group on an - # exact `created_at` tie. This may differ from `ExecutionsReader.latest_execution()`, which - # tie-breaks by `created_at DESC, id DESC` (`_query.latest_execution_for_group`). Both - # behaviours are intentional for their respective consumers (`groups()` vs `latest_execution()`) - # -- do not unify them. + """Timestamp the group was last updated.""" + latest: ExecutionView | None + """ + The group's most recent execution, or ``None`` if it has never been executed. + + Reflects the helper's ``max(created_at)`` outer join + (models/execution.py::get_execution_group_and_latest), which can duplicate a group on an + exact ``created_at`` tie. This may differ from ``ExecutionsReader.latest_execution()``, which + tie-breaks by ``created_at DESC, id DESC`` (``_query.latest_execution_for_group``). Both + behaviours are intentional for their respective consumers (``groups()`` vs ``latest_execution()``) + -- do not unify them. + """ @property def successful(self) -> bool | None: @@ -119,13 +171,28 @@ class ExecutionStats: """Per-(provider, diagnostic) execution status counts, detached from the ORM.""" provider: str + """Provider slug.""" + diagnostic: str + """Diagnostic slug.""" + running: int + """Groups whose latest execution exists with ``successful IS NULL``.""" + failed: int + """Groups whose latest execution exists with ``successful IS False``.""" + successful: int + """Groups whose latest execution exists with ``successful IS True``.""" + not_started: int + """Groups with no execution yet.""" + dirty: int + """Groups flagged for re-execution.""" + total: int + """Total execution groups counted (at the diagnostic's promoted version).""" @attrs.frozen(kw_only=True) @@ -133,12 +200,25 @@ class OutputView: """A single registered execution output, detached from the ORM.""" execution_id: int + """The execution that registered this output.""" + output_type: str + """The output's type (e.g. plot, data, HTML), which determines how it is displayed.""" + filename: str | None + """Path to the output, relative to the diagnostic execution result output directory.""" + short_name: str | None + """Short key of the output, unique for a given result and output type.""" + long_name: str | None + """Human-readable name for the output.""" + description: str | None + """Free-text description of the output.""" + dimensions: Mapping[str, str] + """CV dimension values associated with this output.""" @attrs.frozen(kw_only=True) @@ -146,9 +226,16 @@ class ExecutionGroupCollection: """An immutable page of execution groups plus collection-level metadata.""" items: tuple[ExecutionGroupView, ...] + """The execution groups on this page.""" + total_count: int + """Total execution groups matching the filter before ``offset``/``limit``.""" + offset: int + """Rows skipped before this page.""" + limit: int | None + """Page size requested, or ``None`` when the whole result was returned.""" def __iter__(self) -> Iterator[ExecutionGroupView]: return iter(self.items) diff --git a/packages/climate-ref/src/climate_ref/results/outliers.py b/packages/climate-ref/src/climate_ref/results/outliers.py index d6a38e283..9fa6f4205 100644 --- a/packages/climate-ref/src/climate_ref/results/outliers.py +++ b/packages/climate-ref/src/climate_ref/results/outliers.py @@ -56,8 +56,13 @@ class AnnotatedScalar: """A scalar ORM row paired with its outlier verdict.""" value: ScalarMetricValue + """The underlying scalar metric value row.""" + is_outlier: bool - verification_status: str # "verified" | "unverified" + """Whether this value was flagged as an outlier.""" + + verification_status: str + """``"verified"`` or ``"unverified"``, mirroring ``is_outlier``.""" def _flag_outliers_iqr(values: Sequence[float], factor: float, min_n: int) -> list[bool]: From 078a58386fa539e07dbaf3dc5e27df264682b024 Mon Sep 17 00:00:00 2001 From: Jared Lewis Date: Thu, 2 Jul 2026 16:33:59 +1000 Subject: [PATCH 12/23] fix(results): promote kind out of metric-value DTO dimensions kind is a registered CV dimension, so it has its own column and was surfacing both as the dedicated ScalarValue/SeriesValue.kind field and inside the dimensions mapping. Strip it from dimensions on read so it is exposed once via the typed field; the to_pandas kind column and the kind facet are unaffected. --- .../src/climate_ref/results/values.py | 87 ++++++++++++++++++- .../tests/unit/results/test_results.py | 38 ++++++++ 2 files changed, 121 insertions(+), 4 deletions(-) diff --git a/packages/climate-ref/src/climate_ref/results/values.py b/packages/climate-ref/src/climate_ref/results/values.py index 051f8f64e..7e5e2029b 100644 --- a/packages/climate-ref/src/climate_ref/results/values.py +++ b/packages/climate-ref/src/climate_ref/results/values.py @@ -53,7 +53,10 @@ class Facet: """Distinct values observed for one CV dimension across a filtered query.""" key: str + """The CV dimension name.""" + values: tuple[str, ...] + """Distinct values observed for the dimension.""" @attrs.frozen(kw_only=True) @@ -61,18 +64,41 @@ class ScalarValue: """A single scalar metric value, detached from the ORM.""" id: int + """Primary key of the underlying ``ScalarMetricValue`` row.""" + execution_id: int + """The execution that produced this value.""" + execution_group_id: int + """The execution group the producing execution belongs to.""" + value: float | None + """The scalar value itself.""" + kind: str + """Model/reference kind, resolved from the ``kind`` CV dimension (defaults to ``"model"``).""" + dimensions: Mapping[str, str] + """ + CV dimension values (e.g. ``source_id``, ``statistic``, ``metric``) for this row. + + ``kind`` is excluded here and surfaced as the dedicated ``kind`` field instead. + """ + attributes: Mapping[str, Any] - # per-row outlier verdict (populated when detection ran) + """Free-form, non-CV attributes attached to the row.""" + is_outlier: bool | None = None + """Per-row outlier verdict; ``None`` unless outlier detection ran.""" + verification_status: str | None = None - # optional provenance, only when include_context=True + """``"verified"``/``"unverified"`` status paired with ``is_outlier``; ``None`` unless detection ran.""" + diagnostic_slug: str | None = None + """Owning diagnostic's slug, populated only when ``include_context=True``.""" + provider_slug: str | None = None + """Owning provider's slug, populated only when ``include_context=True``.""" @attrs.frozen(kw_only=True) @@ -80,17 +106,44 @@ class SeriesValue: """A 1-d series metric value, detached from the ORM. The index is snapshotted from the shared axis.""" id: int + """Primary key of the underlying ``SeriesMetricValue`` row.""" + execution_id: int + """The execution that produced this value.""" + execution_group_id: int + """The execution group the producing execution belongs to.""" + values: Sequence[float | int] + """The series' y-values, in index order.""" + index: Sequence[float | int | str] | None + """The shared index axis values, snapshotted at read time.""" + index_name: str | None + """Name of the shared index axis (e.g. a time or latitude label).""" + reference_id: str | None + """Set for observation/reference series; ``None`` for model series.""" + kind: str + """Model/reference kind, resolved from the ``kind`` CV dimension (defaults to ``"model"``).""" + dimensions: Mapping[str, str] + """ + CV dimension values (e.g. ``source_id``, ``statistic``, ``metric``) for this row. + + ``kind`` is excluded here and surfaced as the dedicated ``kind`` field instead. + """ + attributes: Mapping[str, Any] + """Free-form, non-CV attributes attached to the row.""" + diagnostic_slug: str | None = None + """Owning diagnostic's slug, populated only when ``include_context=True``.""" + provider_slug: str | None = None + """Owning provider's slug, populated only when ``include_context=True``.""" @property def is_reference(self) -> bool: @@ -103,12 +156,25 @@ class ScalarValueCollection: """An immutable page of scalar values plus collection-level, outlier-aware metadata.""" items: tuple[ScalarValue, ...] + """The scalar values on this page.""" + total_count: int + """Total values matching the filter before ``offset``/``limit`` (after outlier removal, if any).""" + facets: tuple[Facet, ...] + """Distinct CV dimension values observed over the full filtered set, before pagination.""" + offset: int + """Rows skipped before this page.""" + limit: int | None + """Page size requested, or ``None`` when the whole result was returned.""" + had_outliers: bool + """Whether outlier detection ran and flagged at least one value.""" + outlier_count: int + """Number of values flagged as outliers, or ``0`` when detection did not run.""" def __iter__(self) -> Iterator[ScalarValue]: return iter(self.items) @@ -146,10 +212,19 @@ class SeriesValueCollection: """An immutable page of series values plus collection-level metadata.""" items: tuple[SeriesValue, ...] + """The series values on this page.""" + total_count: int + """Total values matching the filter before ``offset``/``limit``.""" + facets: tuple[Facet, ...] + """Distinct CV dimension values observed over the full filtered set, before pagination.""" + offset: int + """Rows skipped before this page.""" + limit: int | None + """Page size requested, or ``None`` when the whole result was returned.""" def __iter__(self) -> Iterator[SeriesValue]: return iter(self.items) @@ -363,12 +438,14 @@ def _to_scalar_dto( ) -> ScalarValue: diagnostic_slug, provider_slug = self._context_slugs(row.execution, include_context) dims = dict(row.dimensions) + kind = _kind_of(dims) + dims.pop("kind", None) return ScalarValue( id=row.id, execution_id=row.execution_id, execution_group_id=row.execution.execution_group_id, value=row.value, - kind=_kind_of(dims), + kind=kind, dimensions=dims, attributes=dict(row.attributes or {}), is_outlier=is_outlier if detection_ran else None, @@ -380,6 +457,8 @@ def _to_scalar_dto( def _to_series_dto(self, row: SeriesMetricValue, include_context: bool) -> SeriesValue: diagnostic_slug, provider_slug = self._context_slugs(row.execution, include_context) dims = dict(row.dimensions) + kind = _kind_of(dims) + dims.pop("kind", None) return SeriesValue( id=row.id, execution_id=row.execution_id, @@ -388,7 +467,7 @@ def _to_series_dto(self, row: SeriesMetricValue, include_context: bool) -> Serie index=list(row.index) if row.index is not None else None, index_name=row.index_name, reference_id=row.reference_id, - kind=_kind_of(dims), + kind=kind, dimensions=dims, attributes=dict(row.attributes or {}), diagnostic_slug=diagnostic_slug, diff --git a/packages/climate-ref/tests/unit/results/test_results.py b/packages/climate-ref/tests/unit/results/test_results.py index bcec2064f..02cce5fd4 100644 --- a/packages/climate-ref/tests/unit/results/test_results.py +++ b/packages/climate-ref/tests/unit/results/test_results.py @@ -251,6 +251,44 @@ def test_to_pandas_long_form(self, dal_db): assert col in df.columns +class TestKindSeparation: + def test_kind_promoted_out_of_dimensions(self, db_seeded): + # kind is a registered CV dimension, so it has its own column; the read layer promotes it + # to the dedicated `kind` field and must not leave it duplicated inside `dimensions`. + session = db_seeded.session + with session.begin(): + diagnostic = session.query(Diagnostic).first() + group = ExecutionGroup(key="kind-key", diagnostic_id=diagnostic.id, selectors={}) + session.add(group) + session.flush() + execution = Execution( + execution_group_id=group.id, + output_fragment="frag", + dataset_hash="hash-kind", + successful=True, + ) + session.add(execution) + session.flush() + session.add( + ScalarMetricValue.build( + execution_id=execution.id, + value=1.0, + attributes=None, + dimensions={"metric": "tas", "source_id": "ACCESS-CM2", "kind": "reference"}, + ) + ) + group_id = group.id + db_seeded.execution_group_id = group_id + + coll = Reader(db_seeded).values.scalar_values(MetricValueFilter(execution_group_ids=[group_id])) + value = coll.items[0] + assert value.kind == "reference" + assert "kind" not in value.dimensions + assert value.dimensions["source_id"] == "ACCESS-CM2" + # kind is still surfaced as a frame column (promoted explicitly, not via dimensions). + assert "kind" in coll.to_pandas().columns + + class TestLatestExecutionForGroup: def test_returns_most_recent(self, dal_db): session = dal_db.session From d043b758d35c79a2c98169866537a5c2fe862091 Mon Sep 17 00:00:00 2001 From: Jared Lewis Date: Thu, 2 Jul 2026 22:03:22 +1000 Subject: [PATCH 13/23] docs: add changelog entry for the results read layer --- changelog/780.feature.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 changelog/780.feature.md diff --git a/changelog/780.feature.md b/changelog/780.feature.md new file mode 100644 index 000000000..112899a85 --- /dev/null +++ b/changelog/780.feature.md @@ -0,0 +1,5 @@ +Added `climate_ref.results`, a typed read layer for querying results from notebooks and other tools. +A single `Reader` exposes per-domain readers for metric values, executions, datasets, diagnostics, and output artifacts, each returning frozen objects that remain usable after the database session closes and can be turned into a pandas DataFrame. + +Added a `ref diagnostics list` command that lists the registered diagnostics with their promoted version and per-diagnostic successful, in-flight, and total execution-group counts. +Metric values now expose a first-class `kind` (model or reference) field, and `ref datasets list` returns only the latest version of each dataset. From 5264929fe019cd743b856457aa11c33f930576e0 Mon Sep 17 00:00:00 2001 From: Jared Lewis Date: Fri, 3 Jul 2026 11:52:37 +1000 Subject: [PATCH 14/23] fix: address review feedback on the results read layer Fixes across the CLI, dataset query builder, and results read layer raised in review of the results read layer PR: - diagnostics list: always validate/select --column, even when the filtered result set is empty (to_pandas already emits explicit columns for an empty collection). - executions inspect: fall back to the plain filename when an output path would escape the results root instead of propagating the ValueError. - executions values: report the outlier count and --include-unverified hint when every value in a page was flagged as an outlier, instead of a bare "No scalar values found.". - coerce_catalog_times: coerce start_time/end_time independently so a catalog with only one of the two columns doesn't KeyError. - DatasetFilter facets and MetricValueFilter diagnostic/provider filters: guard against a bare str value being iterated character-by-character. - DatasetCollection.to_pandas: emit explicit base columns for an empty collection, matching the other collection types. - reader.datasets: eager-load polymorphic dataset subtypes for mixed (source_type=None) listings to avoid an N+1 lazy load per row. - select_execution_statistics: resolve to a single tie-broken latest execution id per group before joining, so two executions tied on created_at no longer double-count that group's status. - get_execution_group_and_latest: eager-load diagnostic/provider, used by every caller of this query. - OutlierPolicy.method: restrict to the two implemented values. Adds regression tests for the empty-column, tie-count, missing-column guard, and all-outliers-hidden behaviour, plus a --kind series --format json coverage gap for the values command. --- .../src/climate_ref/cli/diagnostics.py | 9 ++-- .../src/climate_ref/cli/executions.py | 15 ++++-- .../src/climate_ref/datasets/utils.py | 13 +++-- .../src/climate_ref/models/dataset_query.py | 10 +++- .../src/climate_ref/models/execution.py | 6 ++- .../src/climate_ref/results/_query.py | 5 +- .../src/climate_ref/results/datasets.py | 20 ++++++-- .../src/climate_ref/results/executions.py | 20 ++++++-- .../src/climate_ref/results/outliers.py | 4 +- .../tests/unit/cli/test_diagnostics.py | 8 +++ .../tests/unit/cli/test_executions.py | 49 +++++++++++++++++++ .../tests/unit/datasets/test_utils.py | 28 +++++++++++ .../tests/unit/results/test_datasets.py | 12 +++++ .../tests/unit/results/test_executions.py | 38 ++++++++++++++ 14 files changed, 212 insertions(+), 25 deletions(-) diff --git a/packages/climate-ref/src/climate_ref/cli/diagnostics.py b/packages/climate-ref/src/climate_ref/cli/diagnostics.py index b5fa2dfe0..92239813e 100644 --- a/packages/climate-ref/src/climate_ref/cli/diagnostics.py +++ b/packages/climate-ref/src/climate_ref/cli/diagnostics.py @@ -68,11 +68,10 @@ def list_( # noqa: PLR0913 results_df = collection.to_pandas() if column: - if not results_df.empty: - if not all(col in results_df.columns for col in column): - logger.error(f"Column not found in data catalog: {column}") - raise typer.Exit(code=1) - results_df = results_df[column] + if not all(col in results_df.columns for col in column): + logger.error(f"Column not found in data catalog: {column}") + raise typer.Exit(code=1) + results_df = results_df[column] render_dataframe(results_df, console=console, output_format=output_format) diff --git a/packages/climate-ref/src/climate_ref/cli/executions.py b/packages/climate-ref/src/climate_ref/cli/executions.py index 5f85bd3a4..4a3664d3b 100644 --- a/packages/climate-ref/src/climate_ref/cli/executions.py +++ b/packages/climate-ref/src/climate_ref/cli/executions.py @@ -471,8 +471,11 @@ def _outputs_panel(outputs: tuple[OutputView, ...], output_fragment: str, reader if out.filename is None: filename_cell: Text | str = "" else: - output_path = reader.artifacts.output_file(output_fragment, out.filename) - filename_cell = Text(out.filename, style=f"link file://{output_path}") + try: + output_path = reader.artifacts.output_file(output_fragment, out.filename) + filename_cell = Text(out.filename, style=f"link file://{output_path}") + except ValueError: + filename_cell = out.filename table.add_row( out.output_type, out.short_name or "", @@ -724,7 +727,13 @@ def _render_scalar_values( # noqa: PLR0913 with_facets=False, ) if not len(collection): - console.print("No scalar values found.") + if collection.had_outliers: + console.print( + f"No scalar values found. {collection.outlier_count} value(s) were flagged as " + f"outliers and hidden. Use --include-unverified to show them." + ) + else: + console.print("No scalar values found.") return render_dataframe(collection.to_pandas(), console=console, output_format=output_format) diff --git a/packages/climate-ref/src/climate_ref/datasets/utils.py b/packages/climate-ref/src/climate_ref/datasets/utils.py index 6651ce210..82f448c0e 100644 --- a/packages/climate-ref/src/climate_ref/datasets/utils.py +++ b/packages/climate-ref/src/climate_ref/datasets/utils.py @@ -144,13 +144,16 @@ def coerce_catalog_times(catalog: pd.DataFrame) -> pd.DataFrame: """ Coerce a catalog's stored ``start_time``/``end_time`` strings to cftime objects. - A no-op when the catalog has no ``start_time`` column (dataset-level catalogs). Each row's - ``calendar`` is used when present, otherwise ``"standard"``. Mutates and returns ``catalog``. + A no-op when the catalog has no ``start_time`` column (dataset-level catalogs). Each of + ``start_time``/``end_time`` is coerced independently, so a catalog with only one of the two + columns present does not raise a ``KeyError``. Each row's ``calendar`` is used when present, + otherwise ``"standard"``. Mutates and returns ``catalog``. """ - if "start_time" in catalog.columns: + if "start_time" in catalog.columns or "end_time" in catalog.columns: cal = catalog["calendar"] if "calendar" in catalog.columns else "standard" - catalog["start_time"] = parse_cftime_dates(catalog["start_time"], cal) - catalog["end_time"] = parse_cftime_dates(catalog["end_time"], cal) + for column in ("start_time", "end_time"): + if column in catalog.columns: + catalog[column] = parse_cftime_dates(catalog[column], cal) return catalog diff --git a/packages/climate-ref/src/climate_ref/models/dataset_query.py b/packages/climate-ref/src/climate_ref/models/dataset_query.py index 479eb7019..e7e684209 100644 --- a/packages/climate-ref/src/climate_ref/models/dataset_query.py +++ b/packages/climate-ref/src/climate_ref/models/dataset_query.py @@ -23,10 +23,16 @@ def _as_facets( value: Mapping[str, Sequence[str]] | None, ) -> Mapping[str, tuple[str, ...]] | None: - """Copy a facets mapping into an immutable ``dict`` of immutable ``tuple`` values.""" + """ + Copy a facets mapping into an immutable ``dict`` of immutable ``tuple`` values. + + A bare ``str`` value is itself a ``Sequence[str]``, so without this guard a caller passing + ``facets={"source_id": "TEST-MODEL"}`` would have it iterated character-by-character + (``"TEST-MODEL"`` -> ``T``, ``E``, ``S``, ...) before it ever reaches the ``IN`` clause. + """ if value is None: return None - return {k: tuple(v) for k, v in value.items()} + return {k: (v,) if isinstance(v, str) else tuple(v) for k, v in value.items()} @attrs.frozen(kw_only=True) diff --git a/packages/climate-ref/src/climate_ref/models/execution.py b/packages/climate-ref/src/climate_ref/models/execution.py index 2084924c4..0fa86e18c 100644 --- a/packages/climate-ref/src/climate_ref/models/execution.py +++ b/packages/climate-ref/src/climate_ref/models/execution.py @@ -6,7 +6,7 @@ from loguru import logger from sqlalchemy import Column, ForeignKey, Table, UniqueConstraint, func, or_ -from sqlalchemy.orm import Mapped, Session, mapped_column, relationship +from sqlalchemy.orm import Mapped, Session, joinedload, mapped_column, relationship from sqlalchemy.orm.query import RowReturningQuery from climate_ref.models.base import Base @@ -500,8 +500,12 @@ def get_execution_group_and_latest( latest = _latest_execution_ids(session, among_executions) # Groups with no matching execution still appear (outer join, Execution=None). + # Every known caller (`cli/executions.py::delete_groups`, `ExecutionsReader.groups()`, + # `executor/reingest.py`) reads `eg.diagnostic.slug`/`eg.diagnostic.provider.slug` off the + # returned groups, so eager-load both relationships here to avoid an N+1 lazy-load per row. query = ( session.query(ExecutionGroup, Execution) + .options(joinedload(ExecutionGroup.diagnostic).joinedload(Diagnostic.provider)) .outerjoin(latest, latest.c.group_id == ExecutionGroup.id) .outerjoin(Execution, Execution.id == latest.c.execution_id) ) diff --git a/packages/climate-ref/src/climate_ref/results/_query.py b/packages/climate-ref/src/climate_ref/results/_query.py index 9c1da8d96..bea9a3878 100644 --- a/packages/climate-ref/src/climate_ref/results/_query.py +++ b/packages/climate-ref/src/climate_ref/results/_query.py @@ -26,6 +26,7 @@ SeriesMetricValue, ) from climate_ref.models.provider import Provider +from climate_ref.results._converters import _as_str_tuple @attrs.frozen(kw_only=True) @@ -60,10 +61,10 @@ class MetricValueFilter: provider_slug: str | None = None """Exact-match provider slug, for API path-scoped queries.""" - diagnostic_contains: Sequence[str] | None = None + diagnostic_contains: Sequence[str] | None = attrs.field(default=None, converter=_as_str_tuple) """Case-insensitive substring matches on diagnostic slug (OR-combined), for CLI-style search.""" - provider_contains: Sequence[str] | None = None + provider_contains: Sequence[str] | None = attrs.field(default=None, converter=_as_str_tuple) """Case-insensitive substring matches on provider slug (OR-combined), for CLI-style search.""" dimensions: Mapping[str, str | Sequence[str]] | None = None diff --git a/packages/climate-ref/src/climate_ref/results/datasets.py b/packages/climate-ref/src/climate_ref/results/datasets.py index 79ace1685..00cf84f9e 100644 --- a/packages/climate-ref/src/climate_ref/results/datasets.py +++ b/packages/climate-ref/src/climate_ref/results/datasets.py @@ -16,7 +16,7 @@ import attrs import pandas as pd -from sqlalchemy.orm import selectinload +from sqlalchemy.orm import selectin_polymorphic, selectinload from climate_ref.database import Database from climate_ref.datasets import get_dataset_adapter @@ -87,7 +87,15 @@ def __len__(self) -> int: return len(self.datasets) def to_pandas(self) -> pd.DataFrame: - """DataFrame with one row per dataset; columns are base fields plus the facet dict expanded.""" + """ + DataFrame with one row per dataset; columns are base fields plus the facet dict expanded. + + The base columns (``id, slug, dataset_type, finalised, created_at, updated_at``) are + emitted explicitly even when the collection is empty, so callers can select columns / + build an empty table without special-casing. Facet columns are dynamic (they depend on + the source type queried) and so are only present when at least one row has them. + """ + base_columns = ["id", "slug", "dataset_type", "finalised", "created_at", "updated_at"] records = [] for ds in self.datasets: rec: dict[str, Any] = { @@ -100,7 +108,7 @@ def to_pandas(self) -> pd.DataFrame: } rec.update(ds.facets) records.append(rec) - return pd.DataFrame.from_records(records) + return pd.DataFrame.from_records(records, columns=base_columns if not records else None) class DatasetsReader: @@ -162,6 +170,12 @@ def datasets( latest_group_by = adapter.dataset_id_metadata if adapter is not None else None stmt = select_datasets(filter, latest_group_by=latest_group_by) + if adapter is None: + # Mixed listing over the base ``Dataset`` entity: eager-load every polymorphic subtype + # table up front (one extra SELECT per subtype) so ``_to_view``'s ``getattr(dataset, + # k)`` subtype-facet reads don't lazy-load one row at a time (N+1). + subtypes = [m.class_ for m in Dataset.__mapper__.polymorphic_map.values()] + stmt = stmt.options(selectin_polymorphic(Dataset, subtypes)) if include_files: stmt = stmt.options(selectinload(entity.files)) # type: ignore[attr-defined] if limit is not None: diff --git a/packages/climate-ref/src/climate_ref/results/executions.py b/packages/climate-ref/src/climate_ref/results/executions.py index 5a5b3978c..75314d7b9 100644 --- a/packages/climate-ref/src/climate_ref/results/executions.py +++ b/packages/climate-ref/src/climate_ref/results/executions.py @@ -305,6 +305,11 @@ def select_execution_statistics( ``promoted_version`` are counted (same promoted-version gate as [get_execution_group_and_latest_filtered][climate_ref.models.execution.get_execution_group_and_latest_filtered]). """ + # Resolve to a single latest execution id per group, tie-broken by max id (matching the + # created_at DESC, id DESC tie-break used elsewhere -- see `_query.latest_execution_for_group`). + # Joining on `execution_group_id` + `created_at` alone (without this tie-break) can match two + # rows for a group whose executions share an exact `created_at`, double-counting that group in + # every aggregate column below. latest_exec_subquery = ( select( Execution.execution_group_id, @@ -313,6 +318,16 @@ def select_execution_statistics( .group_by(Execution.execution_group_id) .subquery() ) + latest_exec_id_subquery = ( + select(func.max(Execution.id).label("latest_execution_id")) + .join( + latest_exec_subquery, + (Execution.execution_group_id == latest_exec_subquery.c.execution_group_id) + & (Execution.created_at == latest_exec_subquery.c.latest_created_at), + ) + .group_by(Execution.execution_group_id) + .subquery() + ) stmt = ( select( @@ -333,11 +348,10 @@ def select_execution_statistics( .join(Diagnostic, ExecutionGroup.diagnostic_id == Diagnostic.id) .where(ExecutionGroup.diagnostic_version == Diagnostic.promoted_version) .join(Provider, Diagnostic.provider_id == Provider.id) - .outerjoin(latest_exec_subquery, ExecutionGroup.id == latest_exec_subquery.c.execution_group_id) .outerjoin( Execution, - (Execution.execution_group_id == ExecutionGroup.id) - & (Execution.created_at == latest_exec_subquery.c.latest_created_at), + Execution.id.in_(select(latest_exec_id_subquery.c.latest_execution_id)) + & (Execution.execution_group_id == ExecutionGroup.id), ) .group_by(Provider.slug, Diagnostic.slug) .order_by(Provider.slug, Diagnostic.slug) diff --git a/packages/climate-ref/src/climate_ref/results/outliers.py b/packages/climate-ref/src/climate_ref/results/outliers.py index 9fa6f4205..82a555ae3 100644 --- a/packages/climate-ref/src/climate_ref/results/outliers.py +++ b/packages/climate-ref/src/climate_ref/results/outliers.py @@ -23,6 +23,8 @@ from climate_ref.models.metric_value import ScalarMetricValue +_SUPPORTED_METHODS = ("off", "iqr") + @attrs.frozen(kw_only=True) class OutlierPolicy: @@ -33,7 +35,7 @@ class OutlierPolicy: (``factor=10.0``, source-id-aware IQR grouped by ``("statistic", "metric")``). """ - method: str = "iqr" + method: str = attrs.field(default="iqr", validator=attrs.validators.in_(_SUPPORTED_METHODS)) """Detection method. ``"off"`` disables detection; ``"iqr"`` enables it.""" factor: float = 10.0 diff --git a/packages/climate-ref/tests/unit/cli/test_diagnostics.py b/packages/climate-ref/tests/unit/cli/test_diagnostics.py index 4f1f1c1f5..d7b61eeec 100644 --- a/packages/climate-ref/tests/unit/cli/test_diagnostics.py +++ b/packages/climate-ref/tests/unit/cli/test_diagnostics.py @@ -99,3 +99,11 @@ def test_list_json_empty(self, db_seeded, invoke_cli): result = invoke_cli(["diagnostics", "list", "--diagnostic", "nonexistent", "--format", "json"]) assert json.loads(result.stdout) == [] + + def test_list_columns_missing_on_empty_results(self, db_seeded, invoke_cli): + """An unknown ``--column`` must still be rejected when the filtered result set is empty -- + ``to_pandas()`` always emits explicit columns, so there is no reason to skip validation.""" + invoke_cli( + ["diagnostics", "list", "--diagnostic", "nonexistent", "--column", "missing"], + expected_exit_code=1, + ) diff --git a/packages/climate-ref/tests/unit/cli/test_executions.py b/packages/climate-ref/tests/unit/cli/test_executions.py index 802db23a5..9787b4918 100644 --- a/packages/climate-ref/tests/unit/cli/test_executions.py +++ b/packages/climate-ref/tests/unit/cli/test_executions.py @@ -1406,6 +1406,43 @@ def test_scalar_outliers_shown(self, db_seeded, invoke_cli): assert result.exit_code == 0 assert "WILD" in result.stdout + def test_scalar_all_outliers_hidden_reports_outlier_count(self, db_seeded, invoke_cli): + """When every value is flagged as an outlier the empty-page message must still surface + that outliers exist, instead of silently reporting "No scalar values found.".""" + with db_seeded.session.begin(): + diagnostic = db_seeded.session.query(Diagnostic).first() + assert diagnostic is not None + group = ExecutionGroup(key="allwild", diagnostic_id=diagnostic.id, selectors={}) + db_seeded.session.add(group) + db_seeded.session.flush() + execution = Execution( + execution_group_id=group.id, + output_fragment="frag-wild", + dataset_hash="wildhash", + successful=True, + ) + db_seeded.session.add(execution) + db_seeded.session.flush() + # A non-finite value is always flagged as an outlier, regardless of group size. + db_seeded.session.add( + ScalarMetricValue.build( + execution_id=execution.id, + value=float("nan"), + attributes=None, + dimensions={"statistic": "mean", "metric": "tas", "source_id": "ONLY-MODEL"}, + ) + ) + db_seeded.session.flush() + group_id = group.id + + result = invoke_cli(["executions", "values", str(group_id), "--outliers"]) + + assert result.exit_code == 0 + assert "No scalar values found" in result.stdout + assert "1" in result.stdout + assert "outlier" in result.stdout + assert "--include-unverified" in result.stdout + def test_dimension_filter(self, db_seeded, invoke_cli): group_id, _ = self._setup(db_seeded) @@ -1434,6 +1471,18 @@ def test_series(self, db_seeded, invoke_cli): assert "n_points" in result.stdout assert "3" in result.stdout + def test_series_json(self, db_seeded, invoke_cli): + group_id, _ = self._setup(db_seeded) + + result = invoke_cli(["executions", "values", str(group_id), "--kind", "series", "--format", "json"]) + + assert result.exit_code == 0 + # JSON output is the long-form, exploded shape: one record per (series, index point). + records = json.loads(result.stdout) + assert len(records) == 3 + assert all(r["source_id"] == "ACCESS-CM2" for r in records) + assert [r["value"] for r in records] == [1.0, 2.0, 3.0] + def test_specific_execution_wrong_group(self, db_seeded, invoke_cli): _group_id, execution_id = self._setup(db_seeded) with db_seeded.session.begin(): diff --git a/packages/climate-ref/tests/unit/datasets/test_utils.py b/packages/climate-ref/tests/unit/datasets/test_utils.py index da6ec7d2c..ff44b09ca 100644 --- a/packages/climate-ref/tests/unit/datasets/test_utils.py +++ b/packages/climate-ref/tests/unit/datasets/test_utils.py @@ -9,6 +9,7 @@ from climate_ref.datasets.utils import ( build_instance_id, clean_branch_time, + coerce_catalog_times, parse_cftime_dates, parse_drs_daterange, validate_path, @@ -254,3 +255,30 @@ def test_does_not_mutate_input(self): original = df.copy() build_instance_id(df, self.drs_items, prefix="CMIP6") pd.testing.assert_frame_equal(df, original) + + +class TestCoerceCatalogTimes: + def test_no_time_columns_is_noop(self): + catalog = pd.DataFrame({"instance_id": ["a"]}) + result = coerce_catalog_times(catalog) + assert "start_time" not in result.columns + assert "end_time" not in result.columns + + def test_both_columns_present(self): + catalog = pd.DataFrame({"start_time": ["2000-01-01"], "end_time": ["2000-12-31"]}) + result = coerce_catalog_times(catalog) + assert isinstance(result["start_time"].iloc[0], cftime.datetime) + assert isinstance(result["end_time"].iloc[0], cftime.datetime) + + def test_start_time_without_end_time_does_not_raise(self): + """A catalog with only ``start_time`` must not KeyError looking up ``end_time``.""" + catalog = pd.DataFrame({"start_time": ["2000-01-01"]}) + result = coerce_catalog_times(catalog) + assert isinstance(result["start_time"].iloc[0], cftime.datetime) + assert "end_time" not in result.columns + + def test_end_time_without_start_time_does_not_raise(self): + catalog = pd.DataFrame({"end_time": ["2000-12-31"]}) + result = coerce_catalog_times(catalog) + assert isinstance(result["end_time"].iloc[0], cftime.datetime) + assert "start_time" not in result.columns diff --git a/packages/climate-ref/tests/unit/results/test_datasets.py b/packages/climate-ref/tests/unit/results/test_datasets.py index b86d91cf2..4ebf7b8d0 100644 --- a/packages/climate-ref/tests/unit/results/test_datasets.py +++ b/packages/climate-ref/tests/unit/results/test_datasets.py @@ -377,3 +377,15 @@ def test_to_pandas_columns(self, db_with_datasets): for col in ("id", "slug", "dataset_type", "finalised", "created_at", "updated_at"): assert col in df.columns assert len(df) == len(coll) + + def test_to_pandas_columns_when_empty(self, db_with_datasets): + """An empty collection still emits the base columns, so callers can select on them.""" + reader = Reader(db_with_datasets) + coll = reader.datasets.datasets( + DatasetFilter(source_type=SourceDatasetType.CMIP6, facets={"source_id": ("NO-SUCH-MODEL",)}) + ) + assert len(coll) == 0 + df = coll.to_pandas() + assert len(df) == 0 + for col in ("id", "slug", "dataset_type", "finalised", "created_at", "updated_at"): + assert col in df.columns diff --git a/packages/climate-ref/tests/unit/results/test_executions.py b/packages/climate-ref/tests/unit/results/test_executions.py index 651f9de83..6fdc1cd1c 100644 --- a/packages/climate-ref/tests/unit/results/test_executions.py +++ b/packages/climate-ref/tests/unit/results/test_executions.py @@ -348,6 +348,44 @@ def test_raw_rows_diagnostic_filter(self, db_with_groups): diagnostics = {row.diagnostic for row in rows} assert diagnostics == {"enso_tel", "enso-characteristics"} + def test_tied_created_at_does_not_double_count(self, db_with_groups): + """Two executions in the same group tied on ``created_at`` must not double-count that + group's status in the aggregate (regression test for the outerjoin picking up both rows).""" + diag = db_with_groups.session.query(Diagnostic).filter_by(slug="enso_tel").first() + assert diag is not None + eg = db_with_groups.session.query(ExecutionGroup).filter_by(diagnostic_id=diag.id).first() + assert eg is not None + + tied_at = datetime.datetime(2024, 1, 1, tzinfo=datetime.UTC) + session = db_with_groups.session + with session.begin_nested() if session.in_transaction() else session.begin(): + db_with_groups.session.add( + Execution( + execution_group_id=eg.id, + successful=True, + output_fragment="tie-a", + dataset_hash="tie-a-hash", + created_at=tied_at, + ) + ) + db_with_groups.session.add( + Execution( + execution_group_id=eg.id, + successful=True, + output_fragment="tie-b", + dataset_hash="tie-b-hash", + created_at=tied_at, + ) + ) + + stmt = select_execution_statistics(diagnostic_contains=["enso_tel"]) + rows = db_with_groups.session.execute(stmt).all() + by_diag = {row.diagnostic: row for row in rows} + row = by_diag["enso_tel"] + # Still one execution group, still one status count -- not doubled by the tie. + assert row.total == 1 + assert row.successful == 1 + class TestStatistics: def test_status_counts(self, db_with_groups): From a4eb16a6c290117fa8a917e5cb1b191d8f12dbef Mon Sep 17 00:00:00 2001 From: Jared Lewis Date: Sat, 4 Jul 2026 22:33:57 +1000 Subject: [PATCH 15/23] refactor(results): consolidate _as_facets and extract SeriesIndex.bulk_get_or_create Dedupe the _as_facets converter: keep the canonical bare-str-guarded version in dataset_query and import it into results.executions, dropping the read layer's copy that lacked the guard. Move the batch axis-resolution logic out of ingest_series_values into a SeriesIndex.bulk_get_or_create classmethod alongside get_or_create, and add a stress test that pins the axis path to a constant statement count (no N+1). --- .../climate_ref/executor/result_handling.py | 25 ++- .../src/climate_ref/models/dataset_query.py | 16 +- .../src/climate_ref/models/metric_value.py | 57 ++++- .../src/climate_ref/results/executions.py | 19 +- .../executor/test_stress_results_layer.py | 204 ++++++++++++++++++ 5 files changed, 284 insertions(+), 37 deletions(-) create mode 100644 packages/climate-ref/tests/unit/executor/test_stress_results_layer.py diff --git a/packages/climate-ref/src/climate_ref/executor/result_handling.py b/packages/climate-ref/src/climate_ref/executor/result_handling.py index dba124e43..b2502f091 100644 --- a/packages/climate-ref/src/climate_ref/executor/result_handling.py +++ b/packages/climate-ref/src/climate_ref/executor/result_handling.py @@ -10,6 +10,7 @@ """ import pathlib +from collections.abc import Sequence from concurrent.futures import Future from typing import TYPE_CHECKING, get_args @@ -204,20 +205,22 @@ def ingest_series_values( "Diagnostic series values do not conform with the controlled vocabulary", exc_info=True ) - # Resolve (deduplicate) the shared index axes for this batch first, - # so each distinct index is stored once in ``index_axis`` and referenced by id rather - # than duplicated on every series row. - axis_id_by_hash: dict[str, int] = {} - for series_result in series_values: - digest = SeriesIndex.compute_hash(series_result.index_name, series_result.index) - if digest not in axis_id_by_hash: - axis = SeriesIndex.get_or_create(database.session, series_result.index_name, series_result.index) - axis_id_by_hash[digest] = axis.id + # Resolve (deduplicate) the shared index axes for this batch up front, + # so each distinct index is stored once in ``index_axis`` + # and referenced by id rather than duplicated on every series row. + digest_by_series = [ + SeriesIndex.compute_hash(series_result.index_name, series_result.index) + for series_result in series_values + ] + axis_payload_by_hash: dict[str, tuple[str | None, Sequence[float | int | str]]] = {} + for series_result, digest in zip(series_values, digest_by_series, strict=True): + axis_payload_by_hash.setdefault(digest, (series_result.index_name, series_result.index)) + + axis_id_by_hash = SeriesIndex.bulk_get_or_create(database.session, axis_payload_by_hash) new_values = [] - for series_result in series_values: + for series_result, digest in zip(series_values, digest_by_series, strict=True): kind, dimensions = _validated_kind_and_dimensions(series_result) - digest = SeriesIndex.compute_hash(series_result.index_name, series_result.index) row = { "execution_id": execution.id, "values": series_result.values, diff --git a/packages/climate-ref/src/climate_ref/models/dataset_query.py b/packages/climate-ref/src/climate_ref/models/dataset_query.py index e7e684209..3869885a9 100644 --- a/packages/climate-ref/src/climate_ref/models/dataset_query.py +++ b/packages/climate-ref/src/climate_ref/models/dataset_query.py @@ -129,15 +129,8 @@ def select_datasets( stmt = stmt.distinct() if filter.latest_only and latest_group_by: - # Rank by version_key within each partition, then keep only ids at rank 1. Applied after all - # the where-filters/joins above so "latest" is chosen among the already-filtered set. - # - # This uses the ``entity.id.in_()`` shape rather than - # ``aliased(entity, subquery)``: making the outer entity an alias would break the class-bound - # ``selectinload(entity.files)`` loader option that callers attach to the returned ``Select`` - # (``ArgumentError`` at execution), since a bare ``Select`` gives callers no handle on the - # alias to rewrite their options against. Keeping the outer entity a plain class means all - # existing ``.options(...)`` calls keep working unchanged. + # Rank by version_key within each partition, then keep only ids at rank 1. + # Applied after all the where-filters/joins above so "latest" is chosen among the filtered set. # # RANK (not ROW_NUMBER) keeps every row tied at the max version_key in a partition. rank = sa.func.rank().over( @@ -146,9 +139,8 @@ def select_datasets( ) inner = stmt.add_columns(rank.label("_rank")).subquery() latest_ids = select(inner.c.id).where(inner.c._rank == 1) - # Plain filtered select of the entity (not the ``stmt`` built above, which now carries the - # window's subquery machinery) -- the id membership already encodes every filter/join from - # above, so the outer select only needs the entity and the id predicate. + # The id membership already encodes every filter/join from above, + # so the outer select only needs the entity and the id predicate. stmt = select(entity).where(entity.id.in_(latest_ids)) return stmt.order_by(entity.updated_at.desc()) diff --git a/packages/climate-ref/src/climate_ref/models/metric_value.py b/packages/climate-ref/src/climate_ref/models/metric_value.py index bc666b576..0ac18122d 100644 --- a/packages/climate-ref/src/climate_ref/models/metric_value.py +++ b/packages/climate-ref/src/climate_ref/models/metric_value.py @@ -4,7 +4,7 @@ from collections.abc import Mapping, Sequence from typing import TYPE_CHECKING, Any, ClassVar -from sqlalchemy import ForeignKey, event, select +from sqlalchemy import ForeignKey, event, insert, select from sqlalchemy.orm import Mapped, Session, mapped_column, relationship from climate_ref.models.base import Base @@ -224,6 +224,61 @@ def get_or_create( session.flush() return axis + @classmethod + def bulk_get_or_create( + cls, + session: Session, + axes_by_hash: Mapping[str, tuple[str | None, Sequence[float | int | str]]], + ) -> dict[str, int]: + """ + Resolve many axes at once, returning a ``{hash: id}`` map. + + Existing axes are fetched in a single query and any missing axes are bulk-inserted, + so a batch of series values costs two queries rather than one ``get_or_create`` per row. + + Parameters + ---------- + session + Active database session. + axes_by_hash + Content hash (see [compute_hash][climate_ref.models.metric_value.SeriesIndex.compute_hash]) + mapped to the axis ``(name, values)`` it represents. + + Returns + ------- + The shared axis id for every hash in ``axes_by_hash``. + """ + if not axes_by_hash: + return {} + + id_by_hash: dict[str, int] = { + digest: axis_id + for digest, axis_id in session.execute(select(cls.hash, cls.id).where(cls.hash.in_(axes_by_hash))) + } + missing = [digest for digest in axes_by_hash if digest not in id_by_hash] + if missing: + session.execute( + insert(cls), + [ + { + "hash": digest, + "name": axes_by_hash[digest][0], + "values": list(axes_by_hash[digest][1]), + "length": len(axes_by_hash[digest][1]), + } + for digest in missing + ], + ) + id_by_hash.update( + { + digest: axis_id + for digest, axis_id in session.execute( + select(cls.hash, cls.id).where(cls.hash.in_(missing)) + ) + } + ) + return id_by_hash + class SeriesMetricValue(MetricValue): """ diff --git a/packages/climate-ref/src/climate_ref/results/executions.py b/packages/climate-ref/src/climate_ref/results/executions.py index 75314d7b9..7bf5ff24c 100644 --- a/packages/climate-ref/src/climate_ref/results/executions.py +++ b/packages/climate-ref/src/climate_ref/results/executions.py @@ -26,6 +26,7 @@ from sqlalchemy.orm import Session from climate_ref.database import Database +from climate_ref.models.dataset_query import _as_facets from climate_ref.models.diagnostic import Diagnostic from climate_ref.models.execution import ( Execution, @@ -38,15 +39,6 @@ from climate_ref.results._query import latest_execution_for_group -def _as_facets( - value: Mapping[str, Sequence[str]] | None, -) -> Mapping[str, tuple[str, ...]] | None: - """Copy a facets mapping into an immutable ``dict`` of immutable ``tuple`` values.""" - if value is None: - return None - return {k: tuple(v) for k, v in value.items()} - - @attrs.frozen(kw_only=True) class ExecutionGroupFilter: """ @@ -153,10 +145,11 @@ class ExecutionGroupView: The group's most recent execution, or ``None`` if it has never been executed. Reflects the helper's ``max(created_at)`` outer join - (models/execution.py::get_execution_group_and_latest), which can duplicate a group on an - exact ``created_at`` tie. This may differ from ``ExecutionsReader.latest_execution()``, which - tie-breaks by ``created_at DESC, id DESC`` (``_query.latest_execution_for_group``). Both - behaviours are intentional for their respective consumers (``groups()`` vs ``latest_execution()``) + (models/execution.py::get_execution_group_and_latest), + which can duplicate a group on an exact ``created_at`` tie. + This may differ from ``ExecutionsReader.latest_execution()``, which + tie-breaks by ``created_at DESC, id DESC`` (``_query.latest_execution_for_group``). + Both behaviours are intentional for their respective consumers (``groups()`` vs ``latest_execution()``) -- do not unify them. """ diff --git a/packages/climate-ref/tests/unit/executor/test_stress_results_layer.py b/packages/climate-ref/tests/unit/executor/test_stress_results_layer.py new file mode 100644 index 000000000..2954cc70e --- /dev/null +++ b/packages/climate-ref/tests/unit/executor/test_stress_results_layer.py @@ -0,0 +1,204 @@ +""" +Stress / performance harness for the results ingest layer. + +These tests quantify the database round-trips made while ingesting metric values, +by counting the SQL statements issued against the engine. They exist to guard the +axis-resolution path inR +[ingest_series_values][climate_ref.executor.result_handling.ingest_series_values] +against regressing into an N+1 over distinct index axes: the statement count must +stay a small constant regardless of how many distinct axes a batch contains. +""" + +import contextlib +import pathlib +from collections.abc import Iterator + +import pytest +from sqlalchemy import event, func, select + +from climate_ref.database import Database +from climate_ref.executor.result_handling import ingest_series_values +from climate_ref.models import SeriesIndex, SeriesMetricValue +from climate_ref.models.diagnostic import Diagnostic as DiagnosticModel +from climate_ref.models.execution import Execution, ExecutionGroup +from climate_ref.models.provider import Provider as ProviderModel +from climate_ref_core.metric_values import SeriesMetricValue as TSeries +from climate_ref_core.pycmec.controlled_vocabulary import CV + + +class _Result: + """Minimal stand-in for an ``ExecutionResult`` exposing only what ingest needs.""" + + def __init__(self, scratch_dir: pathlib.Path) -> None: + self.series_filename = pathlib.Path("series.json") + self._scratch_dir = scratch_dir + + def to_output_path(self, filename: pathlib.Path | str | None) -> pathlib.Path: + return self._scratch_dir / filename if filename else self._scratch_dir + + +@contextlib.contextmanager +def count_statements(database: Database) -> Iterator[dict[str, int]]: + """Count SQL statements issued against the database engine within the block.""" + counter = {"total": 0, "select": 0, "insert": 0} + + def _after_cursor_execute(conn, cursor, statement, parameters, context, executemany): + counter["total"] += 1 + head = statement.lstrip().split(" ", 1)[0].lower() + if head == "select": + counter["select"] += 1 + elif head == "insert": + counter["insert"] += 1 + + event.listen(database._engine, "after_cursor_execute", _after_cursor_execute) + try: + yield counter + finally: + event.remove(database._engine, "after_cursor_execute", _after_cursor_execute) + + +def _make_execution(database: Database, key: str) -> Execution: + """Create a provider/diagnostic/group/execution chain and return the execution.""" + with database.session.begin(): + provider = ProviderModel(name="stress", slug=f"stress-{key}", version="v0.1.0") + database.session.add(provider) + database.session.flush() + + diagnostic = DiagnosticModel(name="stress", slug=f"stress-{key}", provider_id=provider.id) + database.session.add(diagnostic) + database.session.flush() + + group = ExecutionGroup( + key=key, + diagnostic_id=diagnostic.id, + selectors={}, + dirty=False, + ) + database.session.add(group) + database.session.flush() + + execution = Execution( + execution_group_id=group.id, + successful=True, + output_fragment=f"stress/{key}", + dataset_hash=f"hash-{key}", + ) + database.session.add(execution) + database.session.flush() + + return execution + + +def _distinct_axis_series(n: int) -> list[TSeries]: + """``n`` single-point series, each with a distinct index (so ``n`` distinct axes).""" + return [ + TSeries(dimensions={"source_id": "m"}, values=[float(i)], index=[i], index_name="time") + for i in range(n) + ] + + +def _shared_axis_series(n: int) -> list[TSeries]: + """``n`` series that all share a single common index (so one distinct axis).""" + return [ + TSeries( + dimensions={"source_id": "m"}, + values=[float(i), float(i) + 1, float(i) + 2], + index=[0, 1, 2], + index_name="time", + ) + for i in range(n) + ] + + +def _write_and_ingest( + database: Database, + config, + execution: Execution, + series: list[TSeries], + scratch_root: pathlib.Path, +) -> dict[str, int]: + """Write ``series`` to a scratch file and ingest it, counting SQL statements.""" + # Touch the (post-commit expired) execution outside the counted block so its refresh + # SELECT is not attributed to ingest. + scratch_dir = scratch_root / execution.output_fragment + scratch_dir.mkdir(parents=True, exist_ok=True) + TSeries.dump_to_json(scratch_dir / "series.json", series) + + cv = CV.load_from_file(config.paths.dimensions_cv) + result = _Result(scratch_dir) + + with count_statements(database) as counter: + ingest_series_values(database=database, result=result, execution=execution, cv=cv) + database.session.commit() + return counter + + +# Statement counts are a small constant: one SELECT for existing axes, one bulk INSERT +# for the missing axes, one SELECT to read their ids, and one bulk INSERT for the series +# rows. A little head-room is allowed for incidental statements. +_MAX_INGEST_STATEMENTS = 8 + + +class TestAxisDedupHotspot: + """Guard the batched axis resolution against regressing to a per-axis N+1.""" + + @pytest.mark.filterwarnings("ignore:Unknown dimension values.*:UserWarning") + def test_distinct_axes_do_not_scale_statements(self, db, config, tmp_path): + """3000 distinct-index series must not issue O(n) SQL statements.""" + n = 3000 + execution = _make_execution(db, "distinct") + + counter = _write_and_ingest(db, config, execution, _distinct_axis_series(n), tmp_path) + + # The N+1 version issued ~2n+1 statements (a SELECT + INSERT per axis plus the + # series insert); the batched version collapses that to a small constant. + assert counter["total"] <= _MAX_INGEST_STATEMENTS, counter + + # Every series and every distinct axis is still persisted exactly once. + assert db.session.scalar(select(func.count()).select_from(SeriesMetricValue)) == n + assert db.session.scalar(select(func.count()).select_from(SeriesIndex)) == n + + @pytest.mark.filterwarnings("ignore:Unknown dimension values.*:UserWarning") + def test_statement_count_is_independent_of_batch_size(self, db, config, tmp_path): + """Doubling the number of distinct axes must not change the statement count.""" + small = _write_and_ingest( + db, config, _make_execution(db, "small"), _distinct_axis_series(500), tmp_path + ) + large = _write_and_ingest( + db, config, _make_execution(db, "large"), _distinct_axis_series(1000), tmp_path + ) + + assert small["total"] == large["total"], (small, large) + assert large["total"] <= _MAX_INGEST_STATEMENTS, large + + @pytest.mark.filterwarnings("ignore:Unknown dimension values.*:UserWarning") + def test_shared_axis_matches_distinct_axis_cost(self, db, config, tmp_path): + """A batch sharing one axis costs the same handful of statements as many axes.""" + shared = _write_and_ingest( + db, config, _make_execution(db, "shared"), _shared_axis_series(3000), tmp_path + ) + distinct = _write_and_ingest( + db, config, _make_execution(db, "distinct"), _distinct_axis_series(3000), tmp_path + ) + + assert shared["total"] <= _MAX_INGEST_STATEMENTS, shared + assert distinct["total"] <= _MAX_INGEST_STATEMENTS, distinct + # One shared axis is stored once; 3000 distinct axes stored once each. + assert db.session.scalar(select(func.count()).select_from(SeriesIndex)) == 3001 + + @pytest.mark.filterwarnings("ignore:Unknown dimension values.*:UserWarning") + def test_existing_axes_are_reused_without_reinsert(self, db, config, tmp_path): + """A re-solve over identical axes reuses them: no new index_axis rows, no axis INSERT.""" + first = _make_execution(db, "first") + _write_and_ingest(db, config, first, _distinct_axis_series(500), tmp_path) + axes_after_first = db.session.scalar(select(func.count()).select_from(SeriesIndex)) + db.session.commit() # close the read transaction before starting the next execution + + second = _make_execution(db, "second") + counter = _write_and_ingest(db, config, second, _distinct_axis_series(500), tmp_path) + + # No axes were duplicated across executions... + assert db.session.scalar(select(func.count()).select_from(SeriesIndex)) == axes_after_first + # ...and the only INSERT this time was the series rows themselves (axes were reused). + assert counter["insert"] == 1, counter + assert counter["total"] <= _MAX_INGEST_STATEMENTS, counter From a8745822a4f0c2520e33862735484600970b2488 Mon Sep 17 00:00:00 2001 From: Jared Lewis Date: Sun, 5 Jul 2026 10:06:54 +1000 Subject: [PATCH 16/23] docs: fix docstring typo in results stress test --- .../tests/unit/executor/test_stress_results_layer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/climate-ref/tests/unit/executor/test_stress_results_layer.py b/packages/climate-ref/tests/unit/executor/test_stress_results_layer.py index 2954cc70e..ed6bf28c8 100644 --- a/packages/climate-ref/tests/unit/executor/test_stress_results_layer.py +++ b/packages/climate-ref/tests/unit/executor/test_stress_results_layer.py @@ -3,7 +3,7 @@ These tests quantify the database round-trips made while ingesting metric values, by counting the SQL statements issued against the engine. They exist to guard the -axis-resolution path inR +axis-resolution path in [ingest_series_values][climate_ref.executor.result_handling.ingest_series_values] against regressing into an N+1 over distinct index axes: the statement count must stay a small constant regardless of how many distinct axes a batch contains. From 8155ac5c30eb5f01425cf0693571d84e4cb46c98 Mon Sep 17 00:00:00 2001 From: Jared Lewis Date: Mon, 6 Jul 2026 10:36:36 +1000 Subject: [PATCH 17/23] docs(reader): add a docs entry for the results --- docs/how-to-guides/hpc_executor.md | 17 +- docs/how-to-guides/ingest-datasets.md | 6 +- docs/how-to-guides/reading-results-locally.md | 329 ++++++++++++++++++ .../using-pre-computed-results.py | 4 + mkdocs.yml | 1 + .../src/climate_ref/cli/datasets.py | 70 +++- .../src/climate_ref/cli/diagnostics.py | 6 +- .../src/climate_ref/cli/executions.py | 19 +- ...07-02T0338_f6a7b8c9d0e1_add_version_key.py | 15 +- .../src/climate_ref/models/dataset.py | 29 +- .../src/climate_ref/models/dataset_query.py | 49 ++- .../src/climate_ref/models/metric_value.py | 15 + .../src/climate_ref/results/_query.py | 15 +- .../src/climate_ref/results/_stats.py | 32 ++ .../src/climate_ref/results/datasets.py | 123 +++++-- .../src/climate_ref/results/diagnostics.py | 45 +-- .../src/climate_ref/results/executions.py | 94 +++-- .../src/climate_ref/results/frames.py | 101 ++++-- .../src/climate_ref/results/outliers.py | 110 +++--- .../src/climate_ref/results/values.py | 42 +-- .../tests/unit/cli/test_datasets.py | 35 ++ .../tests/unit/cli/test_diagnostics.py | 3 +- .../tests/unit/cli/test_executions.py | 85 ++++- .../tests/unit/datasets/test_dataset_query.py | 59 +++- .../tests/unit/models/test_series_index.py | 80 +++++ .../tests/unit/results/test_artifacts.py | 34 ++ .../tests/unit/results/test_datasets.py | 194 ++++++++--- .../tests/unit/results/test_diagnostics.py | 15 + .../tests/unit/results/test_executions.py | 84 +++++ .../tests/unit/results/test_results.py | 120 +++++++ 30 files changed, 1486 insertions(+), 345 deletions(-) create mode 100644 docs/how-to-guides/reading-results-locally.md create mode 100644 packages/climate-ref/src/climate_ref/results/_stats.py create mode 100644 packages/climate-ref/tests/unit/models/test_series_index.py diff --git a/docs/how-to-guides/hpc_executor.md b/docs/how-to-guides/hpc_executor.md index 718d13e91..2ffbd3e6f 100644 --- a/docs/how-to-guides/hpc_executor.md +++ b/docs/how-to-guides/hpc_executor.md @@ -6,9 +6,9 @@ Here, we introduce the [HPCExecutor][climate_ref.executor.hpc.HPCExecutor] for r You could use HPCExecutor if: - - The login nodes allow users to run a program for a long time like several hours with little computational resources (less than 25% of one CPU core and negligible memory). - - You want to run REF under the HPC workflow i.e., submitting batch jobs. - - The scheduler on your HPC is __slurm__ or __pbs__. We may include other schedules in the future if needed. Please make an [issue](https://github.com/Climate-REF/climate-ref/issues) describing your requirements. +- The login nodes allow users to run a program for a long time like several hours with little computational resources (less than 25% of one CPU core and negligible memory). +- You want to run REF under the HPC workflow i.e., submitting batch jobs. +- The scheduler on your HPC is __slurm__ or __pbs__. We may include other schedules in the future if needed. Please make an [issue](https://github.com/Climate-REF/climate-ref/issues) describing your requirements. ## Pre-requirements @@ -35,11 +35,14 @@ The HPCExecutor will use the slurm provider and srun launch from parsl to submit 1. On the login nodes, if possible, open a `screen` or `tmux` session to keep the master process of HPCExecutor alive. Only the master process will be run on the login nodes with less than 10% of one CPU core and negligible memory, and real computations will be on compute nodes through the job submissions. If the HPC's queue time is short and the connection to the login nodes is stable, users can use the login nodes directly without using the sessions of `screen` and `tmux`. 2. Edit the `ref.toml` under the config directory if not create one. 3. Change the ref executor to HPCExecutor as follows: + ```toml [executor] executor = "climate_ref.executor.HPCExecutor" ``` + 4. Add the configuration or options for the HPCExecutor based on your system and your account. The example of configuration used in the DOE Perlmutter HPC is as follows: + ```toml [executor.config] scheduler = "slurm" @@ -52,7 +55,9 @@ scheduler_options = "#SBATCH -C cpu" cores_per_worker = 1 max_workers_per_node = 64 ``` + If you are using NCI Gadi, you can use the following configuration: + ```toml [executor.config] scheduler = "pbs" @@ -68,9 +73,9 @@ scheduler_options = "" cores_per_worker = 1 max_workers_per_node = 64 ``` -5. Run `ref solve` in the session of `screen` or `tmux` or directly on the login nodes. It will have a master process running on the login nodes to monitor and submit jobs to run the real diagnostics on compute nodes. -6. Check the logs under the `runinfo` directory and `${REF_CONFIGURATION}/log` to see if there are any errors. Once the master process is done, please run `ref executions list-groups` to check the results or `ref executions inspect id` to see the error message from the providers. +1. Run `ref solve` in the session of `screen` or `tmux` or directly on the login nodes. It will have a master process running on the login nodes to monitor and submit jobs to run the real diagnostics on compute nodes. +2. Check the logs under the `runinfo` directory and `${REF_CONFIGURATION}/log` to see if there are any errors. Once the master process is done, please run `ref executions list-groups` to check the results or `ref executions inspect id` to see the error message from the providers. /// admonition | Note @@ -93,7 +98,7 @@ The configurations of the __HPCExecutor__ for the slurm scheduler are validated - `req_nodes: int`, requested node for the REF run - `validation: boolean`, true to validate the above options with pyslurm -The following __HPCExecutor__ options are directly from parsl. Please refer to [link](https://parsl.readthedocs.io/en/stable/stubs/parsl.providers.SlurmProvider.html) +The following __HPCExecutor__ options are directly from parsl. Please refer to [parsl.providers.SlurmProvider](https://parsl.readthedocs.io/en/stable/stubs/parsl.providers.SlurmProvider.html) - `log_dir: str`, default="run_info" - `cores_per_worker: int`, default=1 diff --git a/docs/how-to-guides/ingest-datasets.md b/docs/how-to-guides/ingest-datasets.md index bc5b5c048..d925daf93 100644 --- a/docs/how-to-guides/ingest-datasets.md +++ b/docs/how-to-guides/ingest-datasets.md @@ -12,7 +12,6 @@ but we recommend using the [intake-esgf](https://github.com/esgf2-us/intake-esgf If you have access to a high-performance computing (HPC) system, you may have a local archive of CMIP6 data already available. - ## What is Ingestion? When processing diagnostics, the REF needs to know the location of the datasets and various metadata. @@ -38,7 +37,7 @@ and the type of the dataset being ingested (only cmip6 is currently supported). This will walk through the provided directory looking for datasets to ingest. Metadata will be extracted from each dataset and stored in the database. -``` +```bash >>> ref --log-level INFO datasets ingest --source-type cmip6 /path/to/cmip6 2024-12-05 12:00:05.979 | INFO | climate_ref.database:__init__:77 - Connecting to database at sqlite:///.climate_ref/db/climate_ref.db 2024-12-05 12:00:05.987 | INFO | alembic.runtime.migration:__init__:215 - Context impl SQLiteImpl. @@ -66,7 +65,6 @@ Metadata will be extracted from each dataset and stored in the database. 2024-12-05 12:00:06.459 | INFO | climate_ref.cli.datasets:ingest:131 - Processing dataset CMIP6.ScenarioMIP.CSIRO.ACCESS-ESM1-5.ssp126.r1i1p1f1.fx.areacella.gn ``` - ### Querying ingested datasets You can query the ingested datasets using the `ref datasets list` command. @@ -74,7 +72,7 @@ This will display a list of datasets and their associated metadata. The `--column` flag allows you to specify which columns to display (defaults to all columns). See `ref datasets list-columns` for a list of available columns. -``` +```bash >>> ref datasets list --column instance_id --column variable_id instance_id variable_id diff --git a/docs/how-to-guides/reading-results-locally.md b/docs/how-to-guides/reading-results-locally.md new file mode 100644 index 000000000..a787badb0 --- /dev/null +++ b/docs/how-to-guides/reading-results-locally.md @@ -0,0 +1,329 @@ +# Reading results locally with pandas + +The [`using-pre-computed-results`](using-pre-computed-results.py) guide pulls results from the +hosted [REF API](https://api.climate-ref.org). +If instead you have a local REF database because you have run diagnostics yourself, +or you are working against a copy of the database +you can read those results straight into pandas without standing up an API. + +This is the job of `climate_ref.results`, +a typed read layer over the REF database. +It returns frozen, ORM-free value objects wrapped in collections that offer `to_pandas()`, +so a DataFrame built inside a `with Database(...)` block keeps working after the session closes. + +!!! note + + This guide is not executed when the documentation is built, + because the documentation build ingests datasets but does not run any diagnostics, + so there are no execution results to read. + Every snippet below has been checked against the current source of + [`climate_ref.results`](../api/climate_ref/results/), + but you will need your own populated database to run them. + +## Opening a database and constructing a `Reader` + +Open the configured database read-only and hand it to a `Reader`: + +```python +from climate_ref.config import Config +from climate_ref.database import Database +from climate_ref.results import Reader + +config = Config.default() + +with Database.from_config(config, read_only=True) as db: + reader = Reader(db) + df = reader.values.scalar_values().to_pandas() + +# `df` is a plain DataFrame and remains valid out here, after the session has closed. +``` + +`read_only=True` opens SQLite in immutable read-only mode and skips migrations, +so the read layer never mutates the database you point it at. + +`Reader` is a thin entry point. +It holds the database and exposes one sub-reader per domain as a property: + +- `reader.values` — scalar and series metric values, +- `reader.executions` — execution groups and executions, +- `reader.datasets` — ingested datasets, +- `reader.diagnostics` — diagnostics and their status counts, +- `reader.artifacts` — filesystem paths for execution outputs. + +`reader.artifacts` needs to know where output files live, +so it is only available when you pass a results root to the constructor: + +```python +with Database.from_config(config, read_only=True) as db: + reader = Reader(db, results=config.paths.results) + + execution = reader.executions.execution(execution_id=42) + for output in reader.executions.outputs(execution_id=42): + path = reader.artifacts.output_file(execution.output_fragment, output.filename) + print(path) +``` + +Constructing `Reader(db)` without a `results` root and then touching `reader.artifacts` +raises a `ValueError` telling you to supply one. + +## Scalar and series values + +`reader.values` reads the two shapes of metric value the REF stores: +single scalar numbers, and 1-d series along a shared index. + +### Scalar values + +`scalar_values()` returns a `ScalarValueCollection`. +Iterate over it for the individual `ScalarValue` objects, +or call `to_pandas()` for a tidy DataFrame with one row per value +and one column per controlled-vocabulary (CV) dimension present: + +```python +from climate_ref.results import MetricValueFilter + +with Database.from_config(config, read_only=True) as db: + reader = Reader(db) + collection = reader.values.scalar_values() + + print(collection.total_count) # values matching the filter, before pagination + print(collection.facets_dict()) # {dimension: [distinct values]} over the full set + + df = collection.to_pandas() +``` + +Pass a `MetricValueFilter` to narrow the query. +Every field is optional; `None`/empty means "do not constrain on this axis". +Dimension filters go through the `dimensions` map, +keyed by any registered CV dimension; +a string is an equality match and a sequence is an `IN` match: + +```python +filters = MetricValueFilter( + diagnostic_slug="global-mean-timeseries", # exact-match slug + provider_slug="example", + dimensions={ + "statistic": "mean", # equality + "source_id": ["ACCESS-ESM1-5", "CanESM5"], # IN + }, +) + +with Database.from_config(config, read_only=True) as db: + df = Reader(db).values.scalar_values(filters=filters).to_pandas() +``` + +An unknown dimension key raises `KeyError` rather than silently returning the wrong data. +`diagnostic_slug`/`provider_slug` are exact matches; +`diagnostic_contains`/`provider_contains` are case-insensitive substring matches for search. + +#### Outlier detection + +Scalar reads can flag outliers using a source-id-aware IQR test, +configured with an `OutlierPolicy`. +Detection is off by default: + +```python +from climate_ref.results import OutlierPolicy + +with Database.from_config(config, read_only=True) as db: + collection = Reader(db).values.scalar_values( + outliers=OutlierPolicy(method="iqr", factor=10.0), + include_unverified=True, # keep flagged rows in the result (default drops them) + ) + print(collection.had_outliers, collection.outlier_count) + df = collection.to_pandas() # gains `is_outlier` / `verification_status` columns +``` + +With `include_unverified=False` (the default) flagged values are removed before pagination +and excluded from `total_count`. +When detection runs, `to_pandas()` adds `is_outlier` and `verification_status` columns. + +### Series values + +`series_values()` returns a `SeriesValueCollection` of `SeriesValue` objects, +each carrying its `values`, the shared `index`, and the `index_name`: + +```python +with Database.from_config(config, read_only=True) as db: + collection = Reader(db).values.series_values( + MetricValueFilter(reference_only=False) # model series only; True for reference/observations + ) + + long_df = collection.to_pandas() # one row per (series, index point) + wide_df = collection.to_pandas(explode=False) # one row per series; list-valued cells +``` + +`reference_only` is a series-only axis: +`True` selects observation/reference series, `False` selects model series. + +### Pagination + +All value reads paginate through `offset`/`limit`, +and `total_count` reports the size of the full match so you can tell there are more rows. +Ordering is deterministic (id-tie-broken), so paging is stable across calls: + +```python +with Database.from_config(config, read_only=True) as db: + page = Reader(db).values.scalar_values(offset=0, limit=100) +``` + +## Executions + +`reader.executions` reads execution groups and the individual executions within them. + +`groups()` returns an `ExecutionGroupCollection`; +each `ExecutionGroupView` carries its `latest` execution (or `None` if it has never run): + +```python +from climate_ref.results import ExecutionGroupFilter + +with Database.from_config(config, read_only=True) as db: + reader = Reader(db) + + groups = reader.executions.groups( + filters=ExecutionGroupFilter( + diagnostic_contains=["sea-ice"], # case-insensitive substring, OR-combined + successful=True, # keep groups whose latest execution succeeded + ) + ) + df = groups.to_pandas() # columns mirror the `list-groups` CLI + + for group in groups: + print(group.id, group.diagnostic_slug, group.successful) +``` + +Fetch a single group or execution by id +(both return `None` when nothing matches), +or resolve the latest execution for a group directly: + +```python +with Database.from_config(config, read_only=True) as db: + reader = Reader(db) + + group = reader.executions.group(execution_group_id=1) + execution = reader.executions.execution(execution_id=42) + latest = reader.executions.latest_execution(execution_group_id=1) +``` + +`group()`, `groups()`, `latest_execution()`, and `statistics()` +all agree on which execution is "latest" for a group +(ranked by `created_at DESC, id DESC`). + +`statistics()` returns per-`(provider, diagnostic)` status counts as `ExecutionStats` objects: + +```python +with Database.from_config(config, read_only=True) as db: + stats = Reader(db).executions.statistics(provider_contains=["esmvaltool"]) + for row in stats: + print(row.provider, row.diagnostic, row.successful, row.failed, row.running) +``` + +## Datasets + +`reader.datasets` reads the datasets that have been ingested into the database. +`list()` returns a `DatasetCollection`; +`to_pandas()` gives one row per dataset with the base columns plus the source-type facets expanded: + +```python +from climate_ref.results import DatasetFilter +from climate_ref_core.source_types import SourceDatasetType + +with Database.from_config(config, read_only=True) as db: + reader = Reader(db) + + datasets = reader.datasets.list( + filters=DatasetFilter(source_type=SourceDatasetType.CMIP6) + ) + df = datasets.to_pandas() +``` + +`DatasetFilter.source_type` is required: +a typed listing has to choose a source type, because facet columns are per-type. +This is a deliberate divergence from the other readers, +whose `filters` argument is optional and defaults to "everything". + +By default `DatasetFilter.latest_only` is `True`, +so a listing returns exactly one row per dataset — the latest version — rather than every version, +deduplicated with that source type's version key. + +`get(slug)` fetches a single dataset by slug, returning the latest version on a tie +(or `None` when no dataset has that slug): + +```python +with Database.from_config(config, read_only=True) as db: + dataset = Reader(db).datasets.get("CMIP6.ScenarioMIP.CSIRO.ACCESS-ESM1-5.ssp126.r1i1p1f1.Omon.tos.gn.v20210318") +``` + +Pass `include_files=True` to `list()` to populate each `DatasetView.files` +with its `DatasetFileView` entries. + +## Diagnostics + +`reader.diagnostics.list()` returns a `DiagnosticCollection` of `DiagnosticView` objects, +each joined to its provider and carrying execution-group counts and promoted-version status: + +```python +from climate_ref.results import DiagnosticFilter + +with Database.from_config(config, read_only=True) as db: + diagnostics = Reader(db).diagnostics.list( + filters=DiagnosticFilter(provider_contains=["pmp"]) + ) + df = diagnostics.to_pandas() + + for diagnostic in diagnostics: + print(diagnostic.provider_slug, diagnostic.slug, diagnostic.successful, diagnostic.total) +``` + +## Artifacts: resolving output paths + +`reader.artifacts` turns the fragments an execution stores +(`output_fragment`, the bundle `path`, an output `filename`) +into filesystem `Path`s under the results root. +It is available only when you construct `Reader(db, results=...)`. + +```python +with Database.from_config(config, read_only=True) as db: + reader = Reader(db, results=config.paths.results) + + execution = reader.executions.execution(execution_id=42) + fragment = execution.output_fragment + + output_dir = reader.artifacts.output_directory(fragment) + log_file = reader.artifacts.log_file(fragment) + bundle = reader.artifacts.bundle(fragment, execution.path) # None when no bundle recorded + + for output in reader.executions.outputs(execution_id=42): + file_path = reader.artifacts.output_file(fragment, output.filename) +``` + +Every resolver is containment-guarded: +a fragment that would escape the results root raises `ValueError` rather than returning a path outside it. +The resolver returns paths only — opening and reading the files stays with you +(e.g. via `xarray.open_dataset`). + +## Package conventions + +`climate_ref.results` keeps its top-level namespace small. +It exports only what you *name to make a call*: + +- the `Reader` entry point, +- the filter objects you construct and pass in + (`MetricValueFilter`, `ExecutionGroupFilter`, `DatasetFilter`, `DiagnosticFilter`), +- the `OutlierPolicy` value object. + +Everything the package *returns* — the DTOs (`ScalarValue`, `ExecutionGroupView`, `DatasetView`, …), +their collections, and the per-domain sub-reader classes reached via `reader.values` etc. — +lives in the relevant domain submodule. +Import them from there on the rare occasion you need to name one: + +```python +from climate_ref.results.values import ScalarValue +from climate_ref.results.executions import ExecutionGroupView +``` + +All paginated reads share the same shape: +`offset`/`limit` control the page, +`total_count` reports the full match, +and ordering is deterministic (tie-broken by primary key) so paging is stable. + +``` diff --git a/docs/how-to-guides/using-pre-computed-results.py b/docs/how-to-guides/using-pre-computed-results.py index a44b16397..b3f856fd0 100644 --- a/docs/how-to-guides/using-pre-computed-results.py +++ b/docs/how-to-guides/using-pre-computed-results.py @@ -22,6 +22,10 @@ # # This Jupyter notebook shows how to use this API to download pre-computed results and use those to do # your own analyses. +# +# If you instead have a local REF database (e.g. from running diagnostics yourself), +# you can read those results straight into pandas without a server using the `climate_ref.results` read layer. +# See [Reading results locally with pandas](reading-results-locally.md). # %% [markdown] # ## Generate and install diff --git a/mkdocs.yml b/mkdocs.yml index 02701908c..9c5f54806 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -28,6 +28,7 @@ nav: - How-to guides: - how-to-guides/index.md - how-to-guides/using-pre-computed-results.py + - how-to-guides/reading-results-locally.md - how-to-guides/executors.md - how-to-guides/hpc_executor.md - how-to-guides/control-memory-use.md diff --git a/packages/climate-ref/src/climate_ref/cli/datasets.py b/packages/climate-ref/src/climate_ref/cli/datasets.py index 9b3529248..e2a802b3e 100644 --- a/packages/climate-ref/src/climate_ref/cli/datasets.py +++ b/packages/climate-ref/src/climate_ref/cli/datasets.py @@ -11,19 +11,45 @@ from pathlib import Path from typing import Annotated +import pandas as pd import typer from loguru import logger from climate_ref.cli._utils import OutputFormat, parse_facet_filters, pretty_print_df, render_dataframe from climate_ref.datasets import get_dataset_adapter from climate_ref.models import Dataset -from climate_ref.solver import apply_dataset_filters +from climate_ref.results import DatasetFilter, Reader +from climate_ref.results.datasets import DatasetCollection from climate_ref_core.dataset_registry import dataset_registry_manager, fetch_all_files from climate_ref_core.source_types import SourceDatasetType app = typer.Typer(help=__doc__) +def _collection_to_dataframe(collection: DatasetCollection, *, include_files: bool) -> pd.DataFrame: + """ + Build the ``datasets list`` DataFrame from a dataset collection. + + Without ``include_files`` this is one row per dataset with its facet columns. + With ``include_files`` each dataset is exploded to one row per file, + with the file columns (``path``, ``start_time``, ``end_time``) added. + """ + if not include_files: + return pd.DataFrame.from_records([dict(ds.facets) for ds in collection]) + + records = [ + { + **dict(ds.facets), + "path": f.path, + "start_time": f.start_time, + "end_time": f.end_time, + } + for ds in collection + for f in ds.files + ] + return pd.DataFrame.from_records(records) + + @app.command(name="list") def list_( # noqa: PLR0913 ctx: typer.Context, @@ -57,28 +83,36 @@ def list_( # noqa: PLR0913 List the datasets that have been ingested The data catalog is sorted by the date that the dataset was ingested (first = newest). - """ - database = ctx.obj.database - adapter = get_dataset_adapter(source_type.value) - data_catalog = adapter.load_catalog(database, include_files=include_files, limit=limit) + Only the latest version of each dataset is listed. + """ + console = ctx.obj.console + reader = Reader(ctx.obj.database) + parsed_filters: dict[str, list[str]] = {} if dataset_filter: try: parsed_filters = parse_facet_filters(dataset_filter) except ValueError as e: raise typer.BadParameter(str(e), param_hint="--dataset-filter") - for facet in parsed_filters: - if facet not in data_catalog.columns: - logger.error( - f"Filter facet '{facet}' not found in data catalog. " - f"Choose from: {', '.join(sorted(data_catalog.columns))}" - ) - raise typer.Exit(code=1) + try: + # ``--include-files`` bounds *files* (not datasets), + # so the SQL row limit is only pushed down in the dataset-per-row case. + # Files are limited in Python after exploding, below. + collection = reader.datasets.list( + DatasetFilter(source_type=source_type, facets=parsed_filters or None), + limit=None if include_files else limit, + include_files=include_files, + ) + except ValueError as e: + # Unknown facet on the target entity: mirror the previous exit-1 behaviour. + logger.error(str(e)) + raise typer.Exit(code=1) - filtered = apply_dataset_filters({source_type: data_catalog}, parsed_filters) - data_catalog = filtered[source_type] # type: ignore[assignment] # input is DataFrame + data_catalog = _collection_to_dataframe(collection, include_files=include_files) + if include_files: + data_catalog = data_catalog.head(limit) if column: missing = set(column) - set(data_catalog.columns) @@ -95,7 +129,13 @@ def format_(columns: Iterable[str]) -> str: raise typer.Exit(code=1) data_catalog = data_catalog[column].sort_values(by=column) - render_dataframe(data_catalog, console=ctx.obj.console, output_format=output_format) + render_dataframe(data_catalog, console=console, output_format=output_format) + + if not include_files and collection.total_count > limit: + logger.warning( + f"Displaying {limit} of {collection.total_count} filtered results. " + f"Use the `--limit` option to display more." + ) @app.command() diff --git a/packages/climate-ref/src/climate_ref/cli/diagnostics.py b/packages/climate-ref/src/climate_ref/cli/diagnostics.py index 92239813e..bb0d1157c 100644 --- a/packages/climate-ref/src/climate_ref/cli/diagnostics.py +++ b/packages/climate-ref/src/climate_ref/cli/diagnostics.py @@ -19,14 +19,14 @@ def list_( # noqa: PLR0913 provider: Annotated[ list[str] | None, typer.Option( - help="Filter by provider slug (substring match, case-insensitive)." + help="Filter by provider slug (substring match, case-insensitive). " "Multiple values can be provided." ), ] = None, diagnostic: Annotated[ list[str] | None, typer.Option( - help="Filter by diagnostic slug (substring match, case-insensitive)." + help="Filter by diagnostic slug (substring match, case-insensitive). " "Multiple values can be provided." ), ] = None, @@ -69,7 +69,7 @@ def list_( # noqa: PLR0913 if column: if not all(col in results_df.columns for col in column): - logger.error(f"Column not found in data catalog: {column}") + logger.error(f"Column not found: {column}") raise typer.Exit(code=1) results_df = results_df[column] diff --git a/packages/climate-ref/src/climate_ref/cli/executions.py b/packages/climate-ref/src/climate_ref/cli/executions.py index 4a3664d3b..ebf763c76 100644 --- a/packages/climate-ref/src/climate_ref/cli/executions.py +++ b/packages/climate-ref/src/climate_ref/cli/executions.py @@ -622,9 +622,9 @@ def values( # noqa: PLR0913 ), ] = False, limit: Annotated[ - int | None, + int, typer.Option(help="Maximum number of values to display."), - ] = None, + ] = 100, offset: Annotated[ int, typer.Option(help="Number of values to skip before displaying."), @@ -700,6 +700,8 @@ def values( # noqa: PLR0913 output_format=output_format, offset=offset, limit=limit, + outliers=outliers, + include_unverified=include_unverified, ) except KeyError as e: # Raised by the filter when a --dimension key is not a registered CV dimension. @@ -728,7 +730,7 @@ def _render_scalar_values( # noqa: PLR0913 ) if not len(collection): if collection.had_outliers: - console.print( + logger.warning( f"No scalar values found. {collection.outlier_count} value(s) were flagged as " f"outliers and hidden. Use --include-unverified to show them." ) @@ -760,7 +762,18 @@ def _render_series_values( # noqa: PLR0913 output_format: OutputFormat, offset: int, limit: int | None, + outliers: bool = False, + include_unverified: bool = False, ) -> None: + # Warn if scalar-only flags are used with series values + ignored_flags = [] + if outliers: + ignored_flags.append("--outliers") + if include_unverified: + ignored_flags.append("--include-unverified") + if ignored_flags: + logger.warning(f"{'/'.join(ignored_flags)} only apply to scalar values and were ignored.") + collection = values.series_values(filters, offset=offset, limit=limit, with_facets=False) if not len(collection): console.print("No series values found.") diff --git a/packages/climate-ref/src/climate_ref/migrations/versions/2026-07-02T0338_f6a7b8c9d0e1_add_version_key.py b/packages/climate-ref/src/climate_ref/migrations/versions/2026-07-02T0338_f6a7b8c9d0e1_add_version_key.py index 0a7119908..5010db8d1 100644 --- a/packages/climate-ref/src/climate_ref/migrations/versions/2026-07-02T0338_f6a7b8c9d0e1_add_version_key.py +++ b/packages/climate-ref/src/climate_ref/migrations/versions/2026-07-02T0338_f6a7b8c9d0e1_add_version_key.py @@ -39,8 +39,14 @@ def _backfill() -> None: """Compute ``version_key`` in Python from each subclass table's ``version`` column. - Batches the ``UPDATE`` by distinct version value (there are typically few distinct versions - relative to the number of datasets), rather than issuing one ``UPDATE`` per row. + ``version_sort_key`` matches a leading ``v`` followed by digits and casts them to an int, + falling back to ``-1``. + That regex + int-cast has no expression that compiles portably across SQLite and PostgreSQL, + so the key is derived in Python and written with one ``UPDATE`` per distinct version value. + + Worst case this is one ``UPDATE`` per row: date-versioned datasets are effectively all-distinct, + so the distinct-version count approaches the row count, + but only runs once. """ bind = op.get_bind() metadata = sa.MetaData() @@ -53,9 +59,8 @@ def _backfill() -> None: for version in distinct_versions: key = version_sort_key(version) # ``version`` is NOT NULL on every subclass table, so ``==`` is correct and portable. - # ``col.is_(value)`` would compile to ``version IS 'v10'``, which SQLite tolerates but - # PostgreSQL rejects (``IS`` only accepts NULL/boolean) -- and the unit tests are - # SQLite-only, so that would surface only on a populated Postgres upgrade. + # ``col.is_(value)`` would compile to ``version IS 'v10'``, + # which SQLite tolerates but PostgreSQL rejects (``IS`` only accepts NULL/boolean). id_subquery = sa.select(sub.c.id).where(sub.c.version == version) bind.execute(dataset.update().where(dataset.c.id.in_(id_subquery)).values(version_key=key)) diff --git a/packages/climate-ref/src/climate_ref/models/dataset.py b/packages/climate-ref/src/climate_ref/models/dataset.py index 179f226f5..230bd82d9 100644 --- a/packages/climate-ref/src/climate_ref/models/dataset.py +++ b/packages/climate-ref/src/climate_ref/models/dataset.py @@ -68,6 +68,11 @@ class Dataset(Base): Lives on the base table (not a subclass) so the SQL latest-version window function (``select_datasets(..., latest_group_by=...)``) can read it off any polymorphic row while partitioning on the subclass's ``dataset_id_metadata`` columns. + + Core writes to ``version`` + (``session.execute(update(...))``, ``connection.execute(...)``, ``bulk_update_mappings``) + bypass the event and leave ``version_key`` stale, silently corrupting latest-version dedup. + ALWAYS mutate ``version`` through an ORM instance. """ def __repr__(self) -> str: @@ -181,6 +186,12 @@ class CMIP6Dataset(Dataset): variant_label: Mapped[str] = mapped_column() vertical_levels: Mapped[int] = mapped_column(nullable=True) version: Mapped[str] = mapped_column() + """ + Dataset version string (e.g. ``"v2"``). + + Only write this through an ORM instance. + The base-table ``version_key`` ordering key is synced by the ``_sync_version_key`` mapper event. + """ instance_id: Mapped[str] = mapped_column(index=True) """ @@ -221,6 +232,12 @@ class Obs4MIPsDataset(Dataset): variable_id: Mapped[str] = mapped_column() variant_label: Mapped[str] = mapped_column() version: Mapped[str] = mapped_column() + """ + Dataset version string. + + Only write this through an ORM instance: the base-table ``version_key`` ordering key is + synced by the ``_sync_version_key`` mapper event, which Core-level updates bypass. + """ vertical_levels: Mapped[int] = mapped_column() source_version_number: Mapped[str] = mapped_column() @@ -256,6 +273,12 @@ class PMPClimatologyDataset(Dataset): variable_id: Mapped[str] = mapped_column() variant_label: Mapped[str] = mapped_column() version: Mapped[str] = mapped_column() + """ + Dataset version string. + + Only write this through an ORM instance: the base-table ``version_key`` ordering key is + synced by the ``_sync_version_key`` mapper event, which Core-level updates bypass. + """ vertical_levels: Mapped[int] = mapped_column() source_version_number: Mapped[str] = mapped_column() @@ -309,7 +332,11 @@ class CMIP7Dataset(Dataset): """Template - e.g., "tavg-h2m-hxy-u" """ version: Mapped[str] = mapped_column() - """Template - e.g., "v20250622" """ + """Template - e.g., "v20250622". + + Only write this through an ORM instance. + The base-table ``version_key`` ordering key is synced by the ``_sync_version_key`` mapper event. + """ # Additional Mandatory Attributes mip_era: Mapped[str] = mapped_column() diff --git a/packages/climate-ref/src/climate_ref/models/dataset_query.py b/packages/climate-ref/src/climate_ref/models/dataset_query.py index 3869885a9..7094c1a5e 100644 --- a/packages/climate-ref/src/climate_ref/models/dataset_query.py +++ b/packages/climate-ref/src/climate_ref/models/dataset_query.py @@ -40,18 +40,15 @@ class DatasetFilter: """ Declarative filter over datasets. - Every field is optional. - ``None`` means "do not constrain on this axis". - - ``source_type`` selects which concrete ``Dataset`` subclass - (and therefore which facet columns) the query targets. - When ``source_type=None``, the query stays on the base ``Dataset`` database table, - so only base columns are filterable via ``facets`` - (``slug``, ``finalised``, ``dataset_type``, ``created_at``, ``updated_at``), - and ``latest_only`` is a no-op as there is no ``dataset_id_metadata`` to group by. + ``source_type`` is required. + It selects which which facet columns the query can target. + This limits our filtering to a single source type at a time to ensure that the files can be + collapsed into a dataframe. + + Every other field is optional with ``None`` meaning "do not constrain on this axis". """ - source_type: SourceDatasetType | None = None + source_type: SourceDatasetType facets: Mapping[str, tuple[str, ...]] | None = attrs.field(default=None, converter=_as_facets) finalised: bool | None = None execution_id: int | None = None @@ -59,10 +56,8 @@ class DatasetFilter: latest_only: bool = True -def _entity_for(source_type: SourceDatasetType | None) -> type[Dataset]: +def _entity_for(source_type: SourceDatasetType) -> type[Dataset]: """Resolve a source type to its concrete ``Dataset`` subclass via the polymorphic map.""" - if source_type is None: - return Dataset return cast(type[Dataset], Dataset.__mapper__.polymorphic_map[source_type].class_) @@ -72,19 +67,20 @@ def select_datasets( latest_group_by: Sequence[str] | None = None, ) -> Select[Any]: """ - Build the ``Select`` over the (optionally concrete) ``Dataset`` entity for the given filter. + Build the ``Select`` over the ``Dataset`` subclass for the given filter. - Any limit is deliberately not applied here; callers apply it so a numeric limit is not spent - on superseded versions. + Any limit is deliberately not applied here. + Callers should apply limits after filtering out superseded versions. - ``latest_group_by`` is the adapter's ``dataset_id_metadata`` -- the partition columns for the - latest-version window. It is optional because ``select_datasets`` lives in the models layer and - must not import the adapter registry, so it cannot look this up itself; callers pass it through. + ``latest_group_by`` is the adapter's ``dataset_id_metadata``, + which is used as the partition columns for the latest-version window. + It is optional because ``select_datasets`` lives in the models layer + and must not import the adapter registry, so it cannot look this up itself; callers pass it through. - ``filter.latest_only`` is INERT unless ``latest_group_by`` is also given (non-empty): passing - ``latest_only=True`` alone does NOT dedup. Both must be set together for SQL-side deduplication - to apply. When both are set, rows are deduplicated with a ``RANK() OVER (PARTITION BY - ORDER BY version_key DESC)`` window (applied after all other filters/joins), + ``filter.latest_only`` does not take effect unless ``latest_group_by`` is also given (non-empty). + When both are set, rows are deduplicated with a + ``RANK() OVER (PARTITION BY ORDER BY version_key DESC)`` window + (applied after all other filters/joins), keeping every row tied at the maximum ``version_key`` -- so ties are not silently dropped. Raises @@ -94,9 +90,10 @@ def select_datasets( """ entity = _entity_for(filter.source_type) - stmt = select(entity) - if filter.source_type is not None: - stmt = stmt.where(entity.dataset_type == filter.source_type) + if filter.latest_only and not latest_group_by: + raise ValueError("`latest_group_by` must be provided when `latest_only` is True") + + stmt = select(entity).where(entity.dataset_type == filter.source_type) for facet, values in (filter.facets or {}).items(): column = getattr(entity, facet, None) diff --git a/packages/climate-ref/src/climate_ref/models/metric_value.py b/packages/climate-ref/src/climate_ref/models/metric_value.py index 0ac18122d..1b3ca7807 100644 --- a/packages/climate-ref/src/climate_ref/models/metric_value.py +++ b/packages/climate-ref/src/climate_ref/models/metric_value.py @@ -247,10 +247,25 @@ def bulk_get_or_create( Returns ------- The shared axis id for every hash in ``axes_by_hash``. + + Raises + ------ + ValueError + If any key does not equal ``compute_hash(name, values)`` for the axis it maps to. + A mismatched key would otherwise insert a row whose stored ``hash`` does not match its content, + breaking deduplication. """ if not axes_by_hash: return {} + for digest, (name, values) in axes_by_hash.items(): + expected = cls.compute_hash(name, values) + if digest != expected: + raise ValueError( + f"axes_by_hash key {digest!r} does not match compute_hash of its axis " + f"(expected {expected!r})" + ) + id_by_hash: dict[str, int] = { digest: axis_id for digest, axis_id in session.execute(select(cls.hash, cls.id).where(cls.hash.in_(axes_by_hash))) diff --git a/packages/climate-ref/src/climate_ref/results/_query.py b/packages/climate-ref/src/climate_ref/results/_query.py index bea9a3878..972649e56 100644 --- a/packages/climate-ref/src/climate_ref/results/_query.py +++ b/packages/climate-ref/src/climate_ref/results/_query.py @@ -158,9 +158,14 @@ def _apply_common( # noqa: PLR0912 def select_scalar_values(f: MetricValueFilter | None = None) -> Select[tuple[ScalarMetricValue]]: - """Build a ``Select`` over ``ScalarMetricValue`` for the given filter. No session required.""" + """ + Build a ``Select`` over ``ScalarMetricValue`` for the given filter. + + Ordered by the value id ascending so SQL pagination is deterministic across pages. + """ f = f or MetricValueFilter() - return _apply_common(select(ScalarMetricValue), ScalarMetricValue, f) + stmt = _apply_common(select(ScalarMetricValue), ScalarMetricValue, f) + return stmt.order_by(ScalarMetricValue.id) def select_series_values(f: MetricValueFilter | None = None) -> Select[tuple[SeriesMetricValue]]: @@ -169,6 +174,7 @@ def select_series_values(f: MetricValueFilter | None = None) -> Select[tuple[Ser The shared index axis is eager-loaded via the model relationship (``lazy="joined"``), so ``.index`` / ``.index_name`` are safe to read for the returned rows. + Ordered by the value id ascending so SQL pagination is deterministic across pages. """ f = f or MetricValueFilter() stmt = _apply_common(select(SeriesMetricValue), SeriesMetricValue, f) @@ -176,7 +182,7 @@ def select_series_values(f: MetricValueFilter | None = None) -> Select[tuple[Ser stmt = stmt.where(SeriesMetricValue.reference_id.is_not(None)) elif f.reference_only is False: stmt = stmt.where(SeriesMetricValue.reference_id.is_(None)) - return stmt + return stmt.order_by(SeriesMetricValue.id) def count_values(session: Session, stmt: Select[Any]) -> int: @@ -189,8 +195,7 @@ def latest_execution_for_group(session: Session, execution_group_id: int) -> Exe Return the most recent [Execution][climate_ref.models.execution.Execution] for a group. This is the read-side primitive behind the API's ``/executions/{group_id}/values`` behaviour - of defaulting to the latest execution when no ``execution_id`` is supplied. Centralising it - here means consumers stop re-deriving the "latest execution" lookup. + of defaulting to the latest execution when no ``execution_id`` is supplied. """ return session.execute( select(Execution) diff --git a/packages/climate-ref/src/climate_ref/results/_stats.py b/packages/climate-ref/src/climate_ref/results/_stats.py new file mode 100644 index 000000000..8f280a689 --- /dev/null +++ b/packages/climate-ref/src/climate_ref/results/_stats.py @@ -0,0 +1,32 @@ +"""Shared execution-statistics mapping for the results read layer.""" + +from collections.abc import Iterable +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from climate_ref.results.executions import ExecutionStats + + +def rows_to_execution_stats(rows: Iterable[Any]) -> "tuple[ExecutionStats, ...]": + """ + Map raw ``select_execution_statistics`` rows to detached ``ExecutionStats`` DTOs. + + Both [ExecutionsReader.statistics][climate_ref.results.executions.ExecutionsReader.statistics] + and [DiagnosticsReader.stats][climate_ref.results.diagnostics.DiagnosticsReader.stats] delegate + here so the row-to-DTO mapping is defined exactly once. + """ + from climate_ref.results.executions import ExecutionStats # noqa: PLC0415 + + return tuple( + ExecutionStats( + provider=row.provider, + diagnostic=row.diagnostic, + running=row.running, + failed=row.failed, + successful=row.successful, + not_started=row.not_started, + dirty=row.dirty, + total=row.total, + ) + for row in rows + ) diff --git a/packages/climate-ref/src/climate_ref/results/datasets.py b/packages/climate-ref/src/climate_ref/results/datasets.py index 00cf84f9e..e398ffe3d 100644 --- a/packages/climate-ref/src/climate_ref/results/datasets.py +++ b/packages/climate-ref/src/climate_ref/results/datasets.py @@ -16,7 +16,8 @@ import attrs import pandas as pd -from sqlalchemy.orm import selectin_polymorphic, selectinload +from sqlalchemy import Select, func, select +from sqlalchemy.orm import Session, selectin_polymorphic, selectinload from climate_ref.database import Database from climate_ref.datasets import get_dataset_adapter @@ -75,16 +76,32 @@ class DatasetView: @attrs.frozen(kw_only=True) class DatasetCollection: - """An immutable collection of datasets.""" + """ + An immutable page of datasets plus collection-level metadata. + + ``total_count`` is the number of datasets matching the filter *before* pagination, so a caller + can tell there are more rows than the returned page. ``offset``/``limit`` echo back the + pagination applied to produce ``items`` (``limit`` is ``None`` when the whole result was + returned). + """ + + items: tuple[DatasetView, ...] + """The datasets on this page.""" + + total_count: int + """Total datasets matching the filter before ``offset``/``limit``.""" + + offset: int + """Rows skipped before this page.""" - datasets: tuple[DatasetView, ...] - """The datasets in this collection.""" + limit: int | None + """Page size requested, or ``None`` when the whole result was returned.""" def __iter__(self) -> Iterator[DatasetView]: - return iter(self.datasets) + return iter(self.items) def __len__(self) -> int: - return len(self.datasets) + return len(self.items) def to_pandas(self) -> pd.DataFrame: """ @@ -97,7 +114,7 @@ def to_pandas(self) -> pd.DataFrame: """ base_columns = ["id", "slug", "dataset_type", "finalised", "created_at", "updated_at"] records = [] - for ds in self.datasets: + for ds in self.items: rec: dict[str, Any] = { "id": ds.id, "slug": ds.slug, @@ -111,6 +128,11 @@ def to_pandas(self) -> pd.DataFrame: return pd.DataFrame.from_records(records, columns=base_columns if not records else None) +def _dataset_subtypes() -> list[type[Dataset]]: + """Every concrete ``Dataset`` subclass, for polymorphic eager-loading.""" + return [m.class_ for m in Dataset.__mapper__.polymorphic_map.values()] + + class DatasetsReader: """ Dataset read domain. @@ -118,11 +140,22 @@ class DatasetsReader: Constructed from a [Database][climate_ref.database.Database], which owns the session. All read methods return detached DTOs that outlive the session. + + ``list`` requires a ``DatasetFilter`` (with its required ``source_type``), so unlike the other + readers -- whose ``filters`` argument is optional and defaults to "everything" -- there is no + useful all-datasets default here. This is a deliberate, documented divergence from that shared + contract: dataset facet columns are per-type, so a typed listing has to choose the type. ``get`` + keeps taking a bare slug, which is globally unique and needs no ``source_type``. """ def __init__(self, database: Database) -> None: self._db = database + @property + def session(self) -> Session: + """The underlying database session.""" + return self._db.session + def _to_view(self, dataset: Dataset, *, include_files: bool) -> DatasetView: adapter = get_dataset_adapter(dataset.dataset_type.value) facets = {k: getattr(dataset, k) for k in adapter.dataset_specific_metadata if hasattr(dataset, k)} @@ -147,41 +180,65 @@ def _to_view(self, dataset: Dataset, *, include_files: bool) -> DatasetView: files=files, ) - def datasets( + def _base_statement(self, filter: DatasetFilter) -> Select[Any]: # noqa: A002 + """ + Build the (unpaginated, unordered) ``SELECT`` over the concrete entity for the filter. + + Deduplication to the latest version happens in SQL via a ``RANK`` window keyed off the + concrete adapter's ``dataset_id_metadata`` (inert when ``filter.latest_only`` is ``False``). + """ + adapter = get_dataset_adapter(filter.source_type.value) + return select_datasets(filter, latest_group_by=adapter.dataset_id_metadata) + + def list( self, - filter: DatasetFilter | None = None, # noqa: A002 + filters: DatasetFilter, *, + offset: int = 0, limit: int | None = None, include_files: bool = False, ) -> DatasetCollection: """ - Query datasets, optionally scoped to a source type, execution or diagnostic. + Query one source type's datasets, optionally scoped to an execution or diagnostic. - Deduplication to the latest version (when ``filter.latest_only``) happens in SQL via a - ``RANK`` window, keyed off the adapter's ``dataset_id_metadata``. ``limit`` is pushed into - the same statement, so it is applied after dedup, over the ordered, latest datasets -- - matching ``adapter.load_catalog``'s dedup-then-limit ordering, and only fetching the rows - actually returned instead of the whole table. + ``filters`` is required (its ``source_type`` picks the concrete type and hence the facet + columns). Deduplication to the latest version (``filters.latest_only``, the default) happens + in SQL via a ``RANK`` window keyed off the adapter's ``dataset_id_metadata``, so the default + call returns exactly one row per dataset rather than every version. + + Pagination (``offset``/``limit``) is applied in SQL after dedup, over rows ordered by + ``(slug, id)`` so paging is deterministic even when two datasets share a slug. ``total_count`` + is computed from a separate unpaged count over the same deduplicated statement. """ - filter = filter or DatasetFilter() # noqa: A001 - - adapter = get_dataset_adapter(filter.source_type.value) if filter.source_type else None - entity: type[Dataset] = adapter.dataset_cls if adapter else Dataset - - latest_group_by = adapter.dataset_id_metadata if adapter is not None else None - stmt = select_datasets(filter, latest_group_by=latest_group_by) - if adapter is None: - # Mixed listing over the base ``Dataset`` entity: eager-load every polymorphic subtype - # table up front (one extra SELECT per subtype) so ``_to_view``'s ``getattr(dataset, - # k)`` subtype-facet reads don't lazy-load one row at a time (N+1). - subtypes = [m.class_ for m in Dataset.__mapper__.polymorphic_map.values()] - stmt = stmt.options(selectin_polymorphic(Dataset, subtypes)) + base_stmt = self._base_statement(filters) + count_stmt = select(func.count()).select_from(base_stmt.subquery()) + total_count = self.session.execute(count_stmt).scalar_one() + + stmt = base_stmt.order_by(None).order_by(Dataset.slug, Dataset.id) if include_files: - stmt = stmt.options(selectinload(entity.files)) # type: ignore[attr-defined] + stmt = stmt.options(selectinload(Dataset.files)) # type: ignore[attr-defined] if limit is not None: - stmt = stmt.limit(limit) + stmt = stmt.offset(offset).limit(limit) + elif offset: + stmt = stmt.offset(offset) - session = self._db.session - rows = list(session.execute(stmt).scalars().unique().all()) + rows = list(self.session.execute(stmt).scalars().unique().all()) + items = tuple(self._to_view(r, include_files=include_files) for r in rows) - return DatasetCollection(datasets=tuple(self._to_view(r, include_files=include_files) for r in rows)) + return DatasetCollection(items=items, total_count=total_count, offset=offset, limit=limit) + + def get(self, slug: str) -> DatasetView | None: + """ + Fetch one dataset by slug, or ``None`` when no dataset has that slug. + + When multiple rows share a slug (different versions), the latest is returned, ranked by + ``version_key`` then ``id`` so the choice is deterministic on a version tie. + """ + stmt = ( + select(Dataset) + .where(Dataset.slug == slug) + .order_by(Dataset.version_key.desc(), Dataset.id.desc()) + .options(selectin_polymorphic(Dataset, _dataset_subtypes())) + ) + dataset = self.session.execute(stmt).scalars().first() + return self._to_view(dataset, include_files=False) if dataset is not None else None diff --git a/packages/climate-ref/src/climate_ref/results/diagnostics.py b/packages/climate-ref/src/climate_ref/results/diagnostics.py index 1b8e1b838..97365fc29 100644 --- a/packages/climate-ref/src/climate_ref/results/diagnostics.py +++ b/packages/climate-ref/src/climate_ref/results/diagnostics.py @@ -21,6 +21,7 @@ from climate_ref.models.execution import ExecutionGroup from climate_ref.models.provider import Provider from climate_ref.results._converters import _as_str_tuple +from climate_ref.results._stats import rows_to_execution_stats from climate_ref.results.executions import ExecutionStats, select_execution_statistics @@ -139,16 +140,17 @@ def to_pandas(self) -> pd.DataFrame: return pd.DataFrame.from_records(records, columns=columns) -def select_diagnostics(filter: DiagnosticFilter | None = None) -> Select[Any]: # noqa: A002 +def select_diagnostics(filters: DiagnosticFilter | None = None) -> Select[Any]: """ Build the ``Select`` for diagnostics joined to their provider, with an execution-group count. ``execution_group_count`` counts every ``ExecutionGroup`` row for the diagnostic (all versions, not scoped to ``promoted_version``), so it reflects the diagnostic's full execution history. - Ordered by ``(Provider.slug, Diagnostic.slug)`` for stable output. + Ordered by ``(Provider.slug, Diagnostic.slug, Diagnostic.id)``. + The diagnostic primary key is the final tiebreak so SQL pagination is deterministic across pages. """ - filter = filter or DiagnosticFilter() # noqa: A001 + filters = filters or DiagnosticFilter() group_count_subquery = ( select( @@ -169,13 +171,15 @@ def select_diagnostics(filter: DiagnosticFilter | None = None) -> Select[Any]: ) .join(Provider, Diagnostic.provider_id == Provider.id) .outerjoin(group_count_subquery, Diagnostic.id == group_count_subquery.c.diagnostic_id) - .order_by(Provider.slug, Diagnostic.slug) + .order_by(Provider.slug, Diagnostic.slug, Diagnostic.id) ) - if filter.provider_contains: - stmt = stmt.where(or_(*(Provider.slug.ilike(f"%{s.lower()}%") for s in filter.provider_contains))) - if filter.diagnostic_contains: - stmt = stmt.where(or_(*(Diagnostic.slug.ilike(f"%{s.lower()}%") for s in filter.diagnostic_contains))) + if filters.provider_contains: + stmt = stmt.where(or_(*(Provider.slug.ilike(f"%{s.lower()}%") for s in filters.provider_contains))) + if filters.diagnostic_contains: + stmt = stmt.where( + or_(*(Diagnostic.slug.ilike(f"%{s.lower()}%") for s in filters.diagnostic_contains)) + ) return stmt @@ -211,7 +215,7 @@ def _to_view(self, row: Any, stats_by_key: Mapping[tuple[str, str], ExecutionSta def list( self, - filter: DiagnosticFilter | None = None, # noqa: A002 + filters: DiagnosticFilter | None = None, *, offset: int = 0, limit: int | None = None, @@ -227,9 +231,9 @@ def list( Diagnostics with no execution groups at the promoted version get zeros. """ - filter = filter or DiagnosticFilter() # noqa: A001 + filters = filters or DiagnosticFilter() - base_stmt = select_diagnostics(filter) + base_stmt = select_diagnostics(filters) count_stmt = select(func.count()).select_from(base_stmt.subquery()) total_count = self.session.execute(count_stmt).scalar_one() @@ -242,8 +246,8 @@ def list( rows = self.session.execute(stmt).all() stats = self.stats( - provider_contains=filter.provider_contains, - diagnostic_contains=filter.diagnostic_contains, + provider_contains=filters.provider_contains, + diagnostic_contains=filters.diagnostic_contains, ) stats_by_key = {(s.provider, s.diagnostic): s for s in stats} items = tuple(self._to_view(row, stats_by_key) for row in rows) @@ -268,17 +272,4 @@ def stats( stmt = select_execution_statistics( diagnostic_contains=diagnostic_contains, provider_contains=provider_contains ) - rows = self.session.execute(stmt).all() - return tuple( - ExecutionStats( - provider=row.provider, - diagnostic=row.diagnostic, - running=row.running, - failed=row.failed, - successful=row.successful, - not_started=row.not_started, - dirty=row.dirty, - total=row.total, - ) - for row in rows - ) + return rows_to_execution_stats(self.session.execute(stmt).all()) diff --git a/packages/climate-ref/src/climate_ref/results/executions.py b/packages/climate-ref/src/climate_ref/results/executions.py index 7bf5ff24c..5bcdeb83d 100644 --- a/packages/climate-ref/src/climate_ref/results/executions.py +++ b/packages/climate-ref/src/climate_ref/results/executions.py @@ -15,6 +15,9 @@ - ``executor/reingest.py::get_executions_for_reingest`` (needs ``include_superseded=True`` plus the *oldest* execution in a group, not this reader's "latest" definition). + +A common set of ordering (``created_at DESC, id DESC``) is used to ensure consistent ordering and tie breaks. + """ from collections.abc import Iterator, Mapping, Sequence @@ -37,6 +40,7 @@ from climate_ref.models.provider import Provider from climate_ref.results._converters import _as_str_tuple from climate_ref.results._query import latest_execution_for_group +from climate_ref.results._stats import rows_to_execution_stats @attrs.frozen(kw_only=True) @@ -46,9 +50,10 @@ class ExecutionGroupFilter: Every field is optional; ``None`` means "do not constrain on this axis". This mirrors exactly what [get_execution_group_and_latest_filtered] - [climate_ref.models.execution.get_execution_group_and_latest_filtered] supports today -- + [climate_ref.models.execution.get_execution_group_and_latest_filtered] supports -- ``diagnostic_contains``/``provider_contains`` are case-insensitive substring matches (OR-combined), - ``facets`` is the selector-facet map consumed by the helper's Python-side matching. + ``facets`` is the selector-facet map consumed by the helper's Python-side matching, + and ``successful`` vs ``latest_successful`` expose the helper's post-rank / pre-rank success filters. Exact-match ``diagnostic_slug``/``provider_slug`` are a documented follow-up, as the underlying helper only does substring matching today, @@ -65,7 +70,26 @@ class ExecutionGroupFilter: """Constrain on the group's ``dirty`` flag.""" successful: bool | None = None - """Constrain on the latest execution's ``successful`` status.""" + """ + Post-rank filter on the *winning* execution: keep a group only if its latest execution matches. + + ``True`` keeps groups whose latest execution succeeded. + ``False`` keeps groups whose latest execution failed, is in progress, or does not exist yet. + This does not change which execution is considered latest -- contrast ``latest_successful``. + """ + + latest_successful: bool | None = None + """ + Pre-rank population filter: change which execution is chosen as "latest" before ranking. + + ``True`` ranks only over successful executions, + so a group's ``latest`` becomes its most recent *successful* run + (surfacing an earlier success even when a later run failed). + ``False`` ranks only over unsuccessful / in-progress runs. + + ``None`` (default) ranks over all executions. + Composes with ``successful`` but answers a different question. + """ facets: Mapping[str, tuple[str, ...]] | None = attrs.field(default=None, converter=_as_facets) """Selector-facet map matched against each group's ``selectors``.""" @@ -144,13 +168,8 @@ class ExecutionGroupView: """ The group's most recent execution, or ``None`` if it has never been executed. - Reflects the helper's ``max(created_at)`` outer join - (models/execution.py::get_execution_group_and_latest), - which can duplicate a group on an exact ``created_at`` tie. - This may differ from ``ExecutionsReader.latest_execution()``, which - tie-breaks by ``created_at DESC, id DESC`` (``_query.latest_execution_for_group``). - Both behaviours are intentional for their respective consumers (``groups()`` vs ``latest_execution()``) - -- do not unify them. + These groups are ranked by ``created_at DESC, id DESC``. + This matches ``ExecutionsReader.latest_execution()`` and the ``statistics()`` aggregates. """ @property @@ -428,12 +447,18 @@ def groups( Query execution groups with their per-group latest execution. Wraps [get_execution_group_and_latest_filtered] - [climate_ref.models.execution.get_execution_group_and_latest_filtered] exactly: the full - filtered list is materialised, ``total_count`` is its length, and the page is a Python - slice (``all[offset:offset + limit]``). This preserves today's ``list-groups`` pagination - and ordering exactly -- no SQL-level pagination or additional ``order_by`` is introduced - here. + [climate_ref.models.execution.get_execution_group_and_latest_filtered], + which returns a materialised list. + The ``facets`` axis is matched in Python (against each group's ``selectors``), + and the helper materialises unconditionally, + so pagination here is a Python slice over the fully materialised list rather than SQL. + + ``total_count`` is the post-filter length, + and the results are ordered by ``id`` ascending (the group primary key) + before slicing so paging is deterministic even when two groups share a ``created_at``. """ + # Pushing SQL pagination through the helper would be wrong whenever ``facets`` is set, + # since the Python facet filter runs after the SQL query. filters = filters or ExecutionGroupFilter() all_results = get_execution_group_and_latest_filtered( @@ -443,8 +468,10 @@ def groups( facet_filters={k: list(v) for k, v in filters.facets.items()} if filters.facets else None, dirty=filters.dirty, successful=filters.successful, + latest_successful=filters.latest_successful, include_superseded=filters.include_superseded, ) + all_results = sorted(all_results, key=lambda pair: pair[0].id) total_count = len(all_results) page = all_results[offset : offset + limit] if limit is not None else all_results[offset:] @@ -462,33 +489,36 @@ def statistics( stmt = select_execution_statistics( diagnostic_contains=diagnostic_contains, provider_contains=provider_contains ) - rows = self.session.execute(stmt).all() - return tuple( - ExecutionStats( - provider=row.provider, - diagnostic=row.diagnostic, - running=row.running, - failed=row.failed, - successful=row.successful, - not_started=row.not_started, - dirty=row.dirty, - total=row.total, - ) - for row in rows - ) + return rows_to_execution_stats(self.session.execute(stmt).all()) def latest_execution(self, execution_group_id: int) -> ExecutionView | None: """ Return the most recent execution for a group. Wraps [latest_execution_for_group][climate_ref.results._query.latest_execution_for_group], - which tie-breaks by ``created_at DESC, id DESC``. See the note on - [ExecutionGroupView.latest][climate_ref.results.executions.ExecutionGroupView] for how this - differs from the per-group latest returned by ``groups()`` on an exact timestamp tie. + which tie-breaks using the same ranking ``groups()`` and ``statistics()`` use + (``created_at DESC, id DESC``). """ execution = latest_execution_for_group(self.session, execution_group_id) return self._to_execution_view(execution) if execution is not None else None + def group(self, execution_group_id: int) -> ExecutionGroupView | None: + """ + Fetch one execution group by id, with its latest execution resolved. + + Returns ``None`` when no group has that id. + """ + eg = self.session.get(ExecutionGroup, execution_group_id) + if eg is None: + return None + latest = latest_execution_for_group(self.session, execution_group_id) + return self._to_group_view(eg, latest) + + def execution(self, execution_id: int) -> ExecutionView | None: + """Fetch one execution by id, or ``None`` when no execution has that id.""" + execution = self.session.get(Execution, execution_id) + return self._to_execution_view(execution) if execution is not None else None + def outputs(self, execution_id: int) -> tuple[OutputView, ...]: """Execute the ``select_execution_outputs`` builder and map rows to ``OutputView``.""" stmt = select_execution_outputs(execution_id) diff --git a/packages/climate-ref/src/climate_ref/results/frames.py b/packages/climate-ref/src/climate_ref/results/frames.py index 851753100..9fef20658 100644 --- a/packages/climate-ref/src/climate_ref/results/frames.py +++ b/packages/climate-ref/src/climate_ref/results/frames.py @@ -1,65 +1,92 @@ """ DataFrame conversion and facet collection for metric values. -These pure helpers replace logic duplicated across the CLI (hand-rolled ``pd.DataFrame([...])`` -comprehensions) and the ref-app API (CSV flattening + ``collect_facets_from_query``). They take -rows / a ``Select`` and return plain data, so they can be reused by any consumer. +These pure helpers are the single source of truth for the column layout of a metric-value frame. +Both the collections' ``to_pandas()`` and any other consumer build their frames here, +so a scalar (or series) frame has identical columns. +The builders take detached DTOs (from [climate_ref.results.values][]) rather than ORM rows, +so they never touch a session and are portable. """ from collections.abc import Sequence -from typing import Any +from typing import TYPE_CHECKING, Any import pandas as pd from sqlalchemy import Select, distinct from sqlalchemy.orm import Session -from climate_ref.models.metric_value import ( - MetricValue, - ScalarMetricValue, - SeriesMetricValue, -) +from climate_ref.models.metric_value import MetricValue +if TYPE_CHECKING: + from climate_ref.results.values import ScalarValue, SeriesValue -def scalar_values_to_frame(rows: Sequence[ScalarMetricValue]) -> pd.DataFrame: + +def scalar_values_to_frame(values: "Sequence[ScalarValue]", *, detection_ran: bool = False) -> pd.DataFrame: """ - Flatten scalar rows to a tidy DataFrame. + Flatten scalar value DTOs to a tidy DataFrame. + + One row per value; one column per CV dimension present, plus ``id``, ``execution_id``, + ``execution_group_id``, ``kind`` and ``value``. + + ``kind`` is promoted out of the dimension columns. - One row per value; one column per non-null CV dimension, plus ``value``, ``id``, - ``execution_id`` and ``type``. ``value`` is left raw (NaN/inf preserved); callers that - serialise to JSON/CSV are responsible for any sanitisation. + Outlier columns (``is_outlier``/``verification_status``) are added only when ``detection_ran`` is set, + and context columns (``diagnostic_slug``/``provider_slug``) only when populated on the DTOs. + + ``value`` is left raw (NaN/inf preserved). """ records = [] - for r in rows: - rec: dict[str, Any] = dict(r.dimensions) - rec.update(id=r.id, execution_id=r.execution_id, value=r.value, type="scalar") + for v in values: + rec: dict[str, Any] = dict(v.dimensions) + rec.update( + id=v.id, + execution_id=v.execution_id, + execution_group_id=v.execution_group_id, + kind=v.kind, + value=v.value, + ) + if detection_ran: + rec.update(is_outlier=v.is_outlier, verification_status=v.verification_status) + if v.diagnostic_slug is not None: + rec.update(diagnostic_slug=v.diagnostic_slug, provider_slug=v.provider_slug) records.append(rec) return pd.DataFrame.from_records(records) -def series_values_to_frame(rows: Sequence[SeriesMetricValue]) -> pd.DataFrame: +def series_values_to_frame(values: "Sequence[SeriesValue]", *, explode: bool = True) -> pd.DataFrame: """ - Flatten series rows to a long-form (tidy) DataFrame. + Flatten series value DTOs to a DataFrame. + + With ``explode=True`` (default) the result is long-form: one row per (series, index point), + with columns ``value`` and ``index`` in addition to the shared metadata. + With ``explode=False`` each series is one row with list-valued ``values``/``index`` cells. + Shared columns are the CV dimensions present plus ``id``, ``execution_id``, ``execution_group_id``, + ``kind``, ``index_name`` and ``reference_id``. - One row per (series, index point). Columns: the non-null CV dimensions, plus ``value``, - ``index``, ``index_name``, ``reference_id``, ``id``, ``execution_id`` and ``type``. The index - is resolved from the shared axis; when a series has no index the positional integer is used. + Context columns (``diagnostic_slug``/``provider_slug``) are added only when populated on the DTOs. """ records = [] - for r in rows: - dims = r.dimensions - idx = r.index - name = r.index_name or "index" - for i, v in enumerate(r.values or []): - rec: dict[str, Any] = dict(dims) - rec.update( - id=r.id, - execution_id=r.execution_id, - value=v, - index=idx[i] if idx is not None and i < len(idx) else i, - index_name=name, - reference_id=r.reference_id, - type="series", - ) + for v in values: + base: dict[str, Any] = dict(v.dimensions) + base.update( + id=v.id, + execution_id=v.execution_id, + execution_group_id=v.execution_group_id, + kind=v.kind, + index_name=v.index_name or "index", + reference_id=v.reference_id, + ) + if v.diagnostic_slug is not None: + base.update(diagnostic_slug=v.diagnostic_slug, provider_slug=v.provider_slug) + if explode: + idx = v.index + for i, value in enumerate(v.values): + rec = dict(base) + rec.update(value=value, index=idx[i] if idx is not None and i < len(idx) else i) + records.append(rec) + else: + rec = dict(base) + rec.update(values=list(v.values), index=list(v.index) if v.index is not None else None) records.append(rec) return pd.DataFrame.from_records(records) diff --git a/packages/climate-ref/src/climate_ref/results/outliers.py b/packages/climate-ref/src/climate_ref/results/outliers.py index 82a555ae3..73914db2f 100644 --- a/packages/climate-ref/src/climate_ref/results/outliers.py +++ b/packages/climate-ref/src/climate_ref/results/outliers.py @@ -1,17 +1,25 @@ """ Outlier detection for scalar metric values. -Outlier detection is *read-model* logic, not presentation policy: whether a value is flagged -determines which rows a view returns and what the collection-level counts mean. It therefore -lives in ``climate_ref`` (the source of truth) rather than in any single consumer. +This is a port of the logic previously living in the ref-app backend, +and has been hoisted here to ensure all consumers use the same logic. -The algorithm is source-id-aware IQR: within each ``group_by`` group, IQR bounds are computed on -the per-``source_id`` mean value (so each model is weighted equally regardless of ensemble size), -then applied to individual values. ``"Reference"`` values are never flagged; non-finite values -(NaN/inf) are always flagged. +Outlier detection is read logic, not presentation policy. +All consumers should see the same set of outliers. +The outlier detection is performed at run-time (instead of a pre-computed flag) +as the outlier configuration may depend on the use-case. +We may also adopt other outlier detection algorithms in future. + + +The algorithm is source-id-aware Inter-Quartile Range (IQR). +Within each ``group_by`` group, +IQR bounds are computed on the per-``source_id`` mean value +(so each model is weighted equally regardless of ensemble size), +then applied to individual values. +The values that are outside ``factor`` * IQR are flagged as outliers. +``"Reference"`` values are never flagged. +Non-finite values (NaN/inf) are always flagged. -This is a framework-agnostic port of the logic previously living in the ref-app backend; it takes -and returns plain data and imports no web framework. """ import math @@ -42,7 +50,14 @@ class OutlierPolicy: """Multiplier on the IQR to set the outlier bounds (``Q1 - factor*IQR``, ``Q3 + factor*IQR``).""" min_n: int = 4 - """Minimum number of source_ids (or values) in a group required to run detection.""" + """ + Minimum sample size required to run detection. + + On the source-id-aware path this counts distinct non-reference ``source_id`` with a finite mean. + On the fallback path it counts finite values. + Non-finite values (NaN/inf) are excluded from the count + and are flagged unconditionally regardless of whether detection runs. + """ group_by: tuple[str, ...] = ("statistic", "metric") """CV dimensions to group by before computing bounds. Missing dimensions are ignored.""" @@ -67,36 +82,53 @@ class AnnotatedScalar: """``"verified"`` or ``"unverified"``, mirroring ``is_outlier``.""" +def _is_finite(x: object) -> bool: + return isinstance(x, int | float) and not math.isinf(x) and not math.isnan(x) + + def _flag_outliers_iqr(values: Sequence[float], factor: float, min_n: int) -> list[bool]: - """Flag outliers with a plain IQR test (fallback when no source_id is available).""" - n = len(values) - if n < min_n: - return [False] * n - quantiles = statistics.quantiles(values, n=4, method="inclusive") - q1, q3 = quantiles[0], quantiles[2] + """ + Flag outliers with a plain IQR test (fallback when no source_id is available). + + Detection runs only over finite values: a single NaN/inf would otherwise poison the quantiles + and disable the test for the whole group. + Non-finite values are never flagged here (the caller flags them separately). + When the finite spread is zero the bounds collapse, so nothing is flagged. + Gated on the count of finite values against ``min_n``. + """ + finite = [v for v in values if _is_finite(v)] + if len(finite) < min_n: + return [False] * len(values) + q1, _, q3 = statistics.quantiles(finite, n=4, method="inclusive") iqr = q3 - q1 + if iqr == 0: + return [False] * len(values) lower, upper = q1 - factor * iqr, q3 + factor * iqr - return [v < lower or v > upper for v in values] + return [_is_finite(v) and (v < lower or v > upper) for v in values] def _iqr_bounds_by_source_id(df: pd.DataFrame, factor: float, min_n: int) -> tuple[float, float] | None: - """IQR bounds computed on per-source_id means (equal model weighting).""" + """ + IQR bounds computed on per-source_id means (equal model weighting). + + A source whose mean is non-finite (e.g. all values NaN) is dropped before the ``min_n`` count + and the quantile computation, so it cannot poison the bounds for the others. Returns ``None`` + when fewer than ``min_n`` sources have a finite mean or when the spread is zero. + """ if "source_id" not in df.columns: return None non_reference = df[df["source_id"] != "Reference"] source_id_means = non_reference.groupby("source_id")["value"].mean() - if len(source_id_means) < min_n: + finite_means = source_id_means[source_id_means.map(_is_finite)] + if len(finite_means) < min_n: return None - quantiles = statistics.quantiles(source_id_means.tolist(), n=4, method="inclusive") - q1, q3 = quantiles[0], quantiles[2] + q1, _, q3 = statistics.quantiles(finite_means.tolist(), n=4, method="inclusive") iqr = q3 - q1 + if iqr == 0: + return None return q1 - factor * iqr, q3 + factor * iqr -def _is_finite(x: object) -> bool: - return isinstance(x, int | float) and not math.isinf(x) and not math.isnan(x) - - def detect_scalar_outliers( scalar_values: Sequence[ScalarMetricValue], policy: OutlierPolicy, @@ -137,27 +169,25 @@ def detect_scalar_outliers( verdict_by_id: dict[int, bool] = {} groups = df.groupby(list(group_by)) if group_by else [(None, df)] for _, group in groups: - if "source_id" in group.columns and len(group) >= policy.min_n: + values = group["value"] + finite = values.map(_is_finite) + if "source_id" in group.columns: + # Distinct finite source_ids gate detection (see _iqr_bounds_by_source_id). bounds = _iqr_bounds_by_source_id(group, factor=policy.factor, min_n=policy.min_n) if bounds is not None: lower, upper = bounds - source_flags = [ - (row["value"] < lower or row["value"] > upper) - if row["source_id"] != "Reference" - else False - for _, row in group.iterrows() - ] + out_of_bounds = (values < lower) | (values > upper) + flags = finite & out_of_bounds & (group["source_id"] != "Reference") else: - source_flags = [False] * len(group) - elif len(group) >= policy.min_n: - source_flags = _flag_outliers_iqr( - group["value"].to_list(), factor=policy.factor, min_n=policy.min_n - ) + flags = pd.Series(False, index=group.index) else: - source_flags = [False] * len(group) + # Finite value count gates detection (see _flag_outliers_iqr). + flag_list = _flag_outliers_iqr(values.to_list(), factor=policy.factor, min_n=policy.min_n) + flags = pd.Series(flag_list, index=group.index) - for (_, row), flagged in zip(group.iterrows(), source_flags): - verdict_by_id[row["id"]] = bool(flagged) or not _is_finite(row["value"]) + verdicts = flags | ~finite # non-finite values are always flagged + for row_id, verdict in zip(group["id"], verdicts): + verdict_by_id[row_id] = bool(verdict) annotated: list[AnnotatedScalar] = [] total = 0 diff --git a/packages/climate-ref/src/climate_ref/results/values.py b/packages/climate-ref/src/climate_ref/results/values.py index 7e5e2029b..48fe4c862 100644 --- a/packages/climate-ref/src/climate_ref/results/values.py +++ b/packages/climate-ref/src/climate_ref/results/values.py @@ -34,7 +34,7 @@ select_series_values, ) from climate_ref.results.executions import ExecutionsReader -from climate_ref.results.frames import collect_facets +from climate_ref.results.frames import collect_facets, scalar_values_to_frame, series_values_to_frame from climate_ref.results.outliers import OutlierPolicy, detect_scalar_outliers if TYPE_CHECKING: @@ -189,22 +189,7 @@ def facets_dict(self) -> dict[str, list[str]]: def to_pandas(self) -> pd.DataFrame: """Tidy DataFrame: one row per value, one column per CV dimension present, plus metadata.""" detection_ran = any(v.is_outlier is not None for v in self.items) - records = [] - for v in self.items: - rec: dict[str, Any] = dict(v.dimensions) - rec.update( - id=v.id, - execution_id=v.execution_id, - execution_group_id=v.execution_group_id, - kind=v.kind, - value=v.value, - ) - if detection_ran: - rec.update(is_outlier=v.is_outlier, verification_status=v.verification_status) - if v.diagnostic_slug is not None: - rec.update(diagnostic_slug=v.diagnostic_slug, provider_slug=v.provider_slug) - records.append(rec) - return pd.DataFrame.from_records(records) + return scalar_values_to_frame(self.items, detection_ran=detection_ran) @attrs.frozen(kw_only=True) @@ -244,28 +229,7 @@ def to_pandas(self, *, explode: bool = True) -> pd.DataFrame: matching the API's CSV shape. With ``explode=False`` each series is one row with list-valued ``values``/``index`` cells. """ - records = [] - for v in self.items: - base: dict[str, Any] = dict(v.dimensions) - base.update( - id=v.id, - execution_id=v.execution_id, - execution_group_id=v.execution_group_id, - kind=v.kind, - index_name=v.index_name or "index", - reference_id=v.reference_id, - ) - if explode: - idx = v.index - for i, value in enumerate(v.values): - rec = dict(base) - rec.update(value=value, index=idx[i] if idx is not None and i < len(idx) else i) - records.append(rec) - else: - rec = dict(base) - rec.update(values=list(v.values), index=list(v.index) if v.index is not None else None) - records.append(rec) - return pd.DataFrame.from_records(records) + return series_values_to_frame(self.items, explode=explode) class ValuesReader: diff --git a/packages/climate-ref/tests/unit/cli/test_datasets.py b/packages/climate-ref/tests/unit/cli/test_datasets.py index 12903f62d..53639d20a 100644 --- a/packages/climate-ref/tests/unit/cli/test_datasets.py +++ b/packages/climate-ref/tests/unit/cli/test_datasets.py @@ -82,6 +82,41 @@ def test_list_dataset_filter_invalid_facet(self, db_seeded, invoke_cli): expected_exit_code=1, ) + def test_list_only_latest_version(self, db_seeded, invoke_cli): + """``datasets list`` shows only the latest version of each dataset (v10 wins over v2).""" + existing = db_seeded.session.query(CMIP6Dataset).first() + base = { + "dataset_type": existing.dataset_type, + "activity_id": existing.activity_id, + "experiment_id": existing.experiment_id, + "institution_id": existing.institution_id, + "source_id": "LATEST-TEST", + "member_id": existing.member_id, + "table_id": existing.table_id, + "variable_id": existing.variable_id, + "grid_label": existing.grid_label, + "variant_label": existing.member_id, + } + for version in ("v2", "v10"): + slug = f"latest.test.{version}" + db_seeded.session.add( + CMIP6Dataset(slug=slug, instance_id=slug, version=version, finalised=True, **base) + ) + db_seeded.session.commit() + + result = invoke_cli( + ["datasets", "list", "--dataset-filter", "source_id=LATEST-TEST", "--column", "version"] + ) + assert "v10" in result.stdout + assert "v2" not in result.stdout + + def test_list_limit_warns_when_truncated(self, db_seeded, invoke_cli): + """A ``--limit`` smaller than the match count warns, matching the other list commands.""" + total = db_seeded.session.query(CMIP6Dataset).count() + assert total > 1, "need multiple datasets for the warning to trigger" + result = invoke_cli(["datasets", "list", "--limit", "1"]) + assert f"of {total} filtered results" in result.stderr + def test_list_include_files_limit_bounds_files_not_datasets(self, db_seeded, invoke_cli): """``--limit`` bounds *files* (not datasets) when ``--include-files`` is set. diff --git a/packages/climate-ref/tests/unit/cli/test_diagnostics.py b/packages/climate-ref/tests/unit/cli/test_diagnostics.py index d7b61eeec..ccaafeea1 100644 --- a/packages/climate-ref/tests/unit/cli/test_diagnostics.py +++ b/packages/climate-ref/tests/unit/cli/test_diagnostics.py @@ -90,10 +90,11 @@ def test_list_columns(self, db_with_diagnostics, invoke_cli): assert "promoted_version" not in result.stdout def test_list_columns_missing(self, db_with_diagnostics, invoke_cli): - invoke_cli( + result = invoke_cli( ["diagnostics", "list", "--column", "diagnostic", "--column", "missing"], expected_exit_code=1, ) + assert "Column not found: ['diagnostic', 'missing']" in result.stderr def test_list_json_empty(self, db_seeded, invoke_cli): result = invoke_cli(["diagnostics", "list", "--diagnostic", "nonexistent", "--format", "json"]) diff --git a/packages/climate-ref/tests/unit/cli/test_executions.py b/packages/climate-ref/tests/unit/cli/test_executions.py index 9787b4918..b646e9789 100644 --- a/packages/climate-ref/tests/unit/cli/test_executions.py +++ b/packages/climate-ref/tests/unit/cli/test_executions.py @@ -1438,10 +1438,87 @@ def test_scalar_all_outliers_hidden_reports_outlier_count(self, db_seeded, invok result = invoke_cli(["executions", "values", str(group_id), "--outliers"]) assert result.exit_code == 0 - assert "No scalar values found" in result.stdout - assert "1" in result.stdout - assert "outlier" in result.stdout - assert "--include-unverified" in result.stdout + assert "No scalar values found" in result.stderr + assert "1" in result.stderr + assert "outlier" in result.stderr + assert "--include-unverified" in result.stderr + + def test_values_default_limit_caps_output(self, db_seeded, invoke_cli): + """Test that limit defaults to 100 and caps output when more rows exist.""" + with db_seeded.session.begin(): + diagnostic = db_seeded.session.query(Diagnostic).first() + assert diagnostic is not None + group = ExecutionGroup(key="biggroup", diagnostic_id=diagnostic.id, selectors={}) + db_seeded.session.add(group) + db_seeded.session.flush() + execution = Execution( + execution_group_id=group.id, + output_fragment="frag-big", + dataset_hash="bighash", + successful=True, + ) + db_seeded.session.add(execution) + db_seeded.session.flush() + # Create 150 scalar values to exceed default limit of 100 + for i in range(150): + db_seeded.session.add( + ScalarMetricValue.build( + execution_id=execution.id, + value=float(i), + attributes=None, + dimensions={"statistic": "mean", "metric": "tas", "source_id": f"MODEL-{i}"}, + ) + ) + db_seeded.session.flush() + group_id = group.id + + result = invoke_cli(["executions", "values", str(group_id)]) + + assert result.exit_code == 0 + # Should display 100 values by default + assert "Displaying 100 of 150" in result.stderr + assert "--limit / --offset" in result.stderr + + def test_series_with_outlier_flags_warns(self, db_seeded, invoke_cli): + """Test that --outliers and --include-unverified warn when used with --kind series.""" + group_id, _ = self._setup(db_seeded) + + result = invoke_cli(["executions", "values", str(group_id), "--kind", "series", "--outliers"]) + + assert result.exit_code == 0 + assert "--outliers only apply to scalar values and were ignored." in result.stderr + + def test_series_with_include_unverified_warns(self, db_seeded, invoke_cli): + """Test that --include-unverified warns when used with --kind series.""" + group_id, _ = self._setup(db_seeded) + + result = invoke_cli( + ["executions", "values", str(group_id), "--kind", "series", "--include-unverified"] + ) + + assert result.exit_code == 0 + assert "--include-unverified only apply to scalar values and were ignored." in result.stderr + + def test_series_with_both_flags_warns(self, db_seeded, invoke_cli): + """Test that both flags warn together when used with --kind series.""" + group_id, _ = self._setup(db_seeded) + + result = invoke_cli( + [ + "executions", + "values", + str(group_id), + "--kind", + "series", + "--outliers", + "--include-unverified", + ] + ) + + assert result.exit_code == 0 + assert "--outliers/--include-unverified only apply to scalar values and were ignored." in ( + result.stderr + ) def test_dimension_filter(self, db_seeded, invoke_cli): group_id, _ = self._setup(db_seeded) diff --git a/packages/climate-ref/tests/unit/datasets/test_dataset_query.py b/packages/climate-ref/tests/unit/datasets/test_dataset_query.py index aa0b79ae9..7770e4e1a 100644 --- a/packages/climate-ref/tests/unit/datasets/test_dataset_query.py +++ b/packages/climate-ref/tests/unit/datasets/test_dataset_query.py @@ -4,12 +4,12 @@ 1. ``select_datasets(..., latest_group_by=...)`` returns the same survivor set as the old pandas ``filter_latest_versions`` path (numeric ties, non-conforming versions, combined with finalised/facet/relationship filters -- including ``diagnostic_slug`` + ``latest_only`` together). -2. The base-entity (``source_type=None``) case stays a no-op. -3. The non-nullability invariant that the SQL dedup depends on: every adapter's +2. The non-nullability invariant that the SQL dedup depends on: every adapter's ``dataset_id_metadata`` columns are non-nullable and absent from ``columns_requiring_finalisation``. """ import pytest +from sqlalchemy import update from sqlalchemy.orm import selectinload from climate_ref.datasets import get_dataset_adapter @@ -193,14 +193,10 @@ def test_latest_group_by_none_is_inert(self, db_with_versions): assert db_with_versions.dq_ids["v2"] in ids assert db_with_versions.dq_ids["v10"] in ids - def test_base_entity_no_op(self, db_with_versions): - """``source_type=None`` stays a no-op even if a ``latest_group_by`` were (incorrectly) passed, - because there are no ``dataset_id_metadata`` columns on the base ``Dataset`` entity.""" - stmt_no_dedup = select_datasets(DatasetFilter()) - stmt_with_group_by = select_datasets(DatasetFilter(), latest_group_by=()) - rows_a = db_with_versions.session.execute(stmt_no_dedup).scalars().unique().all() - rows_b = db_with_versions.session.execute(stmt_with_group_by).scalars().unique().all() - assert len(rows_a) == len(rows_b) + def test_source_type_is_required(self): + """``DatasetFilter`` has no default ``source_type``: a typed listing must choose a type.""" + with pytest.raises(TypeError): + DatasetFilter() # type: ignore[call-arg] class TestDatasetIdMetadataNonNullabilityInvariant: @@ -239,6 +235,49 @@ def test_dataset_id_metadata_disjoint_from_finalisation_columns(self, adapter_cl ) +class TestVersionKeyOrmOnlyInvariant: + """Pin the documented ORM-only invariant for ``version_key``. + + The ``_sync_version_key`` mapper event only fires on ORM inserts/updates. A Core-level + ``UPDATE`` to ``version`` bypasses it, leaving ``version_key`` stale. This is a known + limitation documented on the ``version``/``version_key`` column docstrings; the test + exists to pin that behaviour so a future change is a conscious one. + """ + + def test_orm_insert_syncs_version_key(self, db_seeded): + """Baseline: an ORM insert does keep ``version_key`` in sync.""" + with db_seeded.session.begin(): + ds = _make_cmip6(slug="vk.orm-insert.v10", version="v10") + db_seeded.session.add(ds) + db_seeded.session.flush() + assert ds.version_key == 10 + + def test_core_update_leaves_version_key_stale(self, db_seeded): + """A Core-level ``UPDATE`` to ``version`` does NOT re-sync ``version_key``. + + This pins the known limitation: the mapper event never fires for a Core update, so + ``version_key`` keeps its old value (2) even though ``version`` is now ``v10``. + """ + with db_seeded.session.begin(): + ds = _make_cmip6(slug="vk.core-update.v2", version="v2") + db_seeded.session.add(ds) + db_seeded.session.flush() + assert ds.version_key == 2 + ds_id = ds.id + + # Core-level UPDATE on the subclass table, bypassing the ORM mapper event. + with db_seeded.session.begin(): + db_seeded.session.execute( + update(CMIP6Dataset).where(CMIP6Dataset.id == ds_id).values(version="v10") + ) + + db_seeded.session.expire_all() + refreshed = db_seeded.session.get(CMIP6Dataset, ds_id) + assert refreshed.version == "v10" + # version_key is stale: still 2, not the 10 an ORM write would have produced. + assert refreshed.version_key == 2 + + def test_get_dataset_adapter_covers_all_four_subclasses(): """Sanity check that all four Dataset subclasses are reachable via the adapter registry (keeps the parametrised tests above from silently under-covering a fifth subclass added later).""" diff --git a/packages/climate-ref/tests/unit/models/test_series_index.py b/packages/climate-ref/tests/unit/models/test_series_index.py new file mode 100644 index 000000000..ec2dc9d09 --- /dev/null +++ b/packages/climate-ref/tests/unit/models/test_series_index.py @@ -0,0 +1,80 @@ +"""Unit tests for ``SeriesIndex.bulk_get_or_create``. + +Covers: +1. Happy-path roundtrip: a mix of pre-existing and missing axes resolves to the correct ids. +2. Idempotency: a second call over already-resolved axes inserts nothing. +3. Mismatched-key rejection: a key that is not ``compute_hash(name, values)`` raises ``ValueError``. +""" + +import pytest +from sqlalchemy import func, select + +from climate_ref.models import SeriesIndex + + +def _axis_entry(name, values): + """Build the ``{hash: (name, values)}`` entry ``bulk_get_or_create`` expects.""" + return SeriesIndex.compute_hash(name, values), (name, values) + + +class TestBulkGetOrCreate: + def test_empty_input_returns_empty(self, db_seeded): + assert SeriesIndex.bulk_get_or_create(db_seeded.session, {}) == {} + + def test_roundtrip_mix_of_existing_and_missing(self, db_seeded): + session = db_seeded.session + + # Pre-create one axis via the single-row path so the bulk call must reuse it. + existing = SeriesIndex.get_or_create(session, "time", [0, 1, 2]) + session.flush() + + h_existing, e_existing = _axis_entry("time", [0, 1, 2]) + h_missing_a, e_missing_a = _axis_entry("depth", [10, 20]) + h_missing_b, e_missing_b = _axis_entry(None, [1.5, 2.5, 3.5]) + axes_by_hash = {h_existing: e_existing, h_missing_a: e_missing_a, h_missing_b: e_missing_b} + + result = SeriesIndex.bulk_get_or_create(session, axes_by_hash) + + assert set(result) == {h_existing, h_missing_a, h_missing_b} + # The pre-existing axis keeps its original id (reused, not duplicated). + assert result[h_existing] == existing.id + + # Every returned id points at a row whose stored hash matches its key. + for digest, axis_id in result.items(): + axis = session.get(SeriesIndex, axis_id) + assert axis.hash == digest + + # The two missing axes were inserted with the right content. + depth = session.get(SeriesIndex, result[h_missing_a]) + assert depth.name == "depth" + assert depth.values == [10, 20] + assert depth.length == 2 + + def test_idempotent_second_call_inserts_nothing(self, db_seeded): + session = db_seeded.session + + axes_by_hash = dict([_axis_entry("time", [0, 1, 2]), _axis_entry("depth", [10, 20])]) + + first = SeriesIndex.bulk_get_or_create(session, axes_by_hash) + session.flush() + count_after_first = session.execute(select(func.count()).select_from(SeriesIndex)).scalar_one() + + second = SeriesIndex.bulk_get_or_create(session, axes_by_hash) + session.flush() + count_after_second = session.execute(select(func.count()).select_from(SeriesIndex)).scalar_one() + + assert second == first + assert count_after_second == count_after_first + + def test_mismatched_key_is_rejected(self, db_seeded): + session = db_seeded.session + + # A key that does not equal compute_hash(name, values) for its axis. + bad = {"not-the-real-hash": ("time", [0, 1, 2])} + + with pytest.raises(ValueError, match="does not match compute_hash"): + SeriesIndex.bulk_get_or_create(session, bad) + + # Nothing was inserted for the rejected batch. + count = session.execute(select(func.count()).select_from(SeriesIndex)).scalar_one() + assert count == 0 diff --git a/packages/climate-ref/tests/unit/results/test_artifacts.py b/packages/climate-ref/tests/unit/results/test_artifacts.py index 8d2ce30dd..e96f18535 100644 --- a/packages/climate-ref/tests/unit/results/test_artifacts.py +++ b/packages/climate-ref/tests/unit/results/test_artifacts.py @@ -71,3 +71,37 @@ def test_absolute_filename_raises(self, tmp_path): reader = ArtifactsReader(tmp_path) with pytest.raises(ValueError): reader.output_file("frag", "/etc/passwd") + + +class TestSymlinkEscapeIsLexicalOnly: + """Pin the documented lexical-only containment contract. + + ``_within`` normalises paths without touching the filesystem or resolving symlinks + (see the ``ArtifactsReader`` docstring). A symlink that lives *inside* the results root + but points *outside* it therefore passes the guard: the returned path is lexically + contained even though following it escapes on the filesystem. This is the documented + contract, not a bug -- the test exists so a future change to it is a conscious one. + """ + + def test_symlink_inside_root_pointing_outside_passes_guard(self, tmp_path): + results_root = tmp_path / "results" + results_root.mkdir() + outside = tmp_path / "outside" + outside.mkdir() + (outside / "secret.txt").write_text("secret") + + # A symlink that lives inside the results root but targets a sibling outside it. + link = results_root / "link" + link.symlink_to(outside) + + reader = ArtifactsReader(results_root) + + # Lexical guard succeeds: the resolved path is textually under the results root. + resolved = reader.output_file("link", "secret.txt") + assert resolved == results_root / "link" / "secret.txt" + + # But following the symlink on the filesystem escapes the root: the resolved real + # path is the outside file, and it is not contained under the results root. + real = resolved.resolve() + assert real == (outside / "secret.txt").resolve() + assert results_root.resolve() not in real.parents diff --git a/packages/climate-ref/tests/unit/results/test_datasets.py b/packages/climate-ref/tests/unit/results/test_datasets.py index 4ebf7b8d0..8f2b556b7 100644 --- a/packages/climate-ref/tests/unit/results/test_datasets.py +++ b/packages/climate-ref/tests/unit/results/test_datasets.py @@ -136,7 +136,7 @@ def db_with_datasets(db_seeded): class TestFacetFiltering: def test_or_within_facet(self, db_with_datasets): reader = Reader(db_with_datasets) - coll = reader.datasets.datasets( + coll = reader.datasets.list( DatasetFilter( source_type=SourceDatasetType.CMIP6, facets={"variable_id": ("tas", "pr")}, @@ -149,7 +149,7 @@ def test_or_within_facet(self, db_with_datasets): def test_and_across_facets(self, db_with_datasets): reader = Reader(db_with_datasets) - coll = reader.datasets.datasets( + coll = reader.datasets.list( DatasetFilter( source_type=SourceDatasetType.CMIP6, facets={"variable_id": ("pr",), "source_id": ("OTHER-MODEL",)}, @@ -157,12 +157,12 @@ def test_and_across_facets(self, db_with_datasets): ) ) assert len(coll) == 1 - assert coll.datasets[0].facets["variable_id"] == "pr" - assert coll.datasets[0].facets["source_id"] == "OTHER-MODEL" + assert coll.items[0].facets["variable_id"] == "pr" + assert coll.items[0].facets["source_id"] == "OTHER-MODEL" def test_and_across_facets_no_match(self, db_with_datasets): reader = Reader(db_with_datasets) - coll = reader.datasets.datasets( + coll = reader.datasets.list( DatasetFilter( source_type=SourceDatasetType.CMIP6, facets={"variable_id": ("pr",), "source_id": ("TEST-MODEL",)}, @@ -174,27 +174,20 @@ def test_and_across_facets_no_match(self, db_with_datasets): def test_unknown_facet_raises(self, db_with_datasets): reader = Reader(db_with_datasets) with pytest.raises(ValueError, match="Unknown facet"): - reader.datasets.datasets( + reader.datasets.list( DatasetFilter(source_type=SourceDatasetType.CMIP6, facets={"nonexistent_facet": ("x",)}) ) - def test_unknown_facet_on_base_entity_raises(self, db_with_datasets): - reader = Reader(db_with_datasets) - with pytest.raises(ValueError, match="Unknown facet"): - reader.datasets.datasets(DatasetFilter(facets={"variable_id": ("tas",)})) - - def test_facet_on_base_entity_matches_base_column(self, db_with_datasets): - reader = Reader(db_with_datasets) - coll = reader.datasets.datasets(DatasetFilter(facets={"dataset_type": (SourceDatasetType.obs4MIPs,)})) - assert len(coll) > 0 - assert all(d.dataset_type == SourceDatasetType.obs4MIPs for d in coll) - assert db_with_datasets.dataset_ids["obs4mips"] in {d.id for d in coll} + def test_source_type_is_required(self): + """``DatasetFilter`` has no default ``source_type``: a typed listing must choose a type.""" + with pytest.raises(TypeError): + DatasetFilter() # type: ignore[call-arg] class TestFinalisedFilter: def test_finalised_true(self, db_with_datasets): reader = Reader(db_with_datasets) - coll = reader.datasets.datasets( + coll = reader.datasets.list( DatasetFilter(source_type=SourceDatasetType.CMIP6, finalised=True, latest_only=False) ) assert all(d.finalised for d in coll) @@ -203,17 +196,17 @@ def test_finalised_true(self, db_with_datasets): def test_finalised_false(self, db_with_datasets): reader = Reader(db_with_datasets) - coll = reader.datasets.datasets( + coll = reader.datasets.list( DatasetFilter(source_type=SourceDatasetType.CMIP6, finalised=False, latest_only=False) ) assert len(coll) == 1 - assert coll.datasets[0].slug == "CMIP.TEST.OTHER-MODEL.historical.Amon.pr.gn.v1" + assert coll.items[0].slug == "CMIP.TEST.OTHER-MODEL.historical.Amon.pr.gn.v1" class TestLatestOnly: def test_keeps_newest_numeric_version(self, db_with_datasets): reader = Reader(db_with_datasets) - coll = reader.datasets.datasets( + coll = reader.datasets.list( DatasetFilter( source_type=SourceDatasetType.CMIP6, facets={"source_id": ("TEST-MODEL",), "variable_id": ("tas",)}, @@ -221,11 +214,11 @@ def test_keeps_newest_numeric_version(self, db_with_datasets): ) # v10 must win over v2 numerically, not lexically ("v10" < "v2" as strings). assert len(coll) == 1 - assert coll.datasets[0].id == db_with_datasets.dataset_ids["v10"] + assert coll.items[0].id == db_with_datasets.dataset_ids["v10"] def test_latest_only_false_keeps_both_versions(self, db_with_datasets): reader = Reader(db_with_datasets) - coll = reader.datasets.datasets( + coll = reader.datasets.list( DatasetFilter( source_type=SourceDatasetType.CMIP6, facets={"source_id": ("TEST-MODEL",), "variable_id": ("tas",)}, @@ -234,18 +227,21 @@ def test_latest_only_false_keeps_both_versions(self, db_with_datasets): ) assert len(coll) == 2 - def test_latest_only_noop_when_source_type_none(self, db_with_datasets): + def test_latest_only_drops_versions_vs_all(self, db_with_datasets): + """The deduplicated listing is strictly smaller than the every-version listing.""" reader = Reader(db_with_datasets) - coll = reader.datasets.datasets(DatasetFilter(latest_only=True)) - # All datasets (across types) present -- no dataset_id_metadata to group by. - all_coll = reader.datasets.datasets(DatasetFilter(latest_only=False)) - assert len(coll) == len(all_coll) + deduped = reader.datasets.list(DatasetFilter(source_type=SourceDatasetType.CMIP6, latest_only=True)) + every_version = reader.datasets.list( + DatasetFilter(source_type=SourceDatasetType.CMIP6, latest_only=False) + ) + assert len(deduped) < len(every_version) + assert deduped.total_count < every_version.total_count class TestExecutionIdJoin: def test_execution_id_scopes_to_linked_datasets(self, db_with_datasets): reader = Reader(db_with_datasets) - coll = reader.datasets.datasets( + coll = reader.datasets.list( DatasetFilter( source_type=SourceDatasetType.CMIP6, execution_id=db_with_datasets.execution_id, @@ -253,11 +249,11 @@ def test_execution_id_scopes_to_linked_datasets(self, db_with_datasets): ) ) assert len(coll) == 1 - assert coll.datasets[0].id == db_with_datasets.dataset_ids["v10"] + assert coll.items[0].id == db_with_datasets.dataset_ids["v10"] def test_execution_id_no_match(self, db_with_datasets): reader = Reader(db_with_datasets) - coll = reader.datasets.datasets( + coll = reader.datasets.list( DatasetFilter(source_type=SourceDatasetType.CMIP6, execution_id=999999, latest_only=False) ) assert len(coll) == 0 @@ -266,7 +262,7 @@ def test_execution_id_no_match(self, db_with_datasets): class TestDiagnosticSlugJoin: def test_diagnostic_slug_scopes_to_linked_datasets(self, db_with_datasets): reader = Reader(db_with_datasets) - coll = reader.datasets.datasets( + coll = reader.datasets.list( DatasetFilter( source_type=SourceDatasetType.CMIP6, diagnostic_slug=db_with_datasets.diagnostic_slug, @@ -274,11 +270,11 @@ def test_diagnostic_slug_scopes_to_linked_datasets(self, db_with_datasets): ) ) assert len(coll) == 1 - assert coll.datasets[0].id == db_with_datasets.dataset_ids["v10"] + assert coll.items[0].id == db_with_datasets.dataset_ids["v10"] def test_diagnostic_slug_no_match(self, db_with_datasets): reader = Reader(db_with_datasets) - coll = reader.datasets.datasets( + coll = reader.datasets.list( DatasetFilter( source_type=SourceDatasetType.CMIP6, diagnostic_slug="nonexistent-diagnostic", @@ -291,7 +287,7 @@ def test_execution_id_and_diagnostic_slug_together(self, db_with_datasets): # Both axes reach through ``execution_datasets``; setting both must not emit a duplicate, # unaliased self-join. This executes the query, so invalid SQL would raise here. reader = Reader(db_with_datasets) - coll = reader.datasets.datasets( + coll = reader.datasets.list( DatasetFilter( source_type=SourceDatasetType.CMIP6, execution_id=db_with_datasets.execution_id, @@ -300,20 +296,20 @@ def test_execution_id_and_diagnostic_slug_together(self, db_with_datasets): ) ) assert len(coll) == 1 - assert coll.datasets[0].id == db_with_datasets.dataset_ids["v10"] + assert coll.items[0].id == db_with_datasets.dataset_ids["v10"] class TestLimit: def test_limit_applied_after_latest_only(self, db_with_datasets): reader = Reader(db_with_datasets) - coll = reader.datasets.datasets(DatasetFilter(source_type=SourceDatasetType.CMIP6), limit=1) + coll = reader.datasets.list(DatasetFilter(source_type=SourceDatasetType.CMIP6), limit=1) assert len(coll) == 1 class TestIncludeFiles: def test_include_files_true_populates_files(self, db_with_datasets): reader = Reader(db_with_datasets) - coll = reader.datasets.datasets( + coll = reader.datasets.list( DatasetFilter( source_type=SourceDatasetType.CMIP6, facets={"source_id": ("TEST-MODEL",), "variable_id": ("tas",)}, @@ -321,7 +317,7 @@ def test_include_files_true_populates_files(self, db_with_datasets): include_files=True, ) assert len(coll) == 1 - ds = coll.datasets[0] + ds = coll.items[0] assert len(ds.files) == 1 assert ds.files[0].path == "tas_v10.nc" assert ds.files[0].start_time == "2000-01-01" @@ -330,7 +326,7 @@ def test_include_files_true_populates_files(self, db_with_datasets): def test_include_files_false_leaves_files_empty(self, db_with_datasets): reader = Reader(db_with_datasets) - coll = reader.datasets.datasets( + coll = reader.datasets.list( DatasetFilter( source_type=SourceDatasetType.CMIP6, facets={"source_id": ("TEST-MODEL",), "variable_id": ("tas",)}, @@ -338,22 +334,13 @@ def test_include_files_false_leaves_files_empty(self, db_with_datasets): include_files=False, ) assert len(coll) == 1 - assert coll.datasets[0].files == () - - -class TestSourceTypeNonePolymorphic: - def test_base_query_returns_all_types(self, db_with_datasets): - reader = Reader(db_with_datasets) - coll = reader.datasets.datasets(DatasetFilter(latest_only=False)) - types = {d.dataset_type for d in coll} - assert SourceDatasetType.CMIP6 in types - assert SourceDatasetType.obs4MIPs in types + assert coll.items[0].files == () class TestDetachment: def test_dto_fields_survive_session_close(self, db_with_datasets): reader = Reader(db_with_datasets) - coll = reader.datasets.datasets( + coll = reader.datasets.list( DatasetFilter( source_type=SourceDatasetType.CMIP6, facets={"source_id": ("TEST-MODEL",), "variable_id": ("tas",)}, @@ -363,7 +350,7 @@ def test_dto_fields_survive_session_close(self, db_with_datasets): db_with_datasets.session.expunge_all() assert len(coll) == 1 - ds = coll.datasets[0] + ds = coll.items[0] assert ds.slug == "CMIP.TEST.TEST-MODEL.historical.Amon.tas.gn.v10" assert ds.facets["variable_id"] == "tas" assert len(ds.files) == 1 @@ -372,7 +359,7 @@ def test_dto_fields_survive_session_close(self, db_with_datasets): class TestToPandas: def test_to_pandas_columns(self, db_with_datasets): reader = Reader(db_with_datasets) - coll = reader.datasets.datasets(DatasetFilter(source_type=SourceDatasetType.CMIP6, latest_only=False)) + coll = reader.datasets.list(DatasetFilter(source_type=SourceDatasetType.CMIP6, latest_only=False)) df = coll.to_pandas() for col in ("id", "slug", "dataset_type", "finalised", "created_at", "updated_at"): assert col in df.columns @@ -381,7 +368,7 @@ def test_to_pandas_columns(self, db_with_datasets): def test_to_pandas_columns_when_empty(self, db_with_datasets): """An empty collection still emits the base columns, so callers can select on them.""" reader = Reader(db_with_datasets) - coll = reader.datasets.datasets( + coll = reader.datasets.list( DatasetFilter(source_type=SourceDatasetType.CMIP6, facets={"source_id": ("NO-SUCH-MODEL",)}) ) assert len(coll) == 0 @@ -389,3 +376,102 @@ def test_to_pandas_columns_when_empty(self, db_with_datasets): assert len(df) == 0 for col in ("id", "slug", "dataset_type", "finalised", "created_at", "updated_at"): assert col in df.columns + + +class TestCollectionContract: + def test_shape_fields(self, db_with_datasets): + reader = Reader(db_with_datasets) + coll = reader.datasets.list(DatasetFilter(source_type=SourceDatasetType.CMIP6, latest_only=False)) + assert isinstance(coll.items, tuple) + assert coll.offset == 0 + assert coll.limit is None + assert coll.total_count == len(coll.items) + assert list(coll) == list(coll.items) + + def test_total_count_reflects_dedup(self, db_with_datasets): + """``total_count`` is the deduplicated count, not the every-version count.""" + reader = Reader(db_with_datasets) + deduped = reader.datasets.list( + DatasetFilter(source_type=SourceDatasetType.CMIP6, facets={"source_id": ("TEST-MODEL",)}) + ) + # Only v10 survives its group, so the count is 1 even though v2 exists in the DB. + assert deduped.total_count == 1 + + +class TestPagination: + def test_total_count_exceeds_page(self, db_with_datasets): + reader = Reader(db_with_datasets) + coll = reader.datasets.list( + DatasetFilter(source_type=SourceDatasetType.CMIP6, latest_only=False), limit=1 + ) + assert len(coll) == 1 + assert coll.limit == 1 + assert coll.total_count > 1 + + def test_offset_limit_paginate_all_rows_without_overlap(self, db_with_datasets): + """Walking the pages with offset/limit yields every row exactly once, deterministically.""" + reader = Reader(db_with_datasets) + full = reader.datasets.list(DatasetFilter(source_type=SourceDatasetType.CMIP6, latest_only=False)) + total = full.total_count + + seen: list[int] = [] + page_size = 2 + for offset in range(0, total, page_size): + page = reader.datasets.list( + DatasetFilter(source_type=SourceDatasetType.CMIP6, latest_only=False), + offset=offset, + limit=page_size, + ) + assert page.offset == offset + seen.extend(d.id for d in page) + + assert len(seen) == total + assert len(set(seen)) == total # no duplicates or gaps across pages + + def test_paging_one_by_one_matches_full_order(self, db_with_datasets): + """Paging one row at a time yields the same order as one full fetch. + + The deterministic ``(slug, id)`` ordering means the concatenation of single-row pages is + identical to the ordered single fetch -- so a page boundary never reorders, drops, or + repeats a row. + """ + reader = Reader(db_with_datasets) + full = reader.datasets.list(DatasetFilter(source_type=SourceDatasetType.CMIP6, latest_only=False)) + full_ids = [d.id for d in full] + + paged_ids: list[int] = [] + for offset in range(full.total_count): + page = reader.datasets.list( + DatasetFilter(source_type=SourceDatasetType.CMIP6, latest_only=False), + offset=offset, + limit=1, + ) + paged_ids.append(page.items[0].id) + + assert paged_ids == full_ids + + +class TestGet: + def test_get_hit(self, db_with_datasets): + reader = Reader(db_with_datasets) + view = reader.datasets.get("CMIP.TEST.OTHER-MODEL.historical.Amon.pr.gn.v1") + assert view is not None + assert view.id == db_with_datasets.dataset_ids["unfinalised"] + + def test_get_miss(self, db_with_datasets): + reader = Reader(db_with_datasets) + assert reader.datasets.get("no-such-slug") is None + + def test_get_resolves_each_version_by_its_own_slug(self, db_with_datasets): + """Each version has a distinct (globally unique) slug, so ``get`` resolves each precisely. + + ``get``'s ``version_key``-then-``id`` ordering is a defensive tiebreak; the unique-slug + schema means a slug never maps to more than one row, so both versions are individually + addressable. + """ + reader = Reader(db_with_datasets) + v2 = reader.datasets.get("CMIP.TEST.TEST-MODEL.historical.Amon.tas.gn.v2") + v10 = reader.datasets.get("CMIP.TEST.TEST-MODEL.historical.Amon.tas.gn.v10") + assert v2 is not None and v10 is not None + assert v2.id == db_with_datasets.dataset_ids["v2"] + assert v10.id == db_with_datasets.dataset_ids["v10"] diff --git a/packages/climate-ref/tests/unit/results/test_diagnostics.py b/packages/climate-ref/tests/unit/results/test_diagnostics.py index 4b44238b5..17a329e64 100644 --- a/packages/climate-ref/tests/unit/results/test_diagnostics.py +++ b/packages/climate-ref/tests/unit/results/test_diagnostics.py @@ -136,6 +136,21 @@ def test_filter_by_provider(self, db_with_diagnostics): providers = {d.provider_slug for d in coll} assert providers == {"esmvaltool"} + def test_filters_keyword(self, db_with_diagnostics): + # The list-style filter parameter is named `filters`, matching the rest of the contract. + reader = Reader(db_with_diagnostics) + coll = reader.diagnostics.list(filters=DiagnosticFilter(diagnostic_contains=["enso_tel"])) + assert {d.slug for d in coll} == {"enso_tel"} + + def test_pagination_deterministic(self, db_with_diagnostics): + # Full listing is ordered by (provider, diagnostic, id); adjacent pages must partition it. + reader = Reader(db_with_diagnostics) + full = [(d.provider_slug, d.slug) for d in reader.diagnostics.list()] + page_1 = [(d.provider_slug, d.slug) for d in reader.diagnostics.list(offset=0, limit=2)] + page_2 = [(d.provider_slug, d.slug) for d in reader.diagnostics.list(offset=2, limit=2)] + assert set(page_1).isdisjoint(page_2) + assert page_1 + page_2 == full[:4] + def test_pagination_offset_limit(self, db_with_diagnostics): reader = Reader(db_with_diagnostics) full = reader.diagnostics.list() diff --git a/packages/climate-ref/tests/unit/results/test_executions.py b/packages/climate-ref/tests/unit/results/test_executions.py index 6fdc1cd1c..c1efd44d5 100644 --- a/packages/climate-ref/tests/unit/results/test_executions.py +++ b/packages/climate-ref/tests/unit/results/test_executions.py @@ -238,6 +238,90 @@ def test_total_count_vs_page_with_offset_limit(self, db_with_groups): assert page.offset == 2 assert page.limit == 2 + def test_pagination_deterministic_across_tied_created_at(self, db_with_groups): + # Force every group onto one shared created_at so ordering must fall back to the group id. + session = db_with_groups.session + tied_at = datetime.datetime(2024, 1, 1, tzinfo=datetime.UTC) + with session.begin_nested() if session.in_transaction() else session.begin(): + for eg in session.query(ExecutionGroup).all(): + eg.created_at = tied_at + + reader = Reader(db_with_groups) + full_ids = [g.id for g in reader.executions.groups()] + assert full_ids == sorted(full_ids) # ordered by group id, the deterministic tiebreak + + page_1 = [g.id for g in reader.executions.groups(offset=0, limit=3)] + page_2 = [g.id for g in reader.executions.groups(offset=3, limit=3)] + page_3 = [g.id for g in reader.executions.groups(offset=6, limit=3)] + assert set(page_1).isdisjoint(page_2) + assert set(page_2).isdisjoint(page_3) + assert page_1 + page_2 + page_3 == full_ids # no overlap, no gap + + +class TestGroupsLatestSuccessful: + def test_latest_successful_surfaces_earlier_success(self, db_with_groups): + # key3's only execution failed. Add an earlier successful run: successful=True (post-rank) + # still drops it, but latest_successful=True must surface the earlier success. + session = db_with_groups.session + group_id = db_with_groups.group_ids["key3"] + with session.begin_nested() if session.in_transaction() else session.begin(): + session.add( + Execution( + execution_group_id=group_id, + successful=True, + output_fragment="key3-ok", + dataset_hash="key3-ok-hash", + created_at=datetime.datetime(2020, 1, 1, tzinfo=datetime.UTC), + ) + ) + + reader = Reader(db_with_groups) + + post = reader.executions.groups(ExecutionGroupFilter(successful=True)) + assert group_id not in {g.id for g in post} # latest run still the failure + + pre = reader.executions.groups(ExecutionGroupFilter(latest_successful=True)) + view = next(g for g in pre if g.id == group_id) + assert view.latest is not None + assert view.latest.successful is True + assert view.latest.output_fragment == "key3-ok" + + +class TestGroupAndExecutionById: + def test_group_hit_resolves_latest(self, db_with_groups): + reader = Reader(db_with_groups) + group_id = db_with_groups.group_ids["key1"] + view = reader.executions.group(group_id) + assert view is not None + assert view.id == group_id + assert view.key == "key1" + assert view.latest is not None + assert view.latest.successful is True + + def test_group_hit_with_no_executions(self, db_with_groups): + reader = Reader(db_with_groups) + view = reader.executions.group(db_with_groups.group_ids["key6"]) + assert view is not None + assert view.latest is None + + def test_group_miss_returns_none(self, db_with_groups): + reader = Reader(db_with_groups) + assert reader.executions.group(999_999) is None + + def test_execution_hit(self, db_with_groups): + session = db_with_groups.session + group_id = db_with_groups.group_ids["key1"] + execution_id = session.query(Execution).filter_by(execution_group_id=group_id).one().id + reader = Reader(db_with_groups) + view = reader.executions.execution(execution_id) + assert view is not None + assert view.id == execution_id + assert view.execution_group_id == group_id + + def test_execution_miss_returns_none(self, db_with_groups): + reader = Reader(db_with_groups) + assert reader.executions.execution(999_999) is None + class TestGroupsToPandas: def test_to_pandas_columns(self, db_with_groups): diff --git a/packages/climate-ref/tests/unit/results/test_results.py b/packages/climate-ref/tests/unit/results/test_results.py index 02cce5fd4..4a78e3747 100644 --- a/packages/climate-ref/tests/unit/results/test_results.py +++ b/packages/climate-ref/tests/unit/results/test_results.py @@ -3,6 +3,7 @@ import math import pytest +from pandas.testing import assert_frame_equal as pd_assert_frame_equal from climate_ref.models import Execution, ExecutionGroup from climate_ref.models.diagnostic import Diagnostic @@ -13,6 +14,8 @@ Reader, ) from climate_ref.results._query import latest_execution_for_group +from climate_ref.results.frames import scalar_values_to_frame, series_values_to_frame +from climate_ref.results.outliers import detect_scalar_outliers @pytest.fixture @@ -180,6 +183,20 @@ def test_pagination(self, dal_db): assert len(coll) == 3 assert coll.total_count == 8 + def test_pagination_deterministic_across_tied_timestamps(self, dal_db): + # Every scalar row here shares one execution (one created_at), so ordering must fall back to + # the value id. Two adjacent pages must partition the result with no overlap and no gap. + f = MetricValueFilter(execution_group_ids=[dal_db.execution_group_id]) + ref = Reader(dal_db) + page_1 = ref.values.scalar_values(f, offset=0, limit=4) + page_2 = ref.values.scalar_values(f, offset=4, limit=4) + ids_1 = [v.id for v in page_1] + ids_2 = [v.id for v in page_2] + assert ids_1 == sorted(ids_1) + assert ids_2 == sorted(ids_2) + assert set(ids_1).isdisjoint(ids_2) # no overlap + assert ids_1 + ids_2 == sorted(v.id for v in ref.values.scalar_values(f)) # no gap + def test_dimension_filter(self, dal_db): ref = Reader(dal_db) coll = ref.values.scalar_values( @@ -251,6 +268,35 @@ def test_to_pandas_long_form(self, dal_db): assert col in df.columns +class TestFrameParity: + """Both frame doors -- the frames.py builders and the collections' ``to_pandas()`` -- must emit + identical columns and rows for the same DTOs, so no consumer sees a different scalar/series + frame shape depending on which door it came through.""" + + def test_scalar_columns_identical(self, dal_db): + coll = Reader(dal_db).values.scalar_values( + MetricValueFilter(execution_group_ids=[dal_db.execution_group_id]) + ) + via_collection = coll.to_pandas() + via_helper = scalar_values_to_frame(coll.items) + assert list(via_collection.columns) == list(via_helper.columns) + assert "kind" in via_helper.columns # kind promoted out of the dimension columns + assert "type" not in via_helper.columns # no legacy `type` marker column + pd_assert_frame_equal(via_collection, via_helper) + + def test_series_columns_identical(self, dal_db): + coll = Reader(dal_db).values.series_values( + MetricValueFilter(execution_group_ids=[dal_db.execution_group_id]) + ) + for explode in (True, False): + via_collection = coll.to_pandas(explode=explode) + via_helper = series_values_to_frame(coll.items, explode=explode) + assert list(via_collection.columns) == list(via_helper.columns) + assert "kind" in via_helper.columns + assert "execution_group_id" in via_helper.columns + pd_assert_frame_equal(via_collection, via_helper) + + class TestKindSeparation: def test_kind_promoted_out_of_dimensions(self, db_seeded): # kind is a registered CV dimension, so it has its own column; the read layer promotes it @@ -306,3 +352,77 @@ def test_returns_most_recent(self, dal_db): latest = latest_execution_for_group(session, group_id) assert latest is not None assert latest.id == newer_id + + +class _FakeScalar: + """Minimal stand-in exposing the attributes ``detect_scalar_outliers`` reads.""" + + def __init__(self, id_, value, dimensions): + self.id = id_ + self.value = value + self.dimensions = dimensions + + +def _flags(scalars, **policy_kwargs): + """Run detection and return a ``{id: is_outlier}`` mapping.""" + annotated, _ = detect_scalar_outliers(scalars, OutlierPolicy(**policy_kwargs)) + return {a.value.id: a.is_outlier for a in annotated} + + +class TestDetectScalarOutliers: + def test_fallback_path_nan_does_not_disable_detection(self): + # No source_id -> fallback IQR over raw values. A single NaN must not poison the quantiles: + # the wild outlier among the finite values is still flagged, and the NaN is flagged non-finite. + scalars = [ + *[_FakeScalar(i, v, {"metric": "tas"}) for i, v in enumerate([10.0, 10.5, 9.5, 10.2, 9.8])], + _FakeScalar(100, 1000.0, {"metric": "tas"}), # wild outlier + _FakeScalar(200, math.nan, {"metric": "tas"}), # non-finite + ] + flags = _flags(scalars, min_n=4) + assert flags[100] is True + assert flags[200] is True + assert all(flags[i] is False for i in range(5)) + + def test_per_source_all_nan_source_does_not_poison_bounds(self): + # A source whose values are all NaN yields a NaN mean; it must be dropped before the + # quantiles so the well-behaved sources are not all flagged, while the wild source is. + scalars = [ + _FakeScalar(i, v, {"metric": "tas", "source_id": sid}) + for i, (sid, v) in enumerate( + [("A", 10.0), ("B", 10.5), ("C", 9.5), ("D", 10.2), ("WILD", 1000.0)] + ) + ] + scalars += [ + _FakeScalar(500, math.nan, {"metric": "tas", "source_id": "BAD"}), + _FakeScalar(501, math.nan, {"metric": "tas", "source_id": "BAD"}), + ] + flags = _flags(scalars, min_n=4) + assert flags[4] is True # WILD source + assert flags[500] is True and flags[501] is True # non-finite always flagged + assert all(flags[i] is False for i in range(4)) # A-D not poisoned + + def test_zero_iqr_flags_nothing_finite(self): + # Identical per-source means -> zero spread -> collapsed bounds must flag nothing, even for a + # value that differs by an epsilon. Genuinely non-finite values are still flagged. + scalars = [ + _FakeScalar(i, 10.0, {"metric": "tas", "source_id": sid}) + for i, sid in enumerate(["A", "B", "C", "D"]) + ] + scalars.append(_FakeScalar(10, 10.0 + 1e-9, {"metric": "tas", "source_id": "E"})) + scalars.append(_FakeScalar(11, math.nan, {"metric": "tas", "source_id": "F"})) + flags = _flags(scalars, min_n=4) + assert flags[10] is False # epsilon deviation not flagged under zero spread + assert flags[11] is True # non-finite still flagged + assert all(flags[i] is False for i in range(4)) + + def test_min_n_counts_distinct_sources_not_rows(self): + # Many rows but only 3 distinct source_ids; with min_n=4 detection must not run, so the wild + # value is left unflagged despite the row count exceeding min_n. + scalars = [ + _FakeScalar(i, v, {"metric": "tas", "source_id": sid}) + for i, (sid, v) in enumerate( + [("A", 10.0), ("A", 10.1), ("B", 10.5), ("B", 9.9), ("C", 1000.0), ("C", 9.5)] + ) + ] + flags = _flags(scalars, min_n=4) + assert all(v is False for v in flags.values()) From 660315e06e9bc242c2de4b892563ce2980660b37 Mon Sep 17 00:00:00 2001 From: Jared Lewis Date: Sun, 5 Jul 2026 21:59:42 +1000 Subject: [PATCH 18/23] fix(models): tie-break execution ordering on id for deterministic latest --- .../src/climate_ref/models/execution.py | 2 +- .../models/test_execution_latest_ranking.py | 66 +++++++++++++++++-- 2 files changed, 63 insertions(+), 5 deletions(-) diff --git a/packages/climate-ref/src/climate_ref/models/execution.py b/packages/climate-ref/src/climate_ref/models/execution.py index 0fa86e18c..9a89e9ef2 100644 --- a/packages/climate-ref/src/climate_ref/models/execution.py +++ b/packages/climate-ref/src/climate_ref/models/execution.py @@ -82,7 +82,7 @@ class ExecutionGroup(CreatedUpdatedMixin, Base): diagnostic: Mapped["Diagnostic"] = relationship(back_populates="execution_groups") executions: Mapped[list["Execution"]] = relationship( - back_populates="execution_group", order_by="Execution.created_at" + back_populates="execution_group", order_by="Execution.created_at, Execution.id" ) def should_run( diff --git a/packages/climate-ref/tests/unit/models/test_execution_latest_ranking.py b/packages/climate-ref/tests/unit/models/test_execution_latest_ranking.py index c3cbe693d..859d10ed1 100644 --- a/packages/climate-ref/tests/unit/models/test_execution_latest_ranking.py +++ b/packages/climate-ref/tests/unit/models/test_execution_latest_ranking.py @@ -23,13 +23,19 @@ def _add_execution( - db: Database, group_id: int, *, successful: bool | None, offset: int, tag: str + db: Database, + group_id: int, + *, + successful: bool | None, + offset: int, + tag: str, + dataset_hash: str = "h", ) -> Execution: ex = Execution( execution_group_id=group_id, successful=successful, output_fragment=f"out-{tag}", - dataset_hash="h", + dataset_hash=dataset_hash, created_at=_T0 + datetime.timedelta(minutes=offset), ) db.session.add(ex) @@ -37,9 +43,9 @@ def _add_execution( return ex -def _make_group(db: Database) -> ExecutionGroup: +def _make_group(db: Database, *, key: str = "ranking-key") -> ExecutionGroup: diag = db.session.query(Diagnostic).first() - eg = ExecutionGroup(key="ranking-key", diagnostic_id=diag.id, selectors={}, dirty=False) + eg = ExecutionGroup(key=key, diagnostic_id=diag.id, selectors={}, dirty=False) db.session.add(eg) db.session.flush() return eg @@ -96,3 +102,55 @@ def test_pre_rank_default_matches_latest_regardless_of_success(db_seeded: Databa rows = [(g, e) for g, e in get_execution_group_and_latest_filtered(db_seeded.session) if g.id == eg.id] assert len(rows) == 1 assert rows[0][1].output_fragment == "out-fail" + + +def test_executions_relationship_declares_id_tiebreak() -> None: + """The relationship ORDER BY must include Execution.id after created_at. + + A backend-independent assertion on the configured ORDER BY: it fails without the + id tiebreak on any database. The behavioral tests below pass on SQLite regardless + (SQLite happens to return tied rows in rowid order), so this is the true guard. + """ + order_by = ExecutionGroup.executions.property.order_by + names = [col.name for col in order_by] + assert names == ["created_at", "id"] + + +def test_relationship_latest_tie_breaks_on_id(db_seeded: Database) -> None: + """On an identical created_at, executions[-1] must be the higher-id row.""" + with db_seeded.session.begin(): + eg = _make_group(db_seeded) + first = _add_execution(db_seeded, eg.id, successful=True, offset=0, tag="a") + second = _add_execution(db_seeded, eg.id, successful=False, offset=0, tag="b") + + db_seeded.session.expire_all() + reloaded = db_seeded.session.get(ExecutionGroup, eg.id) + assert reloaded is not None + assert reloaded.executions[-1].id == max(first.id, second.id) + assert reloaded.executions[0].id == min(first.id, second.id) + + +def test_should_run_reads_higher_id_execution_on_tie(db_seeded: Database) -> None: + """should_run must read the higher-id execution when created_at ties. + + The latest run (higher id) owns the effective dataset_hash: if it already + matches, nothing is dirty and should_run is False; if it differs, should_run is True. + """ + with db_seeded.session.begin(): + clean = _make_group(db_seeded, key="clean-key") + _add_execution(db_seeded, clean.id, successful=True, offset=0, tag="old", dataset_hash="old") + _add_execution(db_seeded, clean.id, successful=True, offset=0, tag="new", dataset_hash="new") + + dirty = _make_group(db_seeded, key="dirty-key") + _add_execution(db_seeded, dirty.id, successful=True, offset=0, tag="new", dataset_hash="new") + _add_execution(db_seeded, dirty.id, successful=True, offset=0, tag="old", dataset_hash="old") + + db_seeded.session.expire_all() + clean_reloaded = db_seeded.session.get(ExecutionGroup, clean.id) + dirty_reloaded = db_seeded.session.get(ExecutionGroup, dirty.id) + assert clean_reloaded is not None + assert dirty_reloaded is not None + # clean: higher-id execution's hash is "new" -> matches -> no rerun needed. + assert clean_reloaded.should_run("new") is False + # dirty: higher-id execution's hash is "old" -> mismatch -> rerun needed. + assert dirty_reloaded.should_run("new") is True From bafac117bb8fa725655605efe945d2d227808b1d Mon Sep 17 00:00:00 2001 From: Jared Lewis Date: Mon, 6 Jul 2026 10:45:15 +1000 Subject: [PATCH 19/23] fix(cli): allow datasets list --column to select base fields Select and validate --column against DatasetCollection.to_pandas() instead of the facet-only frame, so base fields (slug, id, finalised, created_at, updated_at) are selectable and empty results no longer error on a columnless frame. The --include-files branch keeps validating against its own exploded frame, which owns the file columns. --- .../climate-ref/src/climate_ref/cli/datasets.py | 10 +++++++--- .../climate-ref/tests/unit/cli/test_datasets.py | 13 +++++++++++++ 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/packages/climate-ref/src/climate_ref/cli/datasets.py b/packages/climate-ref/src/climate_ref/cli/datasets.py index e2a802b3e..37c257ff5 100644 --- a/packages/climate-ref/src/climate_ref/cli/datasets.py +++ b/packages/climate-ref/src/climate_ref/cli/datasets.py @@ -115,7 +115,11 @@ def list_( # noqa: PLR0913 data_catalog = data_catalog.head(limit) if column: - missing = set(column) - set(data_catalog.columns) + # Validate/select against a frame that includes base fields (id, slug, ...). + # For --include-files the file columns live only on the exploded frame, so use + # that; otherwise use the empty-safe base+facet frame from the collection. + selectable = data_catalog if include_files else collection.to_pandas() + missing = set(column) - set(selectable.columns) if missing: def format_(columns: Iterable[str]) -> str: @@ -124,10 +128,10 @@ def format_(columns: Iterable[str]) -> str: logger.error( f"Column{'s' if len(missing) > 1 else ''} " f"{format_(missing)} not found in data catalog. " - f"Choose from: {format_(data_catalog.columns)}" + f"Choose from: {format_(selectable.columns)}" ) raise typer.Exit(code=1) - data_catalog = data_catalog[column].sort_values(by=column) + data_catalog = selectable[column].sort_values(by=column) render_dataframe(data_catalog, console=console, output_format=output_format) diff --git a/packages/climate-ref/tests/unit/cli/test_datasets.py b/packages/climate-ref/tests/unit/cli/test_datasets.py index 53639d20a..26cbc0a88 100644 --- a/packages/climate-ref/tests/unit/cli/test_datasets.py +++ b/packages/climate-ref/tests/unit/cli/test_datasets.py @@ -43,6 +43,19 @@ def test_list_column(self, db_seeded, invoke_cli): def test_list_column_invalid(self, db_seeded, invoke_cli): invoke_cli(["datasets", "list", "--column", "wrong"], expected_exit_code=1) + def test_list_column_base_field(self, db_seeded, invoke_cli): + """A base field (not a facet) is selectable, e.g. ``slug``.""" + existing = db_seeded.session.query(CMIP6Dataset).first() + result = invoke_cli(["datasets", "list", "--column", "slug"]) + assert existing.slug in result.stdout + + def test_list_column_on_empty_result(self, db_seeded, invoke_cli): + """``--column`` on an empty result set exits 0 rather than erroring on a columnless frame.""" + result = invoke_cli( + ["datasets", "list", "--dataset-filter", "source_id=DOES-NOT-EXIST", "--column", "slug"] + ) + assert result.exit_code == 0 + def test_list_dataset_filter(self, db_seeded, invoke_cli): result = invoke_cli( ["datasets", "list", "--dataset-filter", "variable_id=tas", "--column", "variable_id"] From c8a97cb5cacc107bb3e6a6abf4047775d5fa2959 Mon Sep 17 00:00:00 2001 From: Jared Lewis Date: Mon, 6 Jul 2026 11:09:54 +1000 Subject: [PATCH 20/23] fix(results): reject OutlierPolicy min_n below 2 --- packages/climate-ref/src/climate_ref/results/outliers.py | 2 +- packages/climate-ref/tests/unit/results/test_results.py | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/packages/climate-ref/src/climate_ref/results/outliers.py b/packages/climate-ref/src/climate_ref/results/outliers.py index 73914db2f..1a12c67ce 100644 --- a/packages/climate-ref/src/climate_ref/results/outliers.py +++ b/packages/climate-ref/src/climate_ref/results/outliers.py @@ -49,7 +49,7 @@ class OutlierPolicy: factor: float = 10.0 """Multiplier on the IQR to set the outlier bounds (``Q1 - factor*IQR``, ``Q3 + factor*IQR``).""" - min_n: int = 4 + min_n: int = attrs.field(default=4, validator=attrs.validators.ge(2)) """ Minimum sample size required to run detection. diff --git a/packages/climate-ref/tests/unit/results/test_results.py b/packages/climate-ref/tests/unit/results/test_results.py index 4a78e3747..8e685c7cc 100644 --- a/packages/climate-ref/tests/unit/results/test_results.py +++ b/packages/climate-ref/tests/unit/results/test_results.py @@ -370,6 +370,15 @@ def _flags(scalars, **policy_kwargs): class TestDetectScalarOutliers: + def test_outlier_policy_rejects_min_n_below_two(self): + with pytest.raises(ValueError): + OutlierPolicy(min_n=1) + with pytest.raises(ValueError): + OutlierPolicy(min_n=0) + # sanity: the safe boundary and default still construct + assert OutlierPolicy(min_n=2).min_n == 2 + assert OutlierPolicy().min_n == 4 + def test_fallback_path_nan_does_not_disable_detection(self): # No source_id -> fallback IQR over raw values. A single NaN must not poison the quantiles: # the wild outlier among the finite values is still flagged, and the NaN is flagged non-finite. From bf2e11a16a90a2ad9355de2214808c4bae865748 Mon Sep 17 00:00:00 2001 From: Jared Lewis Date: Mon, 6 Jul 2026 11:09:21 +1000 Subject: [PATCH 21/23] fix(cli): use safe_path to contain delete-groups rmtree The --remove-outputs guard on delete-groups relied on Path.is_relative_to, which is a purely lexical check and does not collapse '..' or resolve symlinks. A stored output_fragment with a '..' component could pass the guard and let shutil.rmtree delete a directory outside the results root. Reuse the shared safe_path helper from climate_ref_core.paths, which resolves symlinks and '..' before confirming containment under the results root. --- .../src/climate_ref/cli/executions.py | 12 +++--- .../tests/unit/cli/test_executions.py | 43 +++++++++++++++++++ 2 files changed, 50 insertions(+), 5 deletions(-) diff --git a/packages/climate-ref/src/climate_ref/cli/executions.py b/packages/climate-ref/src/climate_ref/cli/executions.py index ebf763c76..c152b0758 100644 --- a/packages/climate-ref/src/climate_ref/cli/executions.py +++ b/packages/climate-ref/src/climate_ref/cli/executions.py @@ -44,6 +44,7 @@ from climate_ref.results.executions import OutputView from climate_ref.results.values import ValuesReader from climate_ref_core.logging import EXECUTION_LOG_FILENAME +from climate_ref_core.paths import safe_path app = typer.Typer(help=__doc__) @@ -348,11 +349,12 @@ def delete_groups( # noqa: PLR0912, PLR0913, PLR0915 config = ctx.obj.config for eg, _ in all_filtered_results: for execution in eg.executions: - output_dir = config.paths.results / execution.output_fragment - - # Safety check - if not output_dir.is_relative_to(config.paths.results): # pragma: no cover - logger.error(f"Skipping unsafe path: {output_dir}") + try: + output_dir = safe_path( + execution.output_fragment, base=config.paths.results, label="results" + ) + except ValueError: + logger.error(f"Skipping unsafe output fragment: {execution.output_fragment!r}") continue if output_dir.exists(): diff --git a/packages/climate-ref/tests/unit/cli/test_executions.py b/packages/climate-ref/tests/unit/cli/test_executions.py index b646e9789..dade80e26 100644 --- a/packages/climate-ref/tests/unit/cli/test_executions.py +++ b/packages/climate-ref/tests/unit/cli/test_executions.py @@ -619,6 +619,49 @@ def test_delete_groups_removes_outputs(self, db_with_groups, tmp_path, invoke_cl # Verify success message includes output directories assert "and their output directories" in result.stdout + def test_delete_groups_skips_escaping_output_fragment(self, db_with_groups, tmp_path, invoke_cli, config): + """An output_fragment that escapes the results root via '..' must not be deleted.""" + results_path = tmp_path / "results" + results_path.mkdir() + + # Mock config.paths.results to use tmp_path + config.paths.results = results_path + config.save() + + # A directory outside the results root that a malicious/corrupt fragment would target. + outside_dir = tmp_path / "outside" + outside_dir.mkdir() + (outside_dir / "sentinel.txt").write_text("do not delete") + + # Point the execution's output_fragment outside the results root. + eg = db_with_groups.session.query(ExecutionGroup).first() + execution = eg.executions[0] + execution.output_fragment = "../outside" + db_with_groups.session.commit() + + # Run command with --remove-outputs + with patch("climate_ref.cli.executions.typer.confirm", return_value=True): + result = invoke_cli( + [ + "executions", + "delete-groups", + "--diagnostic", + "enso", + "--remove-outputs", + "--force", + ] + ) + + # Assert success (the unsafe fragment is skipped, not fatal) + assert result.exit_code == 0 + + # Verify the directory outside the results root was NOT removed + assert outside_dir.exists() + assert (outside_dir / "sentinel.txt").exists() + + # Verify database records were still deleted (only enso diagnostics: eg1 and eg3) + assert db_with_groups.session.query(ExecutionGroup).count() == 4 + def test_delete_groups_without_remove_outputs_flag(self, db_with_groups, tmp_path, invoke_cli, config): """Test that output directories are NOT removed when --remove-outputs flag is omitted""" # Create actual output directories in tmp_path From e27765fa0d4e80c5535243f98fdf6a7cc3a19643 Mon Sep 17 00:00:00 2001 From: Jared Lewis Date: Mon, 6 Jul 2026 11:40:02 +1000 Subject: [PATCH 22/23] fix(cli): let execution filter errors propagate instead of masking them The list-groups and delete-groups commands wrapped their filter query calls in a blanket except Exception that relabelled any failure as 'Error applying filters' and exited 1. This swallowed tracebacks and misattributed DB/programming errors to the user's filter input. Bad facet values already warn-empty rather than raising here, and malformed --filter syntax is caught separately by parse_facet_filters, so the removed wrappers only ever caught unexpected errors that should surface with a full traceback. --- .../src/climate_ref/cli/executions.py | 46 ++++++++----------- .../tests/unit/cli/test_executions.py | 17 +++++++ 2 files changed, 36 insertions(+), 27 deletions(-) diff --git a/packages/climate-ref/src/climate_ref/cli/executions.py b/packages/climate-ref/src/climate_ref/cli/executions.py index c152b0758..037e13fb4 100644 --- a/packages/climate-ref/src/climate_ref/cli/executions.py +++ b/packages/climate-ref/src/climate_ref/cli/executions.py @@ -170,20 +170,16 @@ def list_groups( # noqa: PLR0913 total_count = session.query(ExecutionGroup).count() # Apply filters to query - try: - collection = reader.executions.groups( - ExecutionGroupFilter( - diagnostic_contains=diagnostic, - provider_contains=provider, - facets=facet_filters or None, - successful=successful, - dirty=dirty, - ), - limit=limit, - ) - except Exception as e: # pragma: no cover - logger.error(f"Error applying filters: {e}") - raise typer.Exit(code=1) + collection = reader.executions.groups( + ExecutionGroupFilter( + diagnostic_contains=diagnostic, + provider_contains=provider, + facets=facet_filters or None, + successful=successful, + dirty=dirty, + ), + limit=limit, + ) # Check if any results found if len(collection) == 0: @@ -216,7 +212,7 @@ def list_groups( # noqa: PLR0913 # Stays on `get_execution_group_and_latest_filtered` rather than `reader.executions.groups()` by design. # Revisit once a mutable query surface exists. @app.command() -def delete_groups( # noqa: PLR0912, PLR0913, PLR0915 +def delete_groups( # noqa: PLR0912, PLR0913 ctx: typer.Context, diagnostic: Annotated[ list[str] | None, @@ -295,18 +291,14 @@ def delete_groups( # noqa: PLR0912, PLR0913, PLR0915 logger.debug(f"Applying filters: {filters}") # Apply filters to query - try: - all_filtered_results = get_execution_group_and_latest_filtered( - session, - diagnostic_filters=filters.diagnostic, - provider_filters=filters.provider, - facet_filters=filters.facets, - successful=successful, - dirty=dirty, - ) - except Exception as e: # pragma: no cover - logger.error(f"Error applying filters: {e}") - raise typer.Exit(code=1) + all_filtered_results = get_execution_group_and_latest_filtered( + session, + diagnostic_filters=filters.diagnostic, + provider_filters=filters.provider, + facet_filters=filters.facets, + successful=successful, + dirty=dirty, + ) # Check if any results found if not all_filtered_results: diff --git a/packages/climate-ref/tests/unit/cli/test_executions.py b/packages/climate-ref/tests/unit/cli/test_executions.py index dade80e26..8c267c4bd 100644 --- a/packages/climate-ref/tests/unit/cli/test_executions.py +++ b/packages/climate-ref/tests/unit/cli/test_executions.py @@ -464,6 +464,23 @@ def test_delete_groups_no_filters_error(self, invoke_cli): assert "THIS WILL DELETE ALL EXECUTION GROUPS IN THE DATABASE" in result.stderr + def test_delete_groups_backend_error_propagates(self, db_with_groups, invoke_cli): + """A genuine backend error must surface, not be relabelled 'Error applying filters'.""" + + def _boom(*args, **kwargs): + raise RuntimeError("db exploded") + + with patch("climate_ref.cli.executions.get_execution_group_and_latest_filtered", _boom): + result = invoke_cli( + ["executions", "delete-groups", "--diagnostic", "enso", "--force"], + expected_exit_code=1, + ) + + assert "Error applying filters" not in result.stdout + assert "Error applying filters" not in result.stderr + assert isinstance(result.exception, RuntimeError) + assert str(result.exception) == "db exploded" + def test_delete_groups_no_results_warning(self, db_with_groups, invoke_cli): result = invoke_cli(["executions", "delete-groups", "--filter", "source_id=NONEXISTENT", "--force"]) From c556e241caeeeb4ee0f306669d1b6b7adba3e6f5 Mon Sep 17 00:00:00 2001 From: Jared Lewis Date: Mon, 6 Jul 2026 12:43:08 +1000 Subject: [PATCH 23/23] chore: raise valueerror --- .../src/climate_ref/models/dataset_query.py | 12 ++++++++---- .../tests/unit/datasets/test_dataset_query.py | 16 +++++++--------- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/packages/climate-ref/src/climate_ref/models/dataset_query.py b/packages/climate-ref/src/climate_ref/models/dataset_query.py index 7094c1a5e..a102a8bab 100644 --- a/packages/climate-ref/src/climate_ref/models/dataset_query.py +++ b/packages/climate-ref/src/climate_ref/models/dataset_query.py @@ -74,19 +74,23 @@ def select_datasets( ``latest_group_by`` is the adapter's ``dataset_id_metadata``, which is used as the partition columns for the latest-version window. - It is optional because ``select_datasets`` lives in the models layer - and must not import the adapter registry, so it cannot look this up itself; callers pass it through. + It is passed in rather than looked up here because ``select_datasets`` lives in the models layer + and must not import the adapter registry, so it cannot resolve it itself; callers pass it through. - ``filter.latest_only`` does not take effect unless ``latest_group_by`` is also given (non-empty). + ``latest_group_by`` is required whenever ``filter.latest_only`` is True (the default): + passing ``latest_only=True`` without it raises ``ValueError`` rather than silently returning an + un-deduplicated result. When both are set, rows are deduplicated with a ``RANK() OVER (PARTITION BY ORDER BY version_key DESC)`` window (applied after all other filters/joins), keeping every row tied at the maximum ``version_key`` -- so ties are not silently dropped. + Set ``latest_only=False`` to list every version; ``latest_group_by`` is then ignored. Raises ------ ValueError - If a key in ``filter.facets`` is not a mapped column on the target entity. + If ``filter.latest_only`` is True but ``latest_group_by`` is not provided, + or if a key in ``filter.facets`` is not a mapped column on the target entity. """ entity = _entity_for(filter.source_type) diff --git a/packages/climate-ref/tests/unit/datasets/test_dataset_query.py b/packages/climate-ref/tests/unit/datasets/test_dataset_query.py index 7770e4e1a..e24175b93 100644 --- a/packages/climate-ref/tests/unit/datasets/test_dataset_query.py +++ b/packages/climate-ref/tests/unit/datasets/test_dataset_query.py @@ -183,15 +183,13 @@ def test_latest_only_false_ignores_latest_group_by(self, db_with_versions): assert db_with_versions.dq_ids["v2"] in ids assert db_with_versions.dq_ids["v10"] in ids - def test_latest_group_by_none_is_inert(self, db_with_versions): - """``latest_only=True`` alone (no ``latest_group_by``) must not dedup -- the two-flag API.""" - stmt = select_datasets( - DatasetFilter(source_type=SourceDatasetType.CMIP6, facets={"source_id": ("TEST-MODEL",)}) - ) - rows = db_with_versions.session.execute(stmt).scalars().unique().all() - ids = {r.id for r in rows} - assert db_with_versions.dq_ids["v2"] in ids - assert db_with_versions.dq_ids["v10"] in ids + def test_latest_only_requires_latest_group_by(self): + """``latest_only=True`` (the default) with no ``latest_group_by`` is rejected: the two must be + supplied together, so a caller cannot silently receive an un-deduplicated result.""" + with pytest.raises(ValueError, match="latest_group_by"): + select_datasets( + DatasetFilter(source_type=SourceDatasetType.CMIP6, facets={"source_id": ("TEST-MODEL",)}) + ) def test_source_type_is_required(self): """``DatasetFilter`` has no default ``source_type``: a typed listing must choose a type."""