feat(scorer): add aggregate(key, agg=metric) factory for dict-valued Score.value

CHANGELOG.md

@@ -1,5 +1,6 @@
 ## Unreleased
 
+- Scoring: Added `aggregate(key, agg=metric)` factory composing any existing `Metric` with a dict-key selector for dict-valued `Score.value` -- removes per-eval re-implementation of `mean_of` / `stderr_of` / etc.
 - OpenAI: Add GPT 5.5 as computer use model and exclude 'chat' and 'instant' models from computer use.
 - OpenAI Compatible: Parse OpenRouter-style `reasoning_details` in OpenAI-compatible responses.
 - VLLM: Preserve dotted vLLM server arg keys.

src/inspect_ai/scorer/__init__.py

@@ -22,6 +22,7 @@
     value_to_float,
 )
 from ._metrics.accuracy import accuracy
+from ._metrics.aggregate import aggregate
 from ._metrics.grouped import grouped
 from ._metrics.mean import mean
 from ._metrics.perplexity import perplexity_per_seq, perplexity_per_token
@@ -65,6 +66,7 @@
     "Value",
     "ValueToFloat",
     "accuracy",
+    "aggregate",
     "answer",
     "at_least",
     "bootstrap_stderr",

src/inspect_ai/scorer/_metrics/__init__.py

@@ -1,11 +1,13 @@
 from .accuracy import accuracy
+from .aggregate import aggregate
 from .grouped import grouped
 from .mean import mean
 from .perplexity import perplexity_per_seq, perplexity_per_token
 from .std import bootstrap_stderr, std, stderr, var
 
 __all__ = [
     "accuracy",
+    "aggregate",
     "mean",
     "grouped",
     "perplexity_per_token",

src/inspect_ai/scorer/_metrics/aggregate.py

@@ -0,0 +1,118 @@
+from typing import Literal, cast
+
+from .._metric import (
+    Metric,
+    MetricProtocol,
+    SampleScore,
+    Score,
+    Value,
+    ValueToFloat,
+    metric,
+    value_to_float,
+)
+
+
+@metric
+def aggregate(
+    key: str,
+    agg: Metric,
+    *,
+    to_float: ValueToFloat = value_to_float(),
+    on_missing: Literal["error", "skip", "zero"] = "error",
+) -> Metric:
+    """Apply ``agg`` to the ``key`` field extracted from each dict-valued ``Score.value``.
+
+    Many scorers emit a ``dict`` as ``Score.value`` (multiple numeric fields
+    per sample). ``aggregate`` composes any existing ``Metric`` (``mean``,
+    ``stderr``, ``std``, etc.) with a key-selector so per-key aggregation
+    doesn't have to be re-implemented in every eval.
+
+    Args:
+        key: The key to extract from each sample's ``score.value`` dict.
+        agg: The aggregator metric to apply once the per-sample scalar values
+            have been unwrapped. Typically ``mean()``, ``stderr()``, ``std()``,
+            ``accuracy()``, or any other registered ``Metric`` whose
+            implementation reads ``score.value``.
+        to_float: Function for mapping the extracted dict-value to ``float``.
+            Defaults to :func:`value_to_float` (handles CORRECT/INCORRECT,
+            booleans, and numeric values).
+        on_missing: How to handle samples whose ``score.value`` dict is
+            missing ``key`` or has it set to ``None``:
+
+            - ``"error"`` (default) -- raise ``ValueError``.
+            - ``"skip"`` -- exclude the sample from the aggregator.
+            - ``"zero"`` -- treat the missing entry as ``0.0``.
+
+    Returns:
+        A new metric that returns whatever ``agg`` returns when applied to the
+        per-sample scalar values extracted at ``key``.
+
+    Raises:
+        ValueError: If ``Score.value`` is not a ``dict``, or if ``key`` is
+            missing/``None`` and ``on_missing="error"``.
+
+    Examples:
+        ```python
+        from inspect_ai.scorer import (
+            Metric, aggregate, mean, metric, stderr,
+        )
+
+        @metric
+        def element_accuracy() -> Metric:
+            return aggregate("element_acc", agg=mean(), on_missing="skip")
+
+        @metric
+        def element_accuracy_stderr() -> Metric:
+            return aggregate("element_acc", agg=stderr(), on_missing="skip")
+        ```
+    """
+    agg_protocol = cast(MetricProtocol, agg)
+
+    def aggregate_metric(scores: list[SampleScore]) -> Value:
+        unwrapped: list[SampleScore] = []
+        for sample_score in scores:
+            value = sample_score.score.value
+            if not isinstance(value, dict):
+                raise ValueError(
+                    f"aggregate('{key}') requires Score.value to be a dict, "
+                    f"got {type(value).__name__}: {value!r}"
+                )
+
+            if key not in value or value[key] is None:
+                if on_missing == "error":
+                    if key not in value:
+                        raise ValueError(
+                            f"aggregate('{key}') key not found in Score.value. "
+                            f"Available keys: {list(value.keys())}"
+                        )
+                    raise ValueError(
+                        f"aggregate('{key}') value is None in Score.value."
+                    )
+                if on_missing == "skip":
+                    continue
+                # on_missing == "zero"
+                extracted: float = 0.0
+            else:
+                extracted = to_float(value[key])
+
+            # Rebuild the SampleScore with the unwrapped scalar value so that
+            # ``agg`` (which reads ``score.value`` via ``score.as_float()`` or
+            # ``to_float``) sees a single number rather than the original dict.
+            unwrapped.append(
+                SampleScore(
+                    score=Score(
+                        value=extracted,
+                        answer=sample_score.score.answer,
+                        explanation=sample_score.score.explanation,
+                        metadata=sample_score.score.metadata,
+                        history=sample_score.score.history,
+                    ),
+                    sample_id=sample_score.sample_id,
+                    sample_metadata=sample_score.sample_metadata,
+                    scorer=sample_score.scorer,
+                )
+            )
+
+        return agg_protocol(unwrapped)
+
+    return aggregate_metric

tests/scorer/test_aggregate.py

@@ -0,0 +1,129 @@
+"""Unit tests for the ``aggregate`` metric factory."""
+
+import math
+
+import pytest
+
+from inspect_ai.scorer import (
+    Metric,
+    Score,
+    SampleScore,
+    aggregate,
+    mean,
+    metric,
+    std,
+    stderr,
+    value_to_float,
+)
+
+
+def _sample(value: object, sample_id: int = 0) -> SampleScore:
+    return SampleScore(
+        score=Score(value=value),  # type: ignore[arg-type]
+        sample_id=sample_id,
+    )
+
+
+class TestAggregateBasics:
+    def test_mean_of_key(self) -> None:
+        scores = [
+            _sample({"acc": 1.0, "other": 9.0}, 0),
+            _sample({"acc": 0.0, "other": 8.0}, 1),
+            _sample({"acc": 0.5, "other": 7.0}, 2),
+        ]
+        result = aggregate("acc", agg=mean())(scores)
+        assert math.isclose(float(result), 0.5)
+
+    def test_stderr_of_key(self) -> None:
+        scores = [
+            _sample({"acc": 1.0}, 0),
+            _sample({"acc": 0.0}, 1),
+            _sample({"acc": 1.0}, 2),
+            _sample({"acc": 0.0}, 3),
+        ]
+        result = aggregate("acc", agg=stderr())(scores)
+        assert float(result) > 0.0
+
+    def test_std_of_key(self) -> None:
+        scores = [
+            _sample({"acc": 1.0}, 0),
+            _sample({"acc": 3.0}, 1),
+            _sample({"acc": 5.0}, 2),
+        ]
+        result = aggregate("acc", agg=std())(scores)
+        assert float(result) > 1.0
+
+    def test_passes_metadata_through(self) -> None:
+        seen_metadata: list[dict] = []
+
+        @metric
+        def collect_metadata() -> Metric:
+            def metric_fn(samples: list[SampleScore]) -> float:
+                for s in samples:
+                    seen_metadata.append(s.sample_metadata or {})
+                return float(len(samples))
+
+            return metric_fn
+
+        scores = [
+            SampleScore(
+                score=Score(value={"acc": 1.0}),
+                sample_id=i,
+                sample_metadata={"group": grp},
+            )
+            for i, grp in enumerate(["a", "b", "a"])
+        ]
+        aggregate("acc", agg=collect_metadata())(scores)
+        assert seen_metadata == [{"group": "a"}, {"group": "b"}, {"group": "a"}]
+
+
+class TestOnMissing:
+    def test_error_on_missing_key(self) -> None:
+        scores = [
+            _sample({"acc": 1.0}, 0),
+            _sample({"other": 0.0}, 1),
+        ]
+        with pytest.raises(ValueError, match="key not found"):
+            aggregate("acc", agg=mean())(scores)
+
+    def test_error_on_none_value(self) -> None:
+        scores = [
+            _sample({"acc": 1.0}, 0),
+            _sample({"acc": None}, 1),
+        ]
+        with pytest.raises(ValueError, match="value is None"):
+            aggregate("acc", agg=mean())(scores)
+
+    def test_skip_excludes_missing(self) -> None:
+        scores = [
+            _sample({"acc": 1.0}, 0),
+            _sample({"other": 0.0}, 1),
+            _sample({"acc": 0.0}, 2),
+        ]
+        result = aggregate("acc", agg=mean(), on_missing="skip")(scores)
+        assert math.isclose(float(result), 0.5)
+
+    def test_zero_treats_missing_as_zero(self) -> None:
+        scores = [
+            _sample({"acc": 1.0}, 0),
+            _sample({"other": 0.0}, 1),
+            _sample({"acc": 1.0}, 2),
+        ]
+        result = aggregate("acc", agg=mean(), on_missing="zero")(scores)
+        assert math.isclose(float(result), 2.0 / 3.0)
+
+
+class TestValueShapes:
+    def test_rejects_non_dict_value(self) -> None:
+        scores = [_sample(0.5, 0)]
+        with pytest.raises(ValueError, match="requires Score.value to be a dict"):
+            aggregate("acc", agg=mean())(scores)
+
+    def test_to_float_converts_correctly(self) -> None:
+        scores = [
+            _sample({"label": "C"}, 0),
+            _sample({"label": "I"}, 1),
+            _sample({"label": "C"}, 2),
+        ]
+        result = aggregate("label", agg=mean(), to_float=value_to_float())(scores)
+        assert math.isclose(float(result), 2.0 / 3.0)