diff --git a/CHANGELOG.md b/CHANGELOG.md index 506cc64dbc..7d9de97261 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,6 @@ ## Unreleased +- Scoring: Added `aggregate(key, agg=metric)` factory composing any existing `Metric` with a dict-key selector for dict-valued `Score.value` -- removes per-eval re-implementation of `mean_of` / `stderr_of` / etc. - OpenAI: Add GPT 5.5 as computer use model and exclude 'chat' and 'instant' models from computer use. - OpenAI Compatible: Parse OpenRouter-style `reasoning_details` in OpenAI-compatible responses. - VLLM: Preserve dotted vLLM server arg keys. diff --git a/src/inspect_ai/scorer/__init__.py b/src/inspect_ai/scorer/__init__.py index 30d9f4a9e7..56697ae935 100644 --- a/src/inspect_ai/scorer/__init__.py +++ b/src/inspect_ai/scorer/__init__.py @@ -22,6 +22,7 @@ value_to_float, ) from ._metrics.accuracy import accuracy +from ._metrics.aggregate import aggregate from ._metrics.grouped import grouped from ._metrics.mean import mean from ._metrics.perplexity import perplexity_per_seq, perplexity_per_token @@ -65,6 +66,7 @@ "Value", "ValueToFloat", "accuracy", + "aggregate", "answer", "at_least", "bootstrap_stderr", diff --git a/src/inspect_ai/scorer/_metrics/__init__.py b/src/inspect_ai/scorer/_metrics/__init__.py index a36afc6213..f14e1aa846 100644 --- a/src/inspect_ai/scorer/_metrics/__init__.py +++ b/src/inspect_ai/scorer/_metrics/__init__.py @@ -1,4 +1,5 @@ from .accuracy import accuracy +from .aggregate import aggregate from .grouped import grouped from .mean import mean from .perplexity import perplexity_per_seq, perplexity_per_token @@ -6,6 +7,7 @@ __all__ = [ "accuracy", + "aggregate", "mean", "grouped", "perplexity_per_token", diff --git a/src/inspect_ai/scorer/_metrics/aggregate.py b/src/inspect_ai/scorer/_metrics/aggregate.py new file mode 100644 index 0000000000..aef8419adf --- /dev/null +++ b/src/inspect_ai/scorer/_metrics/aggregate.py @@ -0,0 +1,118 @@ +from typing import Literal, cast + +from .._metric import ( + Metric, + MetricProtocol, + SampleScore, + Score, + Value, + ValueToFloat, + metric, + value_to_float, +) + + +@metric +def aggregate( + key: str, + agg: Metric, + *, + to_float: ValueToFloat = value_to_float(), + on_missing: Literal["error", "skip", "zero"] = "error", +) -> Metric: + """Apply ``agg`` to the ``key`` field extracted from each dict-valued ``Score.value``. + + Many scorers emit a ``dict`` as ``Score.value`` (multiple numeric fields + per sample). ``aggregate`` composes any existing ``Metric`` (``mean``, + ``stderr``, ``std``, etc.) with a key-selector so per-key aggregation + doesn't have to be re-implemented in every eval. + + Args: + key: The key to extract from each sample's ``score.value`` dict. + agg: The aggregator metric to apply once the per-sample scalar values + have been unwrapped. Typically ``mean()``, ``stderr()``, ``std()``, + ``accuracy()``, or any other registered ``Metric`` whose + implementation reads ``score.value``. + to_float: Function for mapping the extracted dict-value to ``float``. + Defaults to :func:`value_to_float` (handles CORRECT/INCORRECT, + booleans, and numeric values). + on_missing: How to handle samples whose ``score.value`` dict is + missing ``key`` or has it set to ``None``: + + - ``"error"`` (default) -- raise ``ValueError``. + - ``"skip"`` -- exclude the sample from the aggregator. + - ``"zero"`` -- treat the missing entry as ``0.0``. + + Returns: + A new metric that returns whatever ``agg`` returns when applied to the + per-sample scalar values extracted at ``key``. + + Raises: + ValueError: If ``Score.value`` is not a ``dict``, or if ``key`` is + missing/``None`` and ``on_missing="error"``. + + Examples: + ```python + from inspect_ai.scorer import ( + Metric, aggregate, mean, metric, stderr, + ) + + @metric + def element_accuracy() -> Metric: + return aggregate("element_acc", agg=mean(), on_missing="skip") + + @metric + def element_accuracy_stderr() -> Metric: + return aggregate("element_acc", agg=stderr(), on_missing="skip") + ``` + """ + agg_protocol = cast(MetricProtocol, agg) + + def aggregate_metric(scores: list[SampleScore]) -> Value: + unwrapped: list[SampleScore] = [] + for sample_score in scores: + value = sample_score.score.value + if not isinstance(value, dict): + raise ValueError( + f"aggregate('{key}') requires Score.value to be a dict, " + f"got {type(value).__name__}: {value!r}" + ) + + if key not in value or value[key] is None: + if on_missing == "error": + if key not in value: + raise ValueError( + f"aggregate('{key}') key not found in Score.value. " + f"Available keys: {list(value.keys())}" + ) + raise ValueError( + f"aggregate('{key}') value is None in Score.value." + ) + if on_missing == "skip": + continue + # on_missing == "zero" + extracted: float = 0.0 + else: + extracted = to_float(value[key]) + + # Rebuild the SampleScore with the unwrapped scalar value so that + # ``agg`` (which reads ``score.value`` via ``score.as_float()`` or + # ``to_float``) sees a single number rather than the original dict. + unwrapped.append( + SampleScore( + score=Score( + value=extracted, + answer=sample_score.score.answer, + explanation=sample_score.score.explanation, + metadata=sample_score.score.metadata, + history=sample_score.score.history, + ), + sample_id=sample_score.sample_id, + sample_metadata=sample_score.sample_metadata, + scorer=sample_score.scorer, + ) + ) + + return agg_protocol(unwrapped) + + return aggregate_metric diff --git a/tests/scorer/test_aggregate.py b/tests/scorer/test_aggregate.py new file mode 100644 index 0000000000..567aa63ac0 --- /dev/null +++ b/tests/scorer/test_aggregate.py @@ -0,0 +1,129 @@ +"""Unit tests for the ``aggregate`` metric factory.""" + +import math + +import pytest + +from inspect_ai.scorer import ( + Metric, + Score, + SampleScore, + aggregate, + mean, + metric, + std, + stderr, + value_to_float, +) + + +def _sample(value: object, sample_id: int = 0) -> SampleScore: + return SampleScore( + score=Score(value=value), # type: ignore[arg-type] + sample_id=sample_id, + ) + + +class TestAggregateBasics: + def test_mean_of_key(self) -> None: + scores = [ + _sample({"acc": 1.0, "other": 9.0}, 0), + _sample({"acc": 0.0, "other": 8.0}, 1), + _sample({"acc": 0.5, "other": 7.0}, 2), + ] + result = aggregate("acc", agg=mean())(scores) + assert math.isclose(float(result), 0.5) + + def test_stderr_of_key(self) -> None: + scores = [ + _sample({"acc": 1.0}, 0), + _sample({"acc": 0.0}, 1), + _sample({"acc": 1.0}, 2), + _sample({"acc": 0.0}, 3), + ] + result = aggregate("acc", agg=stderr())(scores) + assert float(result) > 0.0 + + def test_std_of_key(self) -> None: + scores = [ + _sample({"acc": 1.0}, 0), + _sample({"acc": 3.0}, 1), + _sample({"acc": 5.0}, 2), + ] + result = aggregate("acc", agg=std())(scores) + assert float(result) > 1.0 + + def test_passes_metadata_through(self) -> None: + seen_metadata: list[dict] = [] + + @metric + def collect_metadata() -> Metric: + def metric_fn(samples: list[SampleScore]) -> float: + for s in samples: + seen_metadata.append(s.sample_metadata or {}) + return float(len(samples)) + + return metric_fn + + scores = [ + SampleScore( + score=Score(value={"acc": 1.0}), + sample_id=i, + sample_metadata={"group": grp}, + ) + for i, grp in enumerate(["a", "b", "a"]) + ] + aggregate("acc", agg=collect_metadata())(scores) + assert seen_metadata == [{"group": "a"}, {"group": "b"}, {"group": "a"}] + + +class TestOnMissing: + def test_error_on_missing_key(self) -> None: + scores = [ + _sample({"acc": 1.0}, 0), + _sample({"other": 0.0}, 1), + ] + with pytest.raises(ValueError, match="key not found"): + aggregate("acc", agg=mean())(scores) + + def test_error_on_none_value(self) -> None: + scores = [ + _sample({"acc": 1.0}, 0), + _sample({"acc": None}, 1), + ] + with pytest.raises(ValueError, match="value is None"): + aggregate("acc", agg=mean())(scores) + + def test_skip_excludes_missing(self) -> None: + scores = [ + _sample({"acc": 1.0}, 0), + _sample({"other": 0.0}, 1), + _sample({"acc": 0.0}, 2), + ] + result = aggregate("acc", agg=mean(), on_missing="skip")(scores) + assert math.isclose(float(result), 0.5) + + def test_zero_treats_missing_as_zero(self) -> None: + scores = [ + _sample({"acc": 1.0}, 0), + _sample({"other": 0.0}, 1), + _sample({"acc": 1.0}, 2), + ] + result = aggregate("acc", agg=mean(), on_missing="zero")(scores) + assert math.isclose(float(result), 2.0 / 3.0) + + +class TestValueShapes: + def test_rejects_non_dict_value(self) -> None: + scores = [_sample(0.5, 0)] + with pytest.raises(ValueError, match="requires Score.value to be a dict"): + aggregate("acc", agg=mean())(scores) + + def test_to_float_converts_correctly(self) -> None: + scores = [ + _sample({"label": "C"}, 0), + _sample({"label": "I"}, 1), + _sample({"label": "C"}, 2), + ] + result = aggregate("label", agg=mean(), to_float=value_to_float())(scores) + assert math.isclose(float(result), 2.0 / 3.0)