From 0ac7474a7b30cb26ef319b8e3446685e150d0456 Mon Sep 17 00:00:00 2001
From: antnewman <antjsnewman@outlook.com>
Date: Wed, 6 May 2026 13:58:08 +0100
Subject: [PATCH 1/7] feat(scorer): add aggregate(key, agg=...) metric factory
 (#3735)

Many scorers emit dict-valued Score.value (multiple numeric fields per
sample) and then need per-key metrics. `aggregate(key, agg)` selects one
field by key and applies any standard metric (mean, stderr, accuracy,
...) to it, replacing the need for bespoke per-aggregator factories like
mean_of/std_of/stderr_of.

- on_missing controls samples missing the key: "error" (default), "skip",
  "zero".
- to_float follows the same convention as the other built-in metrics.
- Non-dict Score.value raises ValueError with a clear message.

Includes tests covering composition with mean/stderr, on_missing modes,
and key-selection semantics. Doc section added under "Per-Key
Aggregation" in docs/scorers.qmd.
---
 docs/scorers.qmd                            | 20 +++++
 src/inspect_ai/scorer/__init__.py           |  2 +
 src/inspect_ai/scorer/_metrics/__init__.py  |  2 +
 src/inspect_ai/scorer/_metrics/aggregate.py | 94 +++++++++++++++++++++
 tests/scorer/test_metric.py                 | 92 +++++++++++++++++++-
 5 files changed, 209 insertions(+), 1 deletion(-)
 create mode 100644 src/inspect_ai/scorer/_metrics/aggregate.py

diff --git a/docs/scorers.qmd b/docs/scorers.qmd
index d74160dab8..f19e91f788 100644
--- a/docs/scorers.qmd
+++ b/docs/scorers.qmd
@@ -746,6 +746,26 @@ grouped(accuracy(), "category", name_template="category_{group_name}")
 
 This would produce metrics named `category_physics`, `category_chemistry`, etc. instead of just `physics`, `chemistry`. It does not affect the "all" metric, so that can be named separately.
 
+### Per-Key Aggregation
+
+When a scorer emits a dict-valued `Score.value` (multiple numeric fields per sample), `aggregate()` applies any standard metric to a single field selected by key. This composes with `mean()`, `stderr()`, `accuracy()`, etc., so you do not need a bespoke factory per aggregator.
+
+For example, given a scorer that returns `{"element_acc": ..., "action_f1": ...}` per sample:
+
+``` python
+from inspect_ai.scorer import aggregate, mean, stderr
+
+@metric
+def element_accuracy() -> Metric:
+    return aggregate("element_acc", agg=mean())
+
+@metric
+def element_accuracy_stderr() -> Metric:
+    return aggregate("element_acc", agg=stderr())
+```
+
+Pass `on_missing="skip"` to drop samples that don't have the key, or `on_missing="zero"` to treat them as `0.0`. The default (`"error"`) raises if any sample is missing the key.
+
 ### Clustered Stderr
 
 The `stderr()` metric supports computing [clustered standard errors](https://en.wikipedia.org/wiki/Clustered_standard_errors) via the `cluster` parameter. Most scorers already include `stderr()` as a built-in metric, so to compute clustered standard errors you'll want to specify custom `metrics` for your task (which will override the scorer's built in metrics).
diff --git a/src/inspect_ai/scorer/__init__.py b/src/inspect_ai/scorer/__init__.py
index 30d9f4a9e7..56697ae935 100644
--- a/src/inspect_ai/scorer/__init__.py
+++ b/src/inspect_ai/scorer/__init__.py
@@ -22,6 +22,7 @@
     value_to_float,
 )
 from ._metrics.accuracy import accuracy
+from ._metrics.aggregate import aggregate
 from ._metrics.grouped import grouped
 from ._metrics.mean import mean
 from ._metrics.perplexity import perplexity_per_seq, perplexity_per_token
@@ -65,6 +66,7 @@
     "Value",
     "ValueToFloat",
     "accuracy",
+    "aggregate",
     "answer",
     "at_least",
     "bootstrap_stderr",
diff --git a/src/inspect_ai/scorer/_metrics/__init__.py b/src/inspect_ai/scorer/_metrics/__init__.py
index a36afc6213..f14e1aa846 100644
--- a/src/inspect_ai/scorer/_metrics/__init__.py
+++ b/src/inspect_ai/scorer/_metrics/__init__.py
@@ -1,4 +1,5 @@
 from .accuracy import accuracy
+from .aggregate import aggregate
 from .grouped import grouped
 from .mean import mean
 from .perplexity import perplexity_per_seq, perplexity_per_token
@@ -6,6 +7,7 @@
 
 __all__ = [
     "accuracy",
+    "aggregate",
     "mean",
     "grouped",
     "perplexity_per_token",
diff --git a/src/inspect_ai/scorer/_metrics/aggregate.py b/src/inspect_ai/scorer/_metrics/aggregate.py
new file mode 100644
index 0000000000..dede48d257
--- /dev/null
+++ b/src/inspect_ai/scorer/_metrics/aggregate.py
@@ -0,0 +1,94 @@
+from logging import getLogger
+from typing import Literal, cast
+
+from .._metric import (
+    Metric,
+    MetricProtocol,
+    SampleScore,
+    Value,
+    ValueToFloat,
+    metric,
+    value_to_float,
+)
+
+logger = getLogger(__name__)
+
+
+@metric
+def aggregate(
+    key: str,
+    agg: Metric,
+    *,
+    to_float: ValueToFloat = value_to_float(),
+    on_missing: Literal["error", "skip", "zero"] = "error",
+) -> Metric:
+    """Apply `agg` to a single key extracted from each dict-valued `Score.value`.
+
+    Many scorers emit dict-valued scores (multiple numeric fields per sample).
+    `aggregate` selects one field by `key` and feeds the resulting scalar
+    `SampleScore`s into `agg`, so any standard metric (`mean`, `stderr`,
+    `std`, `accuracy`, ...) can be applied per key.
+
+    Args:
+       key: Field to extract from each sample's dict-valued `Score.value`.
+       agg: Metric to apply to the extracted values.
+       to_float: Function for mapping the extracted `Value` to a float. The
+          default `value_to_float()` maps CORRECT ("C") to 1.0, INCORRECT ("I")
+          to 0, PARTIAL ("P") to 0.5, and NOANSWER ("N") to 0, casts numeric
+          values to float directly, and prints a warning and returns 0 if the
+          Value is a complex object (list or dict).
+       on_missing: How to handle samples whose `score.value` does not contain
+          `key`:
+
+          - `"error"` (default): raise `ValueError`.
+          - `"skip"`: exclude the sample from `agg`.
+          - `"zero"`: include the sample with value `0.0`.
+
+    Returns:
+       Metric that aggregates `agg` over the `key` field.
+
+    Raises:
+       ValueError: If a sample's `score.value` is not a dict, or if `key` is
+          missing from a sample's score and `on_missing="error"`.
+    """
+
+    def aggregate_metric(scores: list[SampleScore]) -> Value:
+        agg_protocol = cast(MetricProtocol, agg)
+
+        extracted: list[SampleScore] = []
+        for sample_score in scores:
+            value = sample_score.score.value
+            if not isinstance(value, dict):
+                raise ValueError(
+                    f"Sample {sample_score.sample_id} has non-dict score value "
+                    f"({type(value).__name__}); aggregate(key=...) requires "
+                    "dict-valued Score.value."
+                )
+
+            if key in value:
+                extracted_value: float = to_float(value[key])
+            elif on_missing == "error":
+                raise ValueError(
+                    f"Sample {sample_score.sample_id} score is missing key "
+                    f"'{key}'. Pass on_missing='skip' or on_missing='zero' to "
+                    "aggregate() to allow missing keys."
+                )
+            elif on_missing == "skip":
+                continue
+            else:  # "zero"
+                extracted_value = 0.0
+
+            extracted.append(
+                SampleScore(
+                    score=sample_score.score.model_copy(
+                        update={"value": extracted_value}
+                    ),
+                    sample_id=sample_score.sample_id,
+                    sample_metadata=sample_score.sample_metadata,
+                    scorer=sample_score.scorer,
+                )
+            )
+
+        return agg_protocol(extracted)
+
+    return aggregate_metric
diff --git a/tests/scorer/test_metric.py b/tests/scorer/test_metric.py
index c90b072026..0e8a7a4b9f 100644
--- a/tests/scorer/test_metric.py
+++ b/tests/scorer/test_metric.py
@@ -27,7 +27,7 @@
     SampleScore,
     metric_create,
 )
-from inspect_ai.scorer._metrics import grouped
+from inspect_ai.scorer._metrics import aggregate, grouped
 from inspect_ai.scorer._metrics.std import stderr
 from inspect_ai.scorer._target import Target
 from inspect_ai.solver._task_state import TaskState
@@ -638,3 +638,93 @@ def test_dict_metric_all_samples_unscored():
         assert r.scored_samples == 0
         assert r.unscored_samples == 3
         assert math.isnan(r.metrics["mean"].value)
+
+
+def test_aggregate_mean():
+    metric = aggregate("element_acc", agg=mean())
+    result = metric(
+        [
+            SampleScore(score=Score(value={"element_acc": 1, "action_f1": 0.5})),
+            SampleScore(score=Score(value={"element_acc": 1, "action_f1": 0.6})),
+            SampleScore(score=Score(value={"element_acc": 4, "action_f1": 0.7})),
+        ]
+    )
+    assert result == 2.0
+
+
+def test_aggregate_selects_key():
+    # Different keys over the same scores should give different aggregates.
+    scores = [
+        SampleScore(score=Score(value={"a": 1, "b": 10})),
+        SampleScore(score=Score(value={"a": 3, "b": 30})),
+    ]
+    assert aggregate("a", agg=mean())(scores) == 2.0
+    assert aggregate("b", agg=mean())(scores) == 20.0
+
+
+def test_aggregate_stderr_composes():
+    # stderr() should work as the inner aggregator.
+    se = aggregate("x", agg=stderr())(
+        [SampleScore(score=Score(value={"x": i, "y": -i})) for i in range(20)]
+    )
+    expected = stderr()([SampleScore(score=Score(value=i)) for i in range(20)])
+    assert se == expected
+
+
+def test_aggregate_missing_key_error_by_default():
+    metric = aggregate("element_acc", agg=mean())
+    with pytest.raises(ValueError, match="missing key"):
+        metric(
+            [
+                SampleScore(score=Score(value={"element_acc": 1})),
+                SampleScore(score=Score(value={"action_f1": 0.5})),
+            ]
+        )
+
+
+def test_aggregate_missing_key_skip():
+    metric = aggregate("element_acc", agg=mean(), on_missing="skip")
+    result = metric(
+        [
+            SampleScore(score=Score(value={"element_acc": 1})),
+            SampleScore(score=Score(value={"action_f1": 0.5})),  # skipped
+            SampleScore(score=Score(value={"element_acc": 3})),
+        ]
+    )
+    assert result == 2.0
+
+
+def test_aggregate_missing_key_zero():
+    metric = aggregate("element_acc", agg=mean(), on_missing="zero")
+    result = metric(
+        [
+            SampleScore(score=Score(value={"element_acc": 3})),
+            SampleScore(score=Score(value={"action_f1": 0.5})),  # treated as 0.0
+            SampleScore(score=Score(value={"element_acc": 3})),
+        ]
+    )
+    assert result == 2.0
+
+
+def test_aggregate_non_dict_value_raises():
+    metric = aggregate("element_acc", agg=mean())
+    with pytest.raises(ValueError, match="non-dict"):
+        metric(
+            [
+                SampleScore(score=Score(value={"element_acc": 1})),
+                SampleScore(score=Score(value=1.0)),  # scalar, not a dict
+            ]
+        )
+
+
+def test_aggregate_to_float_applied():
+    # value_to_float maps "C"->1.0 and "I"->0.0; aggregate should respect it.
+    result = aggregate("verdict", agg=mean())(
+        [
+            SampleScore(score=Score(value={"verdict": "C"})),
+            SampleScore(score=Score(value={"verdict": "I"})),
+            SampleScore(score=Score(value={"verdict": "C"})),
+            SampleScore(score=Score(value={"verdict": "C"})),
+        ]
+    )
+    assert result == 0.75

From 2999740fe38efd74f0aa541d72e086c7236054e1 Mon Sep 17 00:00:00 2001
From: antnewman <antjsnewman@outlook.com>
Date: Fri, 8 May 2026 15:12:08 +0100
Subject: [PATCH 2/7] docs(scorer): clarify aggregate skip-denominator and
 None-value semantics

Address review feedback on #3850:

- Note that `on_missing="skip"` changes the sample count seen by the
  inner aggregator, so denominator-sensitive metrics (`stderr`, `mean`,
  `std`, `var`) can report different values purely from differing
  missing-key rates rather than underlying variance.
- Document that a present-but-`None` key is not treated as missing; it
  is passed through to `to_float`, which with the default
  `value_to_float()` warns and returns `0.0`.

No behaviour change.
---
 src/inspect_ai/scorer/_metrics/aggregate.py | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/src/inspect_ai/scorer/_metrics/aggregate.py b/src/inspect_ai/scorer/_metrics/aggregate.py
index dede48d257..3cd84e0e09 100644
--- a/src/inspect_ai/scorer/_metrics/aggregate.py
+++ b/src/inspect_ai/scorer/_metrics/aggregate.py
@@ -29,6 +29,13 @@ def aggregate(
     `SampleScore`s into `agg`, so any standard metric (`mean`, `stderr`,
     `std`, `accuracy`, ...) can be applied per key.
 
+    `on_missing="skip"` reduces the number of samples seen by `agg`, which
+    changes the result of any aggregator that depends on sample count
+    (e.g. `stderr`, `mean`, `std`, `var`). Two evals run with the same
+    scorer can therefore report different stderrs purely because the rate
+    of missing keys differed, not because of any difference in the
+    underlying variance. Prefer `"zero"` if you want a constant denominator.
+
     Args:
        key: Field to extract from each sample's dict-valued `Score.value`.
        agg: Metric to apply to the extracted values.
@@ -44,6 +51,11 @@ def aggregate(
           - `"skip"`: exclude the sample from `agg`.
           - `"zero"`: include the sample with value `0.0`.
 
+          A key that is *present* with a `None` value is not treated as
+          missing — it is passed straight through to `to_float`. With the
+          default `value_to_float()`, `None` falls through to a warning and
+          returns `0.0`.
+
     Returns:
        Metric that aggregates `agg` over the `key` field.
 

From 64eaff21ee6ed9f5ef97da4921c5cf607df58cf2 Mon Sep 17 00:00:00 2001
From: antnewman <antjsnewman@outlook.com>
Date: Fri, 15 May 2026 08:23:13 +0100
Subject: [PATCH 3/7] docs(changelog): add aggregate() migration note for
 mean_of users

Address @ppcvote's suggestion on #3735: surface the semantic
difference vs `inspect_evals.utils.mean_of` (which treats present-but-
None as missing) so a straight swap doesn't silently change behaviour.
No code change.
---
 CHANGELOG.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index c0f964bb89..f7cf9cc1fe 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,6 @@
 ## Unreleased
 
+- Scoring: Add `aggregate(key, agg=...)` metric factory that applies any standard metric (`mean`, `stderr`, `std`, `accuracy`, …) to a single field of a dict-valued `Score.value`. Migration note for users coming from `inspect_evals.utils.mean_of`: a present-but-`None` key is *not* routed through `on_missing` — it is passed through to `to_float` (which under the default `value_to_float()` warns and returns `0.0`).
 - OpenAI: Add GPT 5.5 as computer use model and exclude 'chat' and 'instant' models from computer use.
 - VLLM: Preserve dotted vLLM server arg keys.
 - SageMaker: Add `prompt_logprobs` support in chat mode via `GenerateConfig`, parse prompt logprobs from completion mode responses, enabling `perplexity()` and `target_perplexity()` scorers end-to-end.

From 3e44527f8db37e1ba35fca91c5a7527aefbdb0e1 Mon Sep 17 00:00:00 2001
From: antnewman <antjsnewman@outlook.com>
Date: Sun, 31 May 2026 09:41:57 +0100
Subject: [PATCH 4/7] fix(scorer): aggregate() treats present-but-None as
 missing + returns NaN on all-skip
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Address @dragonstyle's review of #3850:

- Present-but-None values are now routed through `on_missing` (matching
  inspect_evals.utils.metrics.mean_of). Previously they fell through to
  to_float(None), which under the default value_to_float() warns and
  returns 0.0 — silently conflating "no signal" with "scored 0".
- When `on_missing="skip"` filters every sample, aggregate() now returns
  NaN rather than calling agg([]) (which most built-in metrics raise on).
  Mirrors the Score.unscored() / NaN sentinel used elsewhere in the
  framework.

Adds 4 tests covering the None-handling on each on_missing mode and the
all-skipped → NaN case. CHANGELOG updated to remove the now-obsolete
migration note about the semantic difference vs mean_of (which no longer
exists).
---
 CHANGELOG.md                                |  2 +-
 src/inspect_ai/scorer/_metrics/aggregate.py | 48 +++++++++++++-------
 tests/scorer/test_metric.py                 | 50 ++++++++++++++++++++-
 3 files changed, 83 insertions(+), 17 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index f7cf9cc1fe..ce9526d42b 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,6 +1,6 @@
 ## Unreleased
 
-- Scoring: Add `aggregate(key, agg=...)` metric factory that applies any standard metric (`mean`, `stderr`, `std`, `accuracy`, …) to a single field of a dict-valued `Score.value`. Migration note for users coming from `inspect_evals.utils.mean_of`: a present-but-`None` key is *not* routed through `on_missing` — it is passed through to `to_float` (which under the default `value_to_float()` warns and returns `0.0`).
+- Scoring: Add `aggregate(key, agg=...)` metric factory that applies any standard metric (`mean`, `stderr`, `std`, `accuracy`, …) to a single field of a dict-valued `Score.value`. Behaviour matches `inspect_evals.utils.metrics.mean_of`: both missing keys and present-but-`None` values are routed through `on_missing`. When `on_missing="skip"` filters every sample, returns `NaN`.
 - OpenAI: Add GPT 5.5 as computer use model and exclude 'chat' and 'instant' models from computer use.
 - VLLM: Preserve dotted vLLM server arg keys.
 - SageMaker: Add `prompt_logprobs` support in chat mode via `GenerateConfig`, parse prompt logprobs from completion mode responses, enabling `perplexity()` and `target_perplexity()` scorers end-to-end.
diff --git a/src/inspect_ai/scorer/_metrics/aggregate.py b/src/inspect_ai/scorer/_metrics/aggregate.py
index 3cd84e0e09..ec3a2eb5e3 100644
--- a/src/inspect_ai/scorer/_metrics/aggregate.py
+++ b/src/inspect_ai/scorer/_metrics/aggregate.py
@@ -1,3 +1,4 @@
+import math
 from logging import getLogger
 from typing import Literal, cast
 
@@ -29,6 +30,11 @@ def aggregate(
     `SampleScore`s into `agg`, so any standard metric (`mean`, `stderr`,
     `std`, `accuracy`, ...) can be applied per key.
 
+    A missing key (either `key not in value` or `value[key] is None`) is
+    routed through `on_missing`. This matches the convention used by
+    `inspect_evals.utils.metrics.mean_of`, so a `mean_of` → `aggregate` swap
+    preserves behaviour.
+
     `on_missing="skip"` reduces the number of samples seen by `agg`, which
     changes the result of any aggregator that depends on sample count
     (e.g. `stderr`, `mean`, `std`, `var`). Two evals run with the same
@@ -36,6 +42,11 @@ def aggregate(
     of missing keys differed, not because of any difference in the
     underlying variance. Prefer `"zero"` if you want a constant denominator.
 
+    If every sample is filtered out by `on_missing="skip"`, the aggregator
+    returns `NaN` rather than calling `agg([])` (which most built-in metrics
+    would raise on). This matches the `Score.unscored()` / NaN sentinel used
+    elsewhere in the framework.
+
     Args:
        key: Field to extract from each sample's dict-valued `Score.value`.
        agg: Metric to apply to the extracted values.
@@ -45,23 +56,20 @@ def aggregate(
           values to float directly, and prints a warning and returns 0 if the
           Value is a complex object (list or dict).
        on_missing: How to handle samples whose `score.value` does not contain
-          `key`:
+          `key`, or contains `key` with a `None` value:
 
           - `"error"` (default): raise `ValueError`.
-          - `"skip"`: exclude the sample from `agg`.
+          - `"skip"`: exclude the sample from `agg`. Returns `NaN` if every
+            sample is skipped.
           - `"zero"`: include the sample with value `0.0`.
 
-          A key that is *present* with a `None` value is not treated as
-          missing — it is passed straight through to `to_float`. With the
-          default `value_to_float()`, `None` falls through to a warning and
-          returns `0.0`.
-
     Returns:
-       Metric that aggregates `agg` over the `key` field.
+       Metric that aggregates `agg` over the `key` field. `NaN` if `skip`
+       filters every sample.
 
     Raises:
-       ValueError: If a sample's `score.value` is not a dict, or if `key` is
-          missing from a sample's score and `on_missing="error"`.
+       ValueError: If a sample's `score.value` is not a dict, or if `key`
+          is missing (or `None`) and `on_missing="error"`.
     """
 
     def aggregate_metric(scores: list[SampleScore]) -> Value:
@@ -77,13 +85,17 @@ def aggregate_metric(scores: list[SampleScore]) -> Value:
                     "dict-valued Score.value."
                 )
 
-            if key in value:
-                extracted_value: float = to_float(value[key])
+            # Treat both "key absent" and "key present but None" as missing,
+            # matching inspect_evals.utils.metrics.mean_of.
+            raw = value.get(key)
+            if key in value and raw is not None:
+                extracted_value: float = to_float(raw)
             elif on_missing == "error":
+                reason = "missing" if key not in value else "None"
                 raise ValueError(
-                    f"Sample {sample_score.sample_id} score is missing key "
-                    f"'{key}'. Pass on_missing='skip' or on_missing='zero' to "
-                    "aggregate() to allow missing keys."
+                    f"Sample {sample_score.sample_id} score key '{key}' is "
+                    f"{reason}. Pass on_missing='skip' or on_missing='zero' "
+                    "to aggregate() to allow missing keys."
                 )
             elif on_missing == "skip":
                 continue
@@ -101,6 +113,12 @@ def aggregate_metric(scores: list[SampleScore]) -> Value:
                 )
             )
 
+        # All samples filtered: return NaN rather than call agg([]) (which
+        # most built-in metrics raise on). Mirrors Score.unscored()'s NaN
+        # sentinel.
+        if not extracted:
+            return math.nan
+
         return agg_protocol(extracted)
 
     return aggregate_metric
diff --git a/tests/scorer/test_metric.py b/tests/scorer/test_metric.py
index 0e8a7a4b9f..4b88a1c58f 100644
--- a/tests/scorer/test_metric.py
+++ b/tests/scorer/test_metric.py
@@ -1,3 +1,4 @@
+import math
 from typing import Any, Callable, cast
 
 import pytest
@@ -673,7 +674,7 @@ def test_aggregate_stderr_composes():
 
 def test_aggregate_missing_key_error_by_default():
     metric = aggregate("element_acc", agg=mean())
-    with pytest.raises(ValueError, match="missing key"):
+    with pytest.raises(ValueError, match="is missing"):
         metric(
             [
                 SampleScore(score=Score(value={"element_acc": 1})),
@@ -728,3 +729,50 @@ def test_aggregate_to_float_applied():
         ]
     )
     assert result == 0.75
+
+
+def test_aggregate_none_value_treated_as_missing_error():
+    # A present-but-None value is treated the same as a missing key
+    # (matches inspect_evals.utils.metrics.mean_of).
+    metric = aggregate("element_acc", agg=mean())
+    with pytest.raises(ValueError, match="is None"):
+        metric(
+            [
+                SampleScore(score=Score(value={"element_acc": 1})),
+                SampleScore(score=Score(value={"element_acc": None})),
+            ]
+        )
+
+
+def test_aggregate_none_value_treated_as_missing_skip():
+    result = aggregate("element_acc", agg=mean(), on_missing="skip")(
+        [
+            SampleScore(score=Score(value={"element_acc": 1})),
+            SampleScore(score=Score(value={"element_acc": None})),  # skipped
+            SampleScore(score=Score(value={"element_acc": 3})),
+        ]
+    )
+    assert result == 2.0
+
+
+def test_aggregate_none_value_treated_as_missing_zero():
+    result = aggregate("element_acc", agg=mean(), on_missing="zero")(
+        [
+            SampleScore(score=Score(value={"element_acc": 3})),
+            SampleScore(score=Score(value={"element_acc": None})),  # treated as 0.0
+            SampleScore(score=Score(value={"element_acc": 3})),
+        ]
+    )
+    assert result == 2.0
+
+
+def test_aggregate_all_skipped_returns_nan():
+    # When skip filters every sample, return NaN rather than calling agg([])
+    # (which most built-in metrics raise on).
+    result = aggregate("element_acc", agg=mean(), on_missing="skip")(
+        [
+            SampleScore(score=Score(value={"action_f1": 0.5})),
+            SampleScore(score=Score(value={"action_f1": 0.6})),
+        ]
+    )
+    assert isinstance(result, float) and math.isnan(result)

From 2c3233e352e55899e3b1ebc3c0b9e456805b9b84 Mon Sep 17 00:00:00 2001
From: antnewman <antjsnewman@outlook.com>
Date: Tue, 2 Jun 2026 09:16:44 +0100
Subject: [PATCH 5/7] fix(scorer): skip per-key NaN and validate on_missing in
 aggregate()
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Address two of dragonstyle's review findings on #3850:

- P1: a per-key NaN value (e.g. {"x": NaN}) was converted and passed to
  the inner metric as a scored value, dragging the whole aggregate to
  NaN. The framework's dict-metric expansion instead skips NaN-valued
  keys as unscored. aggregate() now does the same — NaN values are
  skipped regardless of on_missing (which governs *missing keys*, a
  distinct concept). aggregate("x", mean()) over [{x:1},{x:NaN},{x:3}]
  now returns 2.0 instead of NaN.

- P3: an invalid on_missing (e.g. "skpi") silently behaved like "zero",
  and only when a key happened to be missing. It's now validated at
  construction time and raises ValueError immediately.

The third finding (P2 — aggregate's to_float bypassing a custom
converter on the inner metric) is a deliberate API-design choice left
for the maintainer to direct; raised separately in a PR comment.

Adds tests: NaN skipped (default + on_missing="zero"), all-NaN → NaN,
invalid on_missing raises.
---
 src/inspect_ai/scorer/_metrics/aggregate.py | 31 ++++++++++++---
 tests/scorer/test_metric.py                 | 43 +++++++++++++++++++++
 2 files changed, 69 insertions(+), 5 deletions(-)

diff --git a/src/inspect_ai/scorer/_metrics/aggregate.py b/src/inspect_ai/scorer/_metrics/aggregate.py
index ec3a2eb5e3..1ca096187c 100644
--- a/src/inspect_ai/scorer/_metrics/aggregate.py
+++ b/src/inspect_ai/scorer/_metrics/aggregate.py
@@ -63,14 +63,26 @@ def aggregate(
             sample is skipped.
           - `"zero"`: include the sample with value `0.0`.
 
+    A per-key `NaN` value (e.g. `{"key": NaN}`) is always treated as
+    unscored and skipped, independent of `on_missing` (which governs
+    *missing keys*, a distinct concept). This matches the framework's
+    dict-metric expansion, which skips NaN-valued keys rather than feeding
+    them into the inner metric.
+
     Returns:
-       Metric that aggregates `agg` over the `key` field. `NaN` if `skip`
-       filters every sample.
+       Metric that aggregates `agg` over the `key` field. `NaN` if every
+       sample is skipped (by `on_missing="skip"` and/or NaN-skipping).
 
     Raises:
-       ValueError: If a sample's `score.value` is not a dict, or if `key`
-          is missing (or `None`) and `on_missing="error"`.
+       ValueError: If `on_missing` is not one of "error", "skip", "zero";
+          if a sample's `score.value` is not a dict; or if `key` is missing
+          (or `None`) and `on_missing="error"`.
     """
+    if on_missing not in ("error", "skip", "zero"):
+        raise ValueError(
+            f"aggregate() got invalid on_missing={on_missing!r}; "
+            "expected 'error', 'skip', or 'zero'."
+        )
 
     def aggregate_metric(scores: list[SampleScore]) -> Value:
         agg_protocol = cast(MetricProtocol, agg)
@@ -85,9 +97,18 @@ def aggregate_metric(scores: list[SampleScore]) -> Value:
                     "dict-valued Score.value."
                 )
 
+            raw = value.get(key)
+
+            # A per-key NaN is unscored: skip it regardless of on_missing,
+            # matching the framework's dict-metric expansion (results.py),
+            # which excludes NaN-valued keys from the inner metric rather
+            # than letting them drag the aggregate to NaN. (`isinstance`
+            # excludes a missing key, whose `raw` is None.)
+            if isinstance(raw, float) and math.isnan(raw):
+                continue
+
             # Treat both "key absent" and "key present but None" as missing,
             # matching inspect_evals.utils.metrics.mean_of.
-            raw = value.get(key)
             if key in value and raw is not None:
                 extracted_value: float = to_float(raw)
             elif on_missing == "error":
diff --git a/tests/scorer/test_metric.py b/tests/scorer/test_metric.py
index 4b88a1c58f..e3886e8204 100644
--- a/tests/scorer/test_metric.py
+++ b/tests/scorer/test_metric.py
@@ -776,3 +776,46 @@ def test_aggregate_all_skipped_returns_nan():
         ]
     )
     assert isinstance(result, float) and math.isnan(result)
+
+
+def test_aggregate_skips_nan_values():
+    # A per-key NaN is unscored: skipped regardless of on_missing (which is
+    # at its "error" default here), so the aggregate is the mean of the
+    # non-NaN values rather than NaN. Matches dict-metric expansion.
+    result = aggregate("x", agg=mean())(
+        [
+            SampleScore(score=Score(value={"x": 1})),
+            SampleScore(score=Score(value={"x": float("nan")})),
+            SampleScore(score=Score(value={"x": 3})),
+        ]
+    )
+    assert result == 2.0
+
+
+def test_aggregate_all_nan_returns_nan():
+    result = aggregate("x", agg=mean())(
+        [
+            SampleScore(score=Score(value={"x": float("nan")})),
+            SampleScore(score=Score(value={"x": float("nan")})),
+        ]
+    )
+    assert isinstance(result, float) and math.isnan(result)
+
+
+def test_aggregate_nan_skipped_under_zero():
+    # NaN is distinct from a missing key: even with on_missing="zero" a NaN
+    # value is skipped (not coerced to 0.0), so it doesn't drag the mean down.
+    result = aggregate("x", agg=mean(), on_missing="zero")(
+        [
+            SampleScore(score=Score(value={"x": 2})),
+            SampleScore(score=Score(value={"x": float("nan")})),
+        ]
+    )
+    assert result == 2.0
+
+
+def test_aggregate_invalid_on_missing_raises():
+    # Invalid on_missing must fail at construction, not silently behave like
+    # "zero" only when a key happens to be missing.
+    with pytest.raises(ValueError, match="invalid on_missing"):
+        aggregate("x", agg=mean(), on_missing="skpi")  # type: ignore[arg-type]

From de654bb6370084d89358152c632103b59368260c Mon Sep 17 00:00:00 2001
From: antnewman <antjsnewman@outlook.com>
Date: Tue, 2 Jun 2026 14:18:24 +0100
Subject: [PATCH 6/7] =?UTF-8?q?fix(scorer):=20aggregate()=20to=5Ffloat=20d?=
 =?UTF-8?q?efaults=20to=20None=20(passthrough)=20=E2=80=94=20P2?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Implements @dragonstyle's chosen Option A on #3850. Previously aggregate
always pre-converted extracted values with its own `to_float`
(`value_to_float()` by default), silently bypassing any custom converter
on the inner metric — e.g. accuracy(to_float=value_to_float(correct="pass"))
never saw "pass"/"fail" because aggregate had already mapped them to 0.0.

`to_float` now defaults to None: the raw extracted value passes straight
through to `agg`, so the inner metric's own conversion applies
(accuracy()'s to_float, mean()'s as_float). An explicit `to_float=` is
the escape hatch for feeding non-numeric values (e.g. "C"/"I" string
grades) into metrics like mean() that can't convert them.

- test_aggregate_to_float_applied → test_aggregate_explicit_to_float_applied
  (now passes to_float=value_to_float() explicitly).
- Add test_aggregate_passthrough_respects_inner_to_float, asserting a
  custom value_to_float on accuracy() is honoured (the exact case in the
  review).
---
 CHANGELOG.md                                |  2 +-
 src/inspect_ai/scorer/_metrics/aggregate.py | 23 ++++++++++++-------
 tests/scorer/test_metric.py                 | 25 ++++++++++++++++++---
 3 files changed, 38 insertions(+), 12 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index ce9526d42b..662b53a02e 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,6 +1,6 @@
 ## Unreleased
 
-- Scoring: Add `aggregate(key, agg=...)` metric factory that applies any standard metric (`mean`, `stderr`, `std`, `accuracy`, …) to a single field of a dict-valued `Score.value`. Behaviour matches `inspect_evals.utils.metrics.mean_of`: both missing keys and present-but-`None` values are routed through `on_missing`. When `on_missing="skip"` filters every sample, returns `NaN`.
+- Scoring: Add `aggregate(key, agg=...)` metric factory that applies any standard metric (`mean`, `stderr`, `std`, `accuracy`, …) to a single field of a dict-valued `Score.value`. By default the extracted value passes straight through to `agg` so the inner metric's own conversion applies; pass `to_float=` to pre-convert (e.g. to feed string grades into `mean()`). Missing keys, present-but-`None` values, and per-key `NaN` are handled like the framework's dict-metric expansion (missing/`None` routed through `on_missing`; `NaN` skipped as unscored). Returns `NaN` if every sample is filtered out.
 - OpenAI: Add GPT 5.5 as computer use model and exclude 'chat' and 'instant' models from computer use.
 - VLLM: Preserve dotted vLLM server arg keys.
 - SageMaker: Add `prompt_logprobs` support in chat mode via `GenerateConfig`, parse prompt logprobs from completion mode responses, enabling `perplexity()` and `target_perplexity()` scorers end-to-end.
diff --git a/src/inspect_ai/scorer/_metrics/aggregate.py b/src/inspect_ai/scorer/_metrics/aggregate.py
index 1ca096187c..8007d2e623 100644
--- a/src/inspect_ai/scorer/_metrics/aggregate.py
+++ b/src/inspect_ai/scorer/_metrics/aggregate.py
@@ -9,7 +9,6 @@
     Value,
     ValueToFloat,
     metric,
-    value_to_float,
 )
 
 logger = getLogger(__name__)
@@ -20,7 +19,7 @@ def aggregate(
     key: str,
     agg: Metric,
     *,
-    to_float: ValueToFloat = value_to_float(),
+    to_float: ValueToFloat | None = None,
     on_missing: Literal["error", "skip", "zero"] = "error",
 ) -> Metric:
     """Apply `agg` to a single key extracted from each dict-valued `Score.value`.
@@ -50,11 +49,14 @@ def aggregate(
     Args:
        key: Field to extract from each sample's dict-valued `Score.value`.
        agg: Metric to apply to the extracted values.
-       to_float: Function for mapping the extracted `Value` to a float. The
-          default `value_to_float()` maps CORRECT ("C") to 1.0, INCORRECT ("I")
-          to 0, PARTIAL ("P") to 0.5, and NOANSWER ("N") to 0, casts numeric
-          values to float directly, and prints a warning and returns 0 if the
-          Value is a complex object (list or dict).
+       to_float: Optional function for mapping the extracted `Value` to a
+          float before it reaches `agg`. The default (`None`) passes the raw
+          extracted value straight through, so `agg`'s own conversion applies
+          (e.g. `accuracy()`'s `to_float`, or `mean()`'s `as_float()`). Set
+          this only when `agg` cannot convert the value itself — e.g. to feed
+          string grades ("C"/"I") into `mean()`, which expects numerics. When
+          set, pass `value_to_float()` (or a customised variant) to get the
+          standard CORRECT/INCORRECT/PARTIAL/NOANSWER mapping.
        on_missing: How to handle samples whose `score.value` does not contain
           `key`, or contains `key` with a `None` value:
 
@@ -110,7 +112,12 @@ def aggregate_metric(scores: list[SampleScore]) -> Value:
             # Treat both "key absent" and "key present but None" as missing,
             # matching inspect_evals.utils.metrics.mean_of.
             if key in value and raw is not None:
-                extracted_value: float = to_float(raw)
+                # Pass the raw value through by default so `agg`'s own
+                # converter applies; only pre-convert when an explicit
+                # to_float was supplied.
+                extracted_value: str | int | float | bool = (
+                    to_float(raw) if to_float is not None else raw
+                )
             elif on_missing == "error":
                 reason = "missing" if key not in value else "None"
                 raise ValueError(
diff --git a/tests/scorer/test_metric.py b/tests/scorer/test_metric.py
index e3886e8204..f7d5209d0e 100644
--- a/tests/scorer/test_metric.py
+++ b/tests/scorer/test_metric.py
@@ -20,6 +20,7 @@
     metric,
     scorer,
     std,
+    value_to_float,
     var,
 )
 from inspect_ai.scorer._metric import (
@@ -718,9 +719,10 @@ def test_aggregate_non_dict_value_raises():
         )
 
 
-def test_aggregate_to_float_applied():
-    # value_to_float maps "C"->1.0 and "I"->0.0; aggregate should respect it.
-    result = aggregate("verdict", agg=mean())(
+def test_aggregate_explicit_to_float_applied():
+    # An explicit to_float lets string grades flow into mean() (which expects
+    # numerics). value_to_float maps "C"->1.0 and "I"->0.0.
+    result = aggregate("verdict", agg=mean(), to_float=value_to_float())(
         [
             SampleScore(score=Score(value={"verdict": "C"})),
             SampleScore(score=Score(value={"verdict": "I"})),
@@ -731,6 +733,23 @@ def test_aggregate_to_float_applied():
     assert result == 0.75
 
 
+def test_aggregate_passthrough_respects_inner_to_float():
+    # With the default to_float=None, the raw value passes straight through to
+    # the inner metric, so the inner metric's OWN converter applies. Here a
+    # custom value_to_float on accuracy() maps "pass"->1.0 / "fail"->0.0;
+    # aggregate must not pre-convert and bypass it.
+    inner = accuracy(to_float=value_to_float(correct="pass", incorrect="fail"))
+    result = aggregate("verdict", agg=inner)(
+        [
+            SampleScore(score=Score(value={"verdict": "pass"})),
+            SampleScore(score=Score(value={"verdict": "fail"})),
+            SampleScore(score=Score(value={"verdict": "pass"})),
+            SampleScore(score=Score(value={"verdict": "pass"})),
+        ]
+    )
+    assert result == 0.75
+
+
 def test_aggregate_none_value_treated_as_missing_error():
     # A present-but-None value is treated the same as a missing key
     # (matches inspect_evals.utils.metrics.mean_of).

From 151b5797e2e7e368b1c6bea98fc20e02d329832c Mon Sep 17 00:00:00 2001
From: antnewman <antjsnewman@outlook.com>
Date: Wed, 8 Jul 2026 12:41:09 +0100
Subject: [PATCH 7/7] fix(tests): drop now-unused type-ignore on aggregate
 invalid-on_missing test

The @metric decorator erases the Literal["error","skip","zero"] constraint
on on_missing, so passing "skpi" was never a type error and the
# type: ignore[arg-type] was flagged unused by mypy. Remove it.
---
 tests/scorer/test_metric.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/scorer/test_metric.py b/tests/scorer/test_metric.py
index 41c8f5e187..865c9e9eda 100644
--- a/tests/scorer/test_metric.py
+++ b/tests/scorer/test_metric.py
@@ -910,7 +910,7 @@ def test_aggregate_invalid_on_missing_raises():
     # Invalid on_missing must fail at construction, not silently behave like
     # "zero" only when a key happens to be missing.
     with pytest.raises(ValueError, match="invalid on_missing"):
-        aggregate("x", agg=mean(), on_missing="skpi")  # type: ignore[arg-type]
+        aggregate("x", agg=mean(), on_missing="skpi")
 
 
 def test_metrics_return_zero_for_empty_scores() -> None: