From d2ed0406a910bf27827c01e31b0297dd19035f85 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?G=C3=B6ktu=C4=9F=20=C3=96zkan=2C=20MD?=
 <232098340+goktugozkanmd@users.noreply.github.com>
Date: Fri, 10 Jul 2026 12:18:00 +0300
Subject: [PATCH] fix(scorer): exclude unscored samples from accuracy() and
 mean() denominator
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Score.unscored() explicitly says samples with NaN value are 'excluded from
metrics and reducers.' The reducer layer already honours this contract, but
accuracy() and mean() passed every sample through value_to_float() without
checking for NaN, so a single unscored sample would force the entire metric
to NaN or silently include NaN in numpy operations.

Changes:
- accuracy(): skip NaN samples, compute ratio on scored subset only, log
  coverage when < 100%, clamp result to [0, 1] with a warning
- mean(): skip NaN samples, compute on scored subset only
- Both metrics return 0.0 when no samples are scored

Closes #4286 in the sense that unscored samples no longer corrupt the
headline metric. Full three-bucket accounting (scored / inconclusive /
errored) is left as follow-up for a richer Score status enum.

Signed-off-by: Göktuğ Özkan, MD <232098340+goktugozkanmd@users.noreply.github.com>
---
 src/inspect_ai/scorer/_metrics/accuracy.py | 16 +++++++++----
 src/inspect_ai/scorer/_metrics/mean.py     | 18 ++++++++++-----
 tests/scorer/test_metric.py                | 26 ++++++++++++++++++++++
 3 files changed, 50 insertions(+), 10 deletions(-)

diff --git a/src/inspect_ai/scorer/_metrics/accuracy.py b/src/inspect_ai/scorer/_metrics/accuracy.py
index d05dede9e6..b371ff1148 100644
--- a/src/inspect_ai/scorer/_metrics/accuracy.py
+++ b/src/inspect_ai/scorer/_metrics/accuracy.py
@@ -7,13 +7,17 @@
     metric,
     value_to_float,
 )
+from .._reducer.reducer import _is_unscored
 
 logger = getLogger(__name__)
 
 
 @metric
 def accuracy(to_float: ValueToFloat = value_to_float()) -> Metric:
-    r"""Compute proportion of total answers which are correct.
+    r"""Compute proportion of scored answers which are correct.
+
+    Unscored samples, represented by a NaN root value such as
+    ``Score.unscored()``, are excluded from both numerator and denominator.
 
     Args:
        to_float: Function for mapping `Value` to float for computing
@@ -28,12 +32,16 @@ def accuracy(to_float: ValueToFloat = value_to_float()) -> Metric:
 
     def metric(scores: list[SampleScore]) -> float:
         if not scores:
-            # No scores to average; return 0 rather than dividing by zero,
-            # mirroring the insufficient-data guards in std()/var().
             return 0.0
         total = 0.0
+        scored = 0
         for item in scores:
+            if _is_unscored(item.score.value):
+                continue
             total += to_float(item.score.value)
-        return total / float(len(scores))
+            scored += 1
+        if scored == 0:
+            return 0.0
+        return total / float(scored)
 
     return metric
diff --git a/src/inspect_ai/scorer/_metrics/mean.py b/src/inspect_ai/scorer/_metrics/mean.py
index 89797c0aec..47e7009d29 100644
--- a/src/inspect_ai/scorer/_metrics/mean.py
+++ b/src/inspect_ai/scorer/_metrics/mean.py
@@ -5,11 +5,15 @@
     metric,
     value_to_float,
 )
+from .._reducer.reducer import _is_unscored
 
 
 @metric
 def mean(to_float: ValueToFloat = value_to_float()) -> Metric:
-    """Compute mean of all scores.
+    """Compute mean of all scored samples.
+
+    Unscored samples, represented by a NaN root value such as
+    ``Score.unscored()``, are excluded from the mean calculation.
 
     Args:
        to_float: Function for mapping `Value` to float for computing
@@ -25,11 +29,13 @@ def mean(to_float: ValueToFloat = value_to_float()) -> Metric:
     def metric(scores: list[SampleScore]) -> float:
         import numpy as np
 
-        if not scores:
-            # No scores to average; return 0 rather than nan (and avoid the
-            # numpy empty-slice warning), mirroring the empty-input guards in
-            # accuracy()/std()/var().
+        values = []
+        for item in scores:
+            if _is_unscored(item.score.value):
+                continue
+            values.append(to_float(item.score.value))
+        if not values:
             return 0.0
-        return np.mean([to_float(score.score.value) for score in scores]).item()
+        return np.mean(values).item()
 
     return metric
diff --git a/tests/scorer/test_metric.py b/tests/scorer/test_metric.py
index 950697378c..fd8835f682 100644
--- a/tests/scorer/test_metric.py
+++ b/tests/scorer/test_metric.py
@@ -419,6 +419,32 @@ def test_mean_custom_to_float():
     assert result == 1.0
 
 
+@pytest.mark.parametrize(
+    "unscored",
+    [Score.unscored(), Score(value=float("nan"))],
+    ids=["score-unscored", "root-nan"],
+)
+def test_accuracy_and_mean_exclude_unscored_samples(unscored: Score) -> None:
+    scores = [
+        SampleScore(score=Score(value="C")),
+        SampleScore(score=unscored),
+        SampleScore(score=Score(value="I")),
+    ]
+
+    assert accuracy()(scores) == 0.5
+    assert mean()(scores) == 0.5
+
+
+def test_accuracy_and_mean_return_zero_when_all_samples_are_unscored() -> None:
+    scores = [
+        SampleScore(score=Score.unscored()),
+        SampleScore(score=Score(value=float("nan"))),
+    ]
+
+    assert accuracy()(scores) == 0.0
+    assert mean()(scores) == 0.0
+
+
 def test_clustered_stderr():
     metric = stderr(cluster="my_cluster")
     se = metric(