fix(scorer): exclude unscored samples from accuracy() and mean() denominator

src/inspect_ai/scorer/_metrics/accuracy.py

@@ -7,13 +7,17 @@
     metric,
     value_to_float,
 )
+from .._reducer.reducer import _is_unscored
 
 logger = getLogger(__name__)
 
 
 @metric
 def accuracy(to_float: ValueToFloat = value_to_float()) -> Metric:
-    r"""Compute proportion of total answers which are correct.
+    r"""Compute proportion of scored answers which are correct.
+
+    Unscored samples, represented by a NaN root value such as
+    ``Score.unscored()``, are excluded from both numerator and denominator.
 
     Args:
        to_float: Function for mapping `Value` to float for computing
@@ -28,12 +32,16 @@ def accuracy(to_float: ValueToFloat = value_to_float()) -> Metric:
 
     def metric(scores: list[SampleScore]) -> float:
         if not scores:
-            # No scores to average; return 0 rather than dividing by zero,
-            # mirroring the insufficient-data guards in std()/var().
             return 0.0
         total = 0.0
+        scored = 0
         for item in scores:
+            if _is_unscored(item.score.value):
+                continue
             total += to_float(item.score.value)
-        return total / float(len(scores))
+            scored += 1
+        if scored == 0:
+            return 0.0
+        return total / float(scored)
 
     return metric

src/inspect_ai/scorer/_metrics/mean.py

@@ -5,11 +5,15 @@
     metric,
     value_to_float,
 )
+from .._reducer.reducer import _is_unscored
 
 
 @metric
 def mean(to_float: ValueToFloat = value_to_float()) -> Metric:
-    """Compute mean of all scores.
+    """Compute mean of all scored samples.
+
+    Unscored samples, represented by a NaN root value such as
+    ``Score.unscored()``, are excluded from the mean calculation.
 
     Args:
        to_float: Function for mapping `Value` to float for computing
@@ -25,11 +29,13 @@ def mean(to_float: ValueToFloat = value_to_float()) -> Metric:
     def metric(scores: list[SampleScore]) -> float:
         import numpy as np
 
-        if not scores:
-            # No scores to average; return 0 rather than nan (and avoid the
-            # numpy empty-slice warning), mirroring the empty-input guards in
-            # accuracy()/std()/var().
+        values = []
+        for item in scores:
+            if _is_unscored(item.score.value):
+                continue
+            values.append(to_float(item.score.value))
+        if not values:
             return 0.0
-        return np.mean([to_float(score.score.value) for score in scores]).item()
+        return np.mean(values).item()
 
     return metric

tests/scorer/test_metric.py

@@ -419,6 +419,32 @@ def test_mean_custom_to_float():
     assert result == 1.0
 
 
+@pytest.mark.parametrize(
+    "unscored",
+    [Score.unscored(), Score(value=float("nan"))],
+    ids=["score-unscored", "root-nan"],
+)
+def test_accuracy_and_mean_exclude_unscored_samples(unscored: Score) -> None:
+    scores = [
+        SampleScore(score=Score(value="C")),
+        SampleScore(score=unscored),
+        SampleScore(score=Score(value="I")),
+    ]
+
+    assert accuracy()(scores) == 0.5
+    assert mean()(scores) == 0.5
+
+
+def test_accuracy_and_mean_return_zero_when_all_samples_are_unscored() -> None:
+    scores = [
+        SampleScore(score=Score.unscored()),
+        SampleScore(score=Score(value=float("nan"))),
+    ]
+
+    assert accuracy()(scores) == 0.0
+    assert mean()(scores) == 0.0
+
+
 def test_clustered_stderr():
     metric = stderr(cluster="my_cluster")
     se = metric(