From 89d9c8a537a663115b5f6591d5684fbcff238f81 Mon Sep 17 00:00:00 2001 From: Tai An Date: Tue, 14 Jul 2026 03:14:15 -0700 Subject: [PATCH 1/2] fix(scorer): admit None in list-valued Score so NaN survives event logging (#4491) --- src/inspect_ai/scorer/_metric.py | 2 +- tests/scorer/test_metric.py | 23 +++++++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/src/inspect_ai/scorer/_metric.py b/src/inspect_ai/scorer/_metric.py index 3ac4f421b3..f2f3de20f4 100644 --- a/src/inspect_ai/scorer/_metric.py +++ b/src/inspect_ai/scorer/_metric.py @@ -48,7 +48,7 @@ Value = Union[ str | int | float | bool, - Sequence[str | int | float | bool], + Sequence[str | int | float | bool | None], Mapping[str, str | int | float | bool | None], ] """Value provided by a score. diff --git a/tests/scorer/test_metric.py b/tests/scorer/test_metric.py index 950697378c..72c85ac10e 100644 --- a/tests/scorer/test_metric.py +++ b/tests/scorer/test_metric.py @@ -173,6 +173,29 @@ def check_log(log): check_log(log) +def test_list_score_with_nan_logs_without_error() -> None: + # Regression for #4491: a list-valued Score containing NaN must not crash + # sample logging. NaN serializes to JSON null, so the Sequence branch of + # Value has to admit None to survive the event-validation round-trip (the + # Mapping branch already does). Before the fix this eval finished with + # status="error" and no results. + @scorer(metrics=[accuracy()]) + def nan_list_scorer() -> Scorer: + async def score(state: TaskState, target: Target) -> Score: + return Score(value=[float("nan"), 1.0]) + + return score + + task = Task( + dataset=[Sample(input="What is 1 + 1?", target="2")], + scorer=nan_list_scorer(), + ) + + log = eval(tasks=task, model="mockllm/model")[0] + assert log.status == "success" + assert log.results is not None + + def test_alternative_metrics() -> None: # check that we get the alternative metrics def check_log(log): From 6e57b5f73244728bafc12d72ae5734ebf399b682 Mon Sep 17 00:00:00 2001 From: Tai An Date: Tue, 14 Jul 2026 06:11:21 -0700 Subject: [PATCH 2/2] chore(schema): regenerate inspect-openapi.json for nullable list Score value (#4491) --- src/inspect_ai/_view/inspect-openapi.json | 31 +++++++++++++++-------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/src/inspect_ai/_view/inspect-openapi.json b/src/inspect_ai/_view/inspect-openapi.json index eb4faee7d1..7e9739497e 100644 --- a/src/inspect_ai/_view/inspect-openapi.json +++ b/src/inspect_ai/_view/inspect-openapi.json @@ -2,7 +2,7 @@ "components": { "schemas": { "AdaptiveConcurrency": { - "description": "Bounds and tuning for an adaptive concurrency controller.\n\nBasic fields (`min`, `start`, `max`) bound the range the controller will\nscale within. Advanced fields (`cooldown_seconds`, `decrease_factor`,\n`scale_up_percent`) tune the response curve and have sensible defaults\nfor typical evaluation workloads \u2014 see the parallelism docs for guidance.\nAccepts a string shorthand (\"min-max\" or \"min-start-max\") for use in CLI\nflags and config files; advanced fields are Python-only.", + "description": "Bounds and tuning for an adaptive concurrency controller.\n\nBasic fields (`min`, `start`, `max`) bound the range the controller will\nscale within. Advanced fields (`cooldown_seconds`, `decrease_factor`,\n`scale_up_percent`) tune the response curve and have sensible defaults\nfor typical evaluation workloads — see the parallelism docs for guidance.\nAccepts a string shorthand (\"min-max\" or \"min-start-max\") for use in CLI\nflags and config files; advanced fields are Python-only.", "properties": { "cooldown_seconds": { "default": 15.0, @@ -1141,7 +1141,7 @@ }, "CheckpointEvent": { "additionalProperties": true, - "description": "A successful checkpoint commit.\n\nEmitted by the checkpointer immediately after the per-checkpoint\nfile JSON is written \u2014 see working.md \u00a78a. Carries the full\ncheckpoint payload flattened into top-level fields (via multiple\ninheritance from :class:`Checkpoint`), so a consumer of\n``transcript().events`` (or the ``.eval`` log) reads\n``event.checkpoint_id`` / ``event.trigger`` / ``event.host`` etc.\ndirectly \u2014 same data as someone reading\n``/ckpt-NNNNN.json`` from disk.", + "description": "A successful checkpoint commit.\n\nEmitted by the checkpointer immediately after the per-checkpoint\nfile JSON is written — see working.md §8a. Carries the full\ncheckpoint payload flattened into top-level fields (via multiple\ninheritance from :class:`Checkpoint`), so a consumer of\n``transcript().events`` (or the ``.eval`` log) reads\n``event.checkpoint_id`` / ``event.trigger`` / ``event.host`` etc.\ndirectly — same data as someone reading\n``/ckpt-NNNNN.json`` from disk.", "properties": { "checkpoint_id": { "title": "Checkpoint Id", @@ -1278,7 +1278,7 @@ "type": "object" }, "CheckpointSampleConfig": { - "description": "Checkpoint configuration fields that may be set at the sample layer.\n\nThese fields can be specified on ``Sample(checkpoint=...)`` and are\nalso accepted at the task and eval layers (where they participate in\nthe per-field merge \u2014 precedence: eval > sample > task).\n\nThe fields excluded from this base class \u2014 ``checkpoints_location``\nand ``retention`` \u2014 are eval-wide concerns that the sample layer must\nnot influence. They live only on the derived :class:`CheckpointConfig`,\nwhich is the type used at the task and eval layers.", + "description": "Checkpoint configuration fields that may be set at the sample layer.\n\nThese fields can be specified on ``Sample(checkpoint=...)`` and are\nalso accepted at the task and eval layers (where they participate in\nthe per-field merge — precedence: eval > sample > task).\n\nThe fields excluded from this base class — ``checkpoints_location``\nand ``retention`` — are eval-wide concerns that the sample layer must\nnot influence. They live only on the derived :class:`CheckpointConfig`,\nwhich is the type used at the task and eval layers.", "properties": { "max_consecutive_failures": { "anyOf": [ @@ -3889,6 +3889,9 @@ }, { "type": "boolean" + }, + { + "type": "null" } ] }, @@ -5915,7 +5918,7 @@ "type": "object" }, "InterruptEvent": { - "description": "Records that an agent's turn or sample was cut short.\n\nEmitted in three cases:\n\n- ``source=\"user_cancel\"`` \u2014 an ACP client (e.g. an editor or TUI)\n called ``session/cancel`` while a turn was in flight.\n- ``source=\"limit\"`` \u2014 a sample-level limit (tokens, time, cost,\n messages) tripped during execution.\n- ``source=\"system\"`` \u2014 the eval is shutting down for an external\n reason and is cancelling active samples.\n\nThe ``interrupted`` field records what was running at the moment\nthe cancel reached the cancel scope. ``interrupted_tool_call_id``\nand ``interrupted_model_event_id`` give cross-references when\napplicable so downstream consumers can correlate this event with\nthe in-flight ``ToolEvent`` or ``ModelEvent``.", + "description": "Records that an agent's turn or sample was cut short.\n\nEmitted in three cases:\n\n- ``source=\"user_cancel\"`` — an ACP client (e.g. an editor or TUI)\n called ``session/cancel`` while a turn was in flight.\n- ``source=\"limit\"`` — a sample-level limit (tokens, time, cost,\n messages) tripped during execution.\n- ``source=\"system\"`` — the eval is shutting down for an external\n reason and is cancelling active samples.\n\nThe ``interrupted`` field records what was running at the moment\nthe cancel reached the cancel scope. ``interrupted_tool_call_id``\nand ``interrupted_model_event_id`` give cross-references when\napplicable so downstream consumers can correlate this event with\nthe in-flight ``ToolEvent`` or ``ModelEvent``.", "properties": { "event": { "const": "interrupt", @@ -6812,7 +6815,7 @@ "type": "object" }, "Manual": { - "description": "No-op trigger spec.\n\nThe engine's ``tick()`` always returns ``None`` for this spec \u2014\nfires happen only through explicit ``cp.checkpoint()`` calls.", + "description": "No-op trigger spec.\n\nThe engine's ``tick()`` always returns ``None`` for this spec —\nfires happen only through explicit ``cp.checkpoint()`` calls.", "properties": {}, "title": "Manual", "type": "object" @@ -8561,6 +8564,9 @@ }, { "type": "boolean" + }, + { + "type": "null" } ] }, @@ -8600,7 +8606,7 @@ "type": "object" }, "ScoreColorScale": { - "description": "A numeric `score_color_scales` entry with an explicit value range.\n\nBy default the viewer anchors a named palette at the descriptor's\nauto-detected min/max, which is the *observed* range across the\nlog's samples. When the score has a known *conceptual* range \u2014\ne.g. an alignment-judge dimension that's always graded 1..10 \u2014\npin it via `min`/`max` so middling values don't get paint-clamped\nto the extremes when the observed data happens to cluster at one\nend. Either bound can be omitted to fall back to the descriptor's\ndetection for that side.", + "description": "A numeric `score_color_scales` entry with an explicit value range.\n\nBy default the viewer anchors a named palette at the descriptor's\nauto-detected min/max, which is the *observed* range across the\nlog's samples. When the score has a known *conceptual* range —\ne.g. an alignment-judge dimension that's always graded 1..10 —\npin it via `min`/`max` so middling values don't get paint-clamped\nto the extremes when the observed data happens to cluster at one\nend. Either bound can be omitted to fall back to the descriptor's\ndetection for that side.", "properties": { "max": { "anyOf": [ @@ -8728,6 +8734,9 @@ }, { "type": "boolean" + }, + { + "type": "null" } ] }, @@ -9083,7 +9092,7 @@ }, "SnapshotDetails": { "additionalProperties": true, - "description": "Per-backup stats captured in the checkpoint file.\n\nOne per repo (host repo + one per active sandbox repo). Values come\nfrom restic's backup summary \u2014 see :class:`ResticBackupSummary`.", + "description": "Per-backup stats captured in the checkpoint file.\n\nOne per repo (host repo + one per active sandbox repo). Values come\nfrom restic's backup summary — see :class:`ResticBackupSummary`.", "properties": { "additional_files": { "anyOf": [ @@ -9511,7 +9520,7 @@ "type": "object" }, "StopDetails": { - "description": "Additional detail about why a model stopped generating (e.g. a content refusal).\n\n`categories` is the canonical list (always iterable; a single-category provider\nappears as one entry). `category` and `explanation` are a convenience high-level\nsummary derived from the same data \u2014 `category` is the primary category and\n`explanation` is human-readable (synthesized from `categories` when the provider\nsupplies no text). Read either way; both describe the same stop.", + "description": "Additional detail about why a model stopped generating (e.g. a content refusal).\n\n`categories` is the canonical list (always iterable; a single-category provider\nappears as one entry). `category` and `explanation` are a convenience high-level\nsummary derived from the same data — `category` is the primary category and\n`explanation` is human-readable (synthesized from `categories` when the provider\nsupplies no text). Read either way; both describe the same stop.", "properties": { "categories": { "items": { @@ -10082,7 +10091,7 @@ "type": "object" }, "Timeline": { - "description": "A named timeline view over a transcript.\n\nMultiple timelines allow different interpretations of the same event\nstream \u2014 e.g. a default agent-centric view alongside an alternative\ngrouping or filtered view.", + "description": "A named timeline view over a transcript.\n\nMultiple timelines allow different interpretations of the same event\nstream — e.g. a default agent-centric view alongside an alternative\ngrouping or filtered view.", "properties": { "description": { "title": "Description", @@ -10126,7 +10135,7 @@ "type": "object" }, "TimelineSpan": { - "description": "A span of execution \u2014 agent, scorer, tool, or root.", + "description": "A span of execution — agent, scorer, tool, or root.", "properties": { "agent_result": { "anyOf": [ @@ -10812,7 +10821,7 @@ "type": "object" }, "TurnInterval": { - "description": "Fire after every ``every`` agent turns of work.\n\nThe very first ``tick()`` call marks the boundary *before* turn 1\nhas run \u2014 agents place ``cp.tick()`` at the top of their loop, so\nthe opening tick stands between \"no turn yet\" and \"turn 1.\" That\nboundary is informational and doesn't count toward the threshold;\notherwise ``every=1`` would fire an empty checkpoint on the\nopening tick.", + "description": "Fire after every ``every`` agent turns of work.\n\nThe very first ``tick()`` call marks the boundary *before* turn 1\nhas run — agents place ``cp.tick()`` at the top of their loop, so\nthe opening tick stands between \"no turn yet\" and \"turn 1.\" That\nboundary is informational and doesn't count toward the threshold;\notherwise ``every=1`` would fire an empty checkpoint on the\nopening tick.", "properties": { "every": { "title": "Every",