From 7cf102abf101e2f137f008fad7bef7e610f0993d Mon Sep 17 00:00:00 2001 From: vladmesh Date: Tue, 26 May 2026 12:19:42 +0300 Subject: [PATCH 1/4] fix(scorer): return NOANSWER (not INCORRECT) when model_graded grade parser fails When the judge's output does not match DEFAULT_GRADE_PATTERN (or a user-supplied grade_pattern), model_graded_qa / model_graded_fact previously returned value=INCORRECT in the else-branch of _model.py. "Couldn't extract a verdict" and "verdict is incorrect" then became indistinguishable in the score distribution, silently inflating the INCORRECT count in aggregate metrics when judges flubbed the grade prefix word (e.g. GRID:, GRAND, ANSWER:, **Answer: C**). This matches the convention already used by pattern() in _pattern.py: when its regex fails to match, it returns NOANSWER rather than INCORRECT. value_to_float() maps both NOANSWER and INCORRECT to 0.0, so accuracy aggregations are unchanged; the difference is that judge-parse failures are now distinguishable from real INCORRECTs. Closes #4026. Co-authored-by: Yana Bodrasheva Co-authored-by: Elena Arseneva Co-authored-by: Anton Gornakov --- CHANGELOG.md | 1 + src/inspect_ai/scorer/_model.py | 4 +- tests/scorer/test_model_graded.py | 83 +++++++++++++++++++++++++++++-- 3 files changed, 81 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 076c052ece..4f7533e400 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ - Deep Agent: `agent` dispatch tool now ships with a tool-call viewer. - OpenAI: Support OPENAI_SAFETY_IDENTIFIER environment variable. - Scoring: Set `Score.answer` on model_graded parse failure. +- Scoring: `model_graded_qa` / `model_graded_fact` now return `NOANSWER` (not `INCORRECT`) when the judge's output does not match the grade regex, so judge-parse failures no longer silently inflate the `INCORRECT` count in aggregate metrics. - Task Display: Add column to indicate the agent/solver for each task. - Transcript: Various improvements to transcript event subscriber delivery. - Transcript: Complete samples from buffer history database rather than in-memory list. diff --git a/src/inspect_ai/scorer/_model.py b/src/inspect_ai/scorer/_model.py index 6d656873e9..3f38c99b3c 100644 --- a/src/inspect_ai/scorer/_model.py +++ b/src/inspect_ai/scorer/_model.py @@ -18,7 +18,7 @@ from inspect_ai.solver._task_state import TaskState from inspect_ai.util import resource -from ._metric import INCORRECT, Score +from ._metric import NOANSWER, Score from ._metrics import accuracy, stderr from ._multi import multi_scorer from ._scorer import Scorer, scorer @@ -223,7 +223,7 @@ async def score(state: TaskState, target: Target) -> Score: ) else: return Score( - value=INCORRECT, + value=NOANSWER, answer=state.output.completion, explanation="Grade not found in model output: " + f"{result.completion}", diff --git a/tests/scorer/test_model_graded.py b/tests/scorer/test_model_graded.py index b56e0a16d2..c33b9450f2 100644 --- a/tests/scorer/test_model_graded.py +++ b/tests/scorer/test_model_graded.py @@ -12,7 +12,14 @@ from inspect_ai.model import ChatMessageAssistant, ChatMessageUser from inspect_ai.model._model import get_model from inspect_ai.model._model_output import ModelOutput -from inspect_ai.scorer import INCORRECT, model_graded_fact, model_graded_qa +from inspect_ai.scorer import ( + CORRECT, + INCORRECT, + NOANSWER, + PARTIAL, + model_graded_fact, + model_graded_qa, +) from inspect_ai.scorer._model import ( DEFAULT_GRADE_PATTERN, neutralize_structural_delimiters, @@ -150,9 +157,9 @@ def test_model_role_precedence_for_model_graded_scorer( def test_model_graded_answer_set_on_grade_parse_failure(): # issue #4025: when the grader output has no parseable GRADE: token the scorer - # falls into the parse-failure branch. value is INCORRECT, but the answer field - # must still carry the model's completion (matching the grade-found branch) so - # the log viewer doesn't show an empty answer. + # falls into the parse-failure branch. value is NOANSWER (see #4048), but the + # answer field must still carry the model's completion (matching the grade-found + # branch) so the log viewer doesn't show an empty answer. subject_answer = "The capital of France is Paris." grader_model = get_model( "mockllm/model", @@ -176,7 +183,7 @@ def test_model_graded_answer_set_on_grade_parse_failure(): assert log.samples score = log.samples[0].scores["model_graded_fact"] - assert score.value == INCORRECT + assert score.value == NOANSWER assert score.answer == subject_answer @@ -269,6 +276,72 @@ def test_default_grade_pattern_extraction(grader_output: str, expected: str) -> assert match.group(1) == expected +@pytest.mark.parametrize( + "grader_output", + [ + pytest.param("GRID: C", id="typo_grid"), + pytest.param("ANSWER: C", id="wrong_word_answer"), + pytest.param("**Answer: C**", id="markdown_decorated"), + pytest.param("The submission is correct.", id="no_grade_marker_at_all"), + ], +) +def test_grade_parse_failure_yields_noanswer(grader_output: str) -> None: + # When the grade regex doesn't match, the scorer should record NOANSWER + # rather than INCORRECT, so a parse failure is not counted as a wrong + # answer in aggregate metrics. + grader = get_model( + "mockllm/model", + custom_outputs=[ + ModelOutput.from_content("mockllm/model", [ContentText(text=grader_output)]) + ], + ) + task = Task( + dataset=[Sample(input="What is 1 + 1?", target="2")], + scorer=model_graded_fact(model=grader), + ) + log = eval(task, model="mockllm/model")[0] + assert log.samples + scores = log.samples[0].scores + assert scores is not None + score = scores["model_graded_fact"] + assert score.value == NOANSWER, ( + f"expected NOANSWER for unparseable grade {grader_output!r}, " + f"got {score.value!r}" + ) + + +@pytest.mark.parametrize( + "grader_output, expected", + [ + pytest.param("GRADE: C", CORRECT, id="correct"), + pytest.param("GRADE: I", INCORRECT, id="incorrect"), + pytest.param("GRADE: P", PARTIAL, id="partial"), + ], +) +def test_matched_grade_resolves_to_value(grader_output: str, expected: str) -> None: + # Companion to the parse-failure cases: a parseable grade must still resolve + # to its own value (not be swept into NOANSWER), guarding the matched-grade + # branch against drift. + grader = get_model( + "mockllm/model", + custom_outputs=[ + ModelOutput.from_content("mockllm/model", [ContentText(text=grader_output)]) + ], + ) + task = Task( + dataset=[Sample(input="What is 1 + 1?", target="2")], + scorer=model_graded_fact(model=grader), + ) + log = eval(task, model="mockllm/model")[0] + assert log.samples + scores = log.samples[0].scores + assert scores is not None + score = scores["model_graded_fact"] + assert score.value == expected, ( + f"expected {expected!r} for grade {grader_output!r}, got {score.value!r}" + ) + + def test_neutralize_structural_delimiters_is_idempotent() -> None: for text in [ "[END DATA]", From 40413c2dcd338465d560148dd2fee12aeee2cdbf Mon Sep 17 00:00:00 2001 From: vladmesh Date: Wed, 24 Jun 2026 14:57:01 +0300 Subject: [PATCH 2/4] scorer: mark model_graded parse failure unscored with reason tag Use Score.unscored() instead of NOANSWER on grade-regex miss: a judge parse failure is an instrumentation failure, not a model-under-test outcome, so it leaves the rate and is surfaced via unscored_samples. Tag unscored_reason=grade_parse_failure to keep it distinguishable. --- CHANGELOG.md | 2 +- src/inspect_ai/scorer/_model.py | 8 ++++---- tests/scorer/test_model_graded.py | 24 ++++++++---------------- 3 files changed, 13 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 08d58ce480..5b7c51a92e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ - Deep Agent: `agent` dispatch tool now ships with a tool-call viewer. - OpenAI: Support OPENAI_SAFETY_IDENTIFIER environment variable. - Scoring: Set `Score.answer` on model_graded parse failure. -- Scoring: `model_graded_qa` / `model_graded_fact` now return `NOANSWER` (not `INCORRECT`) when the judge's output does not match the grade regex, so judge-parse failures no longer silently inflate the `INCORRECT` count in aggregate metrics. +- Scoring: `model_graded_qa` / `model_graded_fact` now mark a sample unscored (instead of `INCORRECT`) when the judge's output does not match the grade regex, tagging `unscored_reason="grade_parse_failure"` so judge-parse failures leave the rate and stay visible rather than inflating the `INCORRECT` count. - Task Display: Add column to indicate the agent/solver for each task. - Transcript: Various improvements to transcript event subscriber delivery. - Transcript: Complete samples from buffer history database rather than in-memory list. diff --git a/src/inspect_ai/scorer/_model.py b/src/inspect_ai/scorer/_model.py index 3f38c99b3c..4b5ff59f9e 100644 --- a/src/inspect_ai/scorer/_model.py +++ b/src/inspect_ai/scorer/_model.py @@ -18,7 +18,7 @@ from inspect_ai.solver._task_state import TaskState from inspect_ai.util import resource -from ._metric import NOANSWER, Score +from ._metric import Score from ._metrics import accuracy, stderr from ._multi import multi_scorer from ._scorer import Scorer, scorer @@ -222,16 +222,16 @@ async def score(state: TaskState, target: Target) -> Score: ), ) else: - return Score( - value=NOANSWER, + return Score.unscored( answer=state.output.completion, explanation="Grade not found in model output: " + f"{result.completion}", metadata=dict( + unscored_reason="grade_parse_failure", grading=[ scoring_prompt, result.message, - ] + ], ), ) diff --git a/tests/scorer/test_model_graded.py b/tests/scorer/test_model_graded.py index c33b9450f2..77ca4402e1 100644 --- a/tests/scorer/test_model_graded.py +++ b/tests/scorer/test_model_graded.py @@ -1,3 +1,4 @@ +import math import os import re from typing import Any, Callable @@ -15,7 +16,6 @@ from inspect_ai.scorer import ( CORRECT, INCORRECT, - NOANSWER, PARTIAL, model_graded_fact, model_graded_qa, @@ -156,10 +156,7 @@ def test_model_role_precedence_for_model_graded_scorer( def test_model_graded_answer_set_on_grade_parse_failure(): - # issue #4025: when the grader output has no parseable GRADE: token the scorer - # falls into the parse-failure branch. value is NOANSWER (see #4048), but the - # answer field must still carry the model's completion (matching the grade-found - # branch) so the log viewer doesn't show an empty answer. + # #4025: parse failure is unscored, but answer must still carry the completion. subject_answer = "The capital of France is Paris." grader_model = get_model( "mockllm/model", @@ -183,7 +180,7 @@ def test_model_graded_answer_set_on_grade_parse_failure(): assert log.samples score = log.samples[0].scores["model_graded_fact"] - assert score.value == NOANSWER + assert isinstance(score.value, float) and math.isnan(score.value) assert score.answer == subject_answer @@ -285,10 +282,7 @@ def test_default_grade_pattern_extraction(grader_output: str, expected: str) -> pytest.param("The submission is correct.", id="no_grade_marker_at_all"), ], ) -def test_grade_parse_failure_yields_noanswer(grader_output: str) -> None: - # When the grade regex doesn't match, the scorer should record NOANSWER - # rather than INCORRECT, so a parse failure is not counted as a wrong - # answer in aggregate metrics. +def test_grade_parse_failure_is_unscored(grader_output: str) -> None: grader = get_model( "mockllm/model", custom_outputs=[ @@ -304,10 +298,10 @@ def test_grade_parse_failure_yields_noanswer(grader_output: str) -> None: scores = log.samples[0].scores assert scores is not None score = scores["model_graded_fact"] - assert score.value == NOANSWER, ( - f"expected NOANSWER for unparseable grade {grader_output!r}, " - f"got {score.value!r}" + assert isinstance(score.value, float) and math.isnan(score.value), ( + f"expected unscored (NaN) for {grader_output!r}, got {score.value!r}" ) + assert score.metadata["unscored_reason"] == "grade_parse_failure" @pytest.mark.parametrize( @@ -319,9 +313,7 @@ def test_grade_parse_failure_yields_noanswer(grader_output: str) -> None: ], ) def test_matched_grade_resolves_to_value(grader_output: str, expected: str) -> None: - # Companion to the parse-failure cases: a parseable grade must still resolve - # to its own value (not be swept into NOANSWER), guarding the matched-grade - # branch against drift. + # A parseable grade must resolve to its own value, not get swept into unscored. grader = get_model( "mockllm/model", custom_outputs=[ From 9dc87c90fef8fe6d77f4320a1df9cf3fecd15f93 Mon Sep 17 00:00:00 2001 From: vladmesh <16962535+vladmesh@users.noreply.github.com> Date: Sun, 5 Jul 2026 15:56:04 +0300 Subject: [PATCH 3/4] changelog: move grade_parse_failure entry back under Unreleased after upstream merge --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8d578aaf3a..673216d82c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ - Agent Bridge: Sandbox model proxy now returns an HTTP 400 error for malformed requests (missing/empty `model`, missing `messages`/`input`, or a non-object body) instead of crashing the sample. (#4187) - Agent Bridge: Sandbox model proxy now streams Anthropic `compaction` and `fallback` content blocks (this handling was present only in the in-repo copy and missing from the injected proxy build). - Agent Bridge: Sandbox model proxy now forwards a failed generation to the proxied agent as a provider-dialect error response (preserving the original HTTP status where available). +- Scoring: `model_graded_qa` / `model_graded_fact` now mark a sample unscored (instead of `INCORRECT`) when the judge's output does not match the grade regex, tagging `unscored_reason="grade_parse_failure"` so judge-parse failures leave the rate and stay visible rather than inflating the `INCORRECT` count. ## 0.3.244 (01 July 2026) @@ -212,7 +213,6 @@ - OpenAI: Support OPENAI_SAFETY_IDENTIFIER environment variable. - OpenRouter: Always replay `reasoning_content` in addition to `reasoning_details` for Deepseek v4. - Scoring: Set `Score.answer` on model_graded parse failure. -- Scoring: `model_graded_qa` / `model_graded_fact` now mark a sample unscored (instead of `INCORRECT`) when the judge's output does not match the grade regex, tagging `unscored_reason="grade_parse_failure"` so judge-parse failures leave the rate and stay visible rather than inflating the `INCORRECT` count. - Task Display: Add column to indicate the agent/solver for each task. - Transcript: Various improvements to transcript event subscriber delivery. - Transcript: Complete samples from buffer history database rather than in-memory list. From 88cb58708d3e659cba7798225e9d3c95fbe644c0 Mon Sep 17 00:00:00 2001 From: vladmesh <16962535+vladmesh@users.noreply.github.com> Date: Tue, 7 Jul 2026 22:53:04 +0300 Subject: [PATCH 4/4] test(scorer): guard score.metadata is not None before indexing (mypy) --- tests/scorer/test_model_graded.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/scorer/test_model_graded.py b/tests/scorer/test_model_graded.py index 31ef88b3ed..c42041db2b 100644 --- a/tests/scorer/test_model_graded.py +++ b/tests/scorer/test_model_graded.py @@ -323,6 +323,7 @@ def test_grade_parse_failure_is_unscored(grader_output: str) -> None: assert isinstance(score.value, float) and math.isnan(score.value), ( f"expected unscored (NaN) for {grader_output!r}, got {score.value!r}" ) + assert score.metadata is not None assert score.metadata["unscored_reason"] == "grade_parse_failure"