diff --git a/CHANGELOG.md b/CHANGELOG.md index 6fef15204b..dbdab68dff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ - Performance: make `stable_message_ids()` linear per turn. - Performance: Reading `.eval` logs now looks up zip members via a cached O(1) name index instead of an O(members) scan per member, removing quadratic (O(members²)) overhead when loading logs with many samples (e.g. `read_eval_log`, `eval-retry`, `samples_df(full=True)`). - Scoring: Fix edge case where `pattern` `match_all=True` could incorrectly return the target value when no matches were present. +- Scoring: `model_graded_qa` / `model_graded_fact` now mark a sample unscored (instead of `INCORRECT`) when the judge's output does not match the grade regex, tagging `unscored_reason="grade_parse_failure"` so judge-parse failures leave the rate and stay visible rather than inflating the `INCORRECT` count. - Security: Constrain Docker sandbox `read_file()` staging to a generated regular file so container paths cannot copy outside the private host temporary directory. - vLLM: Keep the connection-pool/adaptive-concurrency scope stable across lazy server startup instead of splitting it on the first generate. - Bugfix `--score-on-error` and `--continue-on-fail` (when absent on the command line) silently overwriting a value set in a `@task`, a `--run-config` file, or a prior eval log being retried. diff --git a/src/inspect_ai/scorer/_model.py b/src/inspect_ai/scorer/_model.py index fcad6ea754..5f413cd198 100644 --- a/src/inspect_ai/scorer/_model.py +++ b/src/inspect_ai/scorer/_model.py @@ -18,7 +18,7 @@ from inspect_ai.solver._task_state import TaskState from inspect_ai.util import resource -from ._metric import INCORRECT, Score +from ._metric import Score from ._metrics import accuracy, stderr from ._multi import multi_scorer from ._scorer import Scorer, scorer @@ -226,16 +226,16 @@ async def score(state: TaskState, target: Target) -> Score: ), ) else: - return Score( - value=INCORRECT, + return Score.unscored( answer=state.output.completion, explanation="Grade not found in model output: " + f"{result.completion}", metadata=dict( + unscored_reason="grade_parse_failure", grading=[ scoring_prompt, result.message, - ] + ], ), ) diff --git a/tests/scorer/test_model_graded.py b/tests/scorer/test_model_graded.py index 3d01bf9ed8..c42041db2b 100644 --- a/tests/scorer/test_model_graded.py +++ b/tests/scorer/test_model_graded.py @@ -1,4 +1,5 @@ import asyncio +import math import os import re from typing import Any, Callable @@ -13,7 +14,14 @@ from inspect_ai.model import ChatMessageAssistant, ChatMessageUser, ModelName from inspect_ai.model._model import get_model from inspect_ai.model._model_output import ModelOutput -from inspect_ai.scorer import INCORRECT, Target, model_graded_fact, model_graded_qa +from inspect_ai.scorer import ( + CORRECT, + INCORRECT, + PARTIAL, + Target, + model_graded_fact, + model_graded_qa, +) from inspect_ai.scorer._model import ( DEFAULT_GRADE_PATTERN, neutralize_structural_delimiters, @@ -150,10 +158,7 @@ def test_model_role_precedence_for_model_graded_scorer( def test_model_graded_answer_set_on_grade_parse_failure(): - # issue #4025: when the grader output has no parseable GRADE: token the scorer - # falls into the parse-failure branch. value is INCORRECT, but the answer field - # must still carry the model's completion (matching the grade-found branch) so - # the log viewer doesn't show an empty answer. + # #4025: parse failure is unscored, but answer must still carry the completion. subject_answer = "The capital of France is Paris." grader_model = get_model( "mockllm/model", @@ -177,7 +182,7 @@ def test_model_graded_answer_set_on_grade_parse_failure(): assert log.samples score = log.samples[0].scores["model_graded_fact"] - assert score.value == INCORRECT + assert isinstance(score.value, float) and math.isnan(score.value) assert score.answer == subject_answer @@ -290,6 +295,68 @@ def test_default_grade_pattern_extraction(grader_output: str, expected: str) -> assert match.group(1) == expected +@pytest.mark.parametrize( + "grader_output", + [ + pytest.param("GRID: C", id="typo_grid"), + pytest.param("ANSWER: C", id="wrong_word_answer"), + pytest.param("**Answer: C**", id="markdown_decorated"), + pytest.param("The submission is correct.", id="no_grade_marker_at_all"), + ], +) +def test_grade_parse_failure_is_unscored(grader_output: str) -> None: + grader = get_model( + "mockllm/model", + custom_outputs=[ + ModelOutput.from_content("mockllm/model", [ContentText(text=grader_output)]) + ], + ) + task = Task( + dataset=[Sample(input="What is 1 + 1?", target="2")], + scorer=model_graded_fact(model=grader), + ) + log = eval(task, model="mockllm/model")[0] + assert log.samples + scores = log.samples[0].scores + assert scores is not None + score = scores["model_graded_fact"] + assert isinstance(score.value, float) and math.isnan(score.value), ( + f"expected unscored (NaN) for {grader_output!r}, got {score.value!r}" + ) + assert score.metadata is not None + assert score.metadata["unscored_reason"] == "grade_parse_failure" + + +@pytest.mark.parametrize( + "grader_output, expected", + [ + pytest.param("GRADE: C", CORRECT, id="correct"), + pytest.param("GRADE: I", INCORRECT, id="incorrect"), + pytest.param("GRADE: P", PARTIAL, id="partial"), + ], +) +def test_matched_grade_resolves_to_value(grader_output: str, expected: str) -> None: + # A parseable grade must resolve to its own value, not get swept into unscored. + grader = get_model( + "mockllm/model", + custom_outputs=[ + ModelOutput.from_content("mockllm/model", [ContentText(text=grader_output)]) + ], + ) + task = Task( + dataset=[Sample(input="What is 1 + 1?", target="2")], + scorer=model_graded_fact(model=grader), + ) + log = eval(task, model="mockllm/model")[0] + assert log.samples + scores = log.samples[0].scores + assert scores is not None + score = scores["model_graded_fact"] + assert score.value == expected, ( + f"expected {expected!r} for grade {grader_output!r}, got {score.value!r}" + ) + + @pytest.mark.parametrize( "grader_output", [