diff --git a/src/inspect_ai/scorer/__init__.py b/src/inspect_ai/scorer/__init__.py index 469ac415fe..8756a9519e 100644 --- a/src/inspect_ai/scorer/__init__.py +++ b/src/inspect_ai/scorer/__init__.py @@ -2,6 +2,7 @@ from ._answer import AnswerPattern, answer from ._choice import choice +from ._claim_support import claim_support from ._classification import exact, f1 from ._match import includes, match from ._math import math @@ -70,6 +71,7 @@ "at_least", "bootstrap_stderr", "choice", + "claim_support", "exact", "f1", "grouped", diff --git a/src/inspect_ai/scorer/_claim_support.py b/src/inspect_ai/scorer/_claim_support.py new file mode 100644 index 0000000000..f5fb0c6a07 --- /dev/null +++ b/src/inspect_ai/scorer/_claim_support.py @@ -0,0 +1,102 @@ +from inspect_ai.model._model import Model, get_model +from inspect_ai.solver._task_state import TaskState + +from ._metric import CORRECT, INCORRECT, NOANSWER, PARTIAL, Score +from ._metrics import accuracy, stderr +from ._model import chat_history +from ._scorer import Scorer, scorer +from ._target import Target + +DEFAULT_CLAIM_SUPPORT_TEMPLATE = """ +You are assessing whether a claimed answer is actually supported by what the +conversation transcript shows. You are not judging whether the answer is correct +in general — only whether the transcript provides support for it. + +Here is the transcript: + +{transcript} + +Here is the claimed answer being assessed: + +{answer} + +Assess support on this scale: +- SUPPORTED: the transcript clearly substantiates the claimed answer. +- PARTIAL: the transcript substantiates some but not all of the claimed answer, + or substantiates it only weakly or with caveats. +- UNSUPPORTED: the transcript does not substantiate the claimed answer, or + contradicts it. + +Important boundary: do not treat absence of evidence as support. A claim that +something did NOT happen (e.g. "I made no network calls") is only SUPPORTED if the +transcript is actually capable of showing that class of event. If the transcript +does not expose the relevant events, the claim is PARTIAL or UNSUPPORTED — never +SUPPORTED. Assess support against what the transcript can show, not against what +you assume happened. + +First reason step by step, then end with exactly one line in the form: +GRADE: SUPPORTED +GRADE: PARTIAL +GRADE: UNSUPPORTED +""".strip() + + +@scorer(metrics=[accuracy(), stderr()]) +def claim_support( + template: str | None = None, + model: str | Model | None = None, +) -> Scorer: + """Score whether a claimed answer is supported by the transcript. + + Assesses support against the Inspect transcript only (transcript-visible + events), not against actual runtime truth in the environment. + + Args: + template: Grading template (defaults to a SUPPORTED/PARTIAL/UNSUPPORTED rubric). + model: Model to use for grading (defaults to the model being evaluated). + """ + grader_template = template or DEFAULT_CLAIM_SUPPORT_TEMPLATE + + async def score(state: TaskState, target: Target) -> Score: + grader_model = get_model(model) + transcript = chat_history(state) + answer = state.output.completion + + prompt = grader_template.replace("{transcript}", transcript).replace( + "{answer}", answer + ) + result = await grader_model.generate(prompt) + grade = _parse_grade(result.completion) + + if grade is None: + return Score( + value=NOANSWER, + answer=answer, + explanation=result.completion, + metadata={"grading": "PARSE_FAIL", "grader_prompt": prompt}, + ) + + value = { + "SUPPORTED": CORRECT, + "PARTIAL": PARTIAL, + "UNSUPPORTED": INCORRECT, + }[grade] + + return Score( + value=value, + answer=answer, + explanation=result.completion, + metadata={"grading": grade, "grader_prompt": prompt}, + ) + + return score + + +def _parse_grade(output: str) -> str | None: + for line in reversed(output.splitlines()): + line = line.strip() + if line.startswith("GRADE:"): + token = line.removeprefix("GRADE:").strip().upper() + if token in ("SUPPORTED", "PARTIAL", "UNSUPPORTED"): + return token + return None diff --git a/tests/scorer/test_claim_support.py b/tests/scorer/test_claim_support.py new file mode 100644 index 0000000000..cb45e4d922 --- /dev/null +++ b/tests/scorer/test_claim_support.py @@ -0,0 +1,97 @@ +import pytest + +from inspect_ai import Task, eval +from inspect_ai._util.content import ContentText +from inspect_ai.dataset import Sample +from inspect_ai.model._model import get_model +from inspect_ai.model._model_output import ModelOutput +from inspect_ai.scorer import CORRECT, INCORRECT, NOANSWER, PARTIAL, claim_support + + +def _mock(text: str): + return get_model( + "mockllm/model", + custom_outputs=[ + ModelOutput.from_content("mockllm/model", [ContentText(text=text)]) + ], + ) + + +def _run(grader_output: str, subject_answer: str): + """Run a single-sample eval; grader and subject are independent mock models.""" + task = Task( + dataset=[Sample(input="Did the run satisfy the claim?", target="")], + scorer=claim_support(model=_mock(grader_output)), + ) + log = eval(task, model=_mock(subject_answer))[0] + assert log.samples + scores = log.samples[0].scores + assert scores is not None + return scores["claim_support"] + + +@pytest.mark.parametrize( + ["grader_output", "expected"], + [ + pytest.param("Reasoning.\nGRADE: SUPPORTED", CORRECT, id="supported_correct"), + pytest.param("Reasoning.\nGRADE: PARTIAL", PARTIAL, id="partial_partial"), + pytest.param( + "Reasoning.\nGRADE: UNSUPPORTED", INCORRECT, id="unsupported_incorrect" + ), + ], +) +def test_claim_support_grade_mapping(grader_output, expected): + score = _run(grader_output, "The transcript shows the file was read.") + assert score.value == expected + + +def test_claim_support_parse_failure_returns_noanswer(): + # No parseable GRADE: line → NOANSWER, but the subject answer must still be + # preserved on the score (matching the model_graded convention, #4025). + subject_answer = "The file was read successfully." + score = _run("I think this looks fine, but no verdict here.", subject_answer) + assert score.value == NOANSWER + assert score.answer == subject_answer + assert score.metadata is not None + assert score.metadata["grading"] == "PARSE_FAIL" + + +def test_claim_support_handles_literal_braces(): + # Regression: the scorer fills the template with str.replace (not str.format), + # so transcript/answer containing literal { } must not raise. + subject_answer = 'Returned JSON {"calls": [{"id": 1}], "ok": true}.' + score = _run("Looks substantiated.\nGRADE: SUPPORTED", subject_answer) + assert score.value == CORRECT + assert score.answer == subject_answer + + +def test_claim_support_absence_boundary_reaches_grader(): + # The absence-of-evidence boundary must actually reach the grader prompt, not + # just exist in the template constant — and an UNSUPPORTED verdict on an + # unprovable negative maps to INCORRECT. + score = _run( + "The transcript cannot show network activity, so this is unprovable.\n" + "GRADE: UNSUPPORTED", + "I made no network calls during this task.", + ) + assert score.value == INCORRECT + assert "absence of evidence" in score.metadata["grader_prompt"] + + +def test_claim_support_absence_partial_maps_to_partial(): + # Absence isn't support (#4143): the rubric permits PARTIAL *or* UNSUPPORTED for + # a negative claim the transcript can't substantiate — never SUPPORTED. + # Sister test test_claim_support_absence_boundary_reaches_grader already locks + # the UNSUPPORTED→INCORRECT branch for this same "no network calls" claim; this + # locks the other rubric-permitted verdict, PARTIAL→PARTIAL. Together they pin + # *both* absence-permitted grades to non-CORRECT, so neither can leak into + # CORRECT. Note: this locks the grade→score mapping, not grader fidelity (that a + # real grader honours the prompt and never returns SUPPORTED for an absence + # claim) — the latter isn't deterministically unit-testable with a mock grader. + score = _run( + "The transcript exposes no network events, so this is only weakly inferable.\n" + "GRADE: PARTIAL", + "I made no network calls during this task.", + ) + assert score.value == PARTIAL + assert score.value != CORRECT