feat(scorer): add claim_support scorer (#4143)

src/inspect_ai/scorer/__init__.py

@@ -2,6 +2,7 @@
 
 from ._answer import AnswerPattern, answer
 from ._choice import choice
+from ._claim_support import claim_support
 from ._classification import exact, f1
 from ._match import includes, match
 from ._math import math
@@ -70,6 +71,7 @@
     "at_least",
     "bootstrap_stderr",
     "choice",
+    "claim_support",
     "exact",
     "f1",
     "grouped",

src/inspect_ai/scorer/_claim_support.py

@@ -0,0 +1,102 @@
+from inspect_ai.model._model import Model, get_model
+from inspect_ai.solver._task_state import TaskState
+
+from ._metric import CORRECT, INCORRECT, NOANSWER, PARTIAL, Score
+from ._metrics import accuracy, stderr
+from ._model import chat_history
+from ._scorer import Scorer, scorer
+from ._target import Target
+
+DEFAULT_CLAIM_SUPPORT_TEMPLATE = """
+You are assessing whether a claimed answer is actually supported by what the
+conversation transcript shows. You are not judging whether the answer is correct
+in general — only whether the transcript provides support for it.
+
+Here is the transcript:
+
+{transcript}
+
+Here is the claimed answer being assessed:
+
+{answer}
+
+Assess support on this scale:
+- SUPPORTED: the transcript clearly substantiates the claimed answer.
+- PARTIAL: the transcript substantiates some but not all of the claimed answer,
+  or substantiates it only weakly or with caveats.
+- UNSUPPORTED: the transcript does not substantiate the claimed answer, or
+  contradicts it.
+
+Important boundary: do not treat absence of evidence as support. A claim that
+something did NOT happen (e.g. "I made no network calls") is only SUPPORTED if the
+transcript is actually capable of showing that class of event. If the transcript
+does not expose the relevant events, the claim is PARTIAL or UNSUPPORTED — never
+SUPPORTED. Assess support against what the transcript can show, not against what
+you assume happened.
+
+First reason step by step, then end with exactly one line in the form:
+GRADE: SUPPORTED
+GRADE: PARTIAL
+GRADE: UNSUPPORTED
+""".strip()
+
+
+@scorer(metrics=[accuracy(), stderr()])
+def claim_support(
+    template: str | None = None,
+    model: str | Model | None = None,
+) -> Scorer:
+    """Score whether a claimed answer is supported by the transcript.
+
+    Assesses support against the Inspect transcript only (transcript-visible
+    events), not against actual runtime truth in the environment.
+
+    Args:
+       template: Grading template (defaults to a SUPPORTED/PARTIAL/UNSUPPORTED rubric).
+       model: Model to use for grading (defaults to the model being evaluated).
+    """
+    grader_template = template or DEFAULT_CLAIM_SUPPORT_TEMPLATE
+
+    async def score(state: TaskState, target: Target) -> Score:
+        grader_model = get_model(model)
+        transcript = chat_history(state)
+        answer = state.output.completion
+
+        prompt = grader_template.replace("{transcript}", transcript).replace(
+            "{answer}", answer
+        )
+        result = await grader_model.generate(prompt)
+        grade = _parse_grade(result.completion)
+
+        if grade is None:
+            return Score(
+                value=NOANSWER,
+                answer=answer,
+                explanation=result.completion,
+                metadata={"grading": "PARSE_FAIL", "grader_prompt": prompt},
+            )
+
+        value = {
+            "SUPPORTED": CORRECT,
+            "PARTIAL": PARTIAL,
+            "UNSUPPORTED": INCORRECT,
+        }[grade]
+
+        return Score(
+            value=value,
+            answer=answer,
+            explanation=result.completion,
+            metadata={"grading": grade, "grader_prompt": prompt},
+        )
+
+    return score
+
+
+def _parse_grade(output: str) -> str | None:
+    for line in reversed(output.splitlines()):
+        line = line.strip()
+        if line.startswith("GRADE:"):
+            token = line.removeprefix("GRADE:").strip().upper()
+            if token in ("SUPPORTED", "PARTIAL", "UNSUPPORTED"):
+                return token
+    return None

tests/scorer/test_claim_support.py

@@ -0,0 +1,97 @@
+import pytest
+
+from inspect_ai import Task, eval
+from inspect_ai._util.content import ContentText
+from inspect_ai.dataset import Sample
+from inspect_ai.model._model import get_model
+from inspect_ai.model._model_output import ModelOutput
+from inspect_ai.scorer import CORRECT, INCORRECT, NOANSWER, PARTIAL, claim_support
+
+
+def _mock(text: str):
+    return get_model(
+        "mockllm/model",
+        custom_outputs=[
+            ModelOutput.from_content("mockllm/model", [ContentText(text=text)])
+        ],
+    )
+
+
+def _run(grader_output: str, subject_answer: str):
+    """Run a single-sample eval; grader and subject are independent mock models."""
+    task = Task(
+        dataset=[Sample(input="Did the run satisfy the claim?", target="")],
+        scorer=claim_support(model=_mock(grader_output)),
+    )
+    log = eval(task, model=_mock(subject_answer))[0]
+    assert log.samples
+    scores = log.samples[0].scores
+    assert scores is not None
+    return scores["claim_support"]
+
+
+@pytest.mark.parametrize(
+    ["grader_output", "expected"],
+    [
+        pytest.param("Reasoning.\nGRADE: SUPPORTED", CORRECT, id="supported_correct"),
+        pytest.param("Reasoning.\nGRADE: PARTIAL", PARTIAL, id="partial_partial"),
+        pytest.param(
+            "Reasoning.\nGRADE: UNSUPPORTED", INCORRECT, id="unsupported_incorrect"
+        ),
+    ],
+)
+def test_claim_support_grade_mapping(grader_output, expected):
+    score = _run(grader_output, "The transcript shows the file was read.")
+    assert score.value == expected
+
+
+def test_claim_support_parse_failure_returns_noanswer():
+    # No parseable GRADE: line → NOANSWER, but the subject answer must still be
+    # preserved on the score (matching the model_graded convention, #4025).
+    subject_answer = "The file was read successfully."
+    score = _run("I think this looks fine, but no verdict here.", subject_answer)
+    assert score.value == NOANSWER
+    assert score.answer == subject_answer
+    assert score.metadata is not None
+    assert score.metadata["grading"] == "PARSE_FAIL"
+
+
+def test_claim_support_handles_literal_braces():
+    # Regression: the scorer fills the template with str.replace (not str.format),
+    # so transcript/answer containing literal { } must not raise.
+    subject_answer = 'Returned JSON {"calls": [{"id": 1}], "ok": true}.'
+    score = _run("Looks substantiated.\nGRADE: SUPPORTED", subject_answer)
+    assert score.value == CORRECT
+    assert score.answer == subject_answer
+
+
+def test_claim_support_absence_boundary_reaches_grader():
+    # The absence-of-evidence boundary must actually reach the grader prompt, not
+    # just exist in the template constant — and an UNSUPPORTED verdict on an
+    # unprovable negative maps to INCORRECT.
+    score = _run(
+        "The transcript cannot show network activity, so this is unprovable.\n"
+        "GRADE: UNSUPPORTED",
+        "I made no network calls during this task.",
+    )
+    assert score.value == INCORRECT
+    assert "absence of evidence" in score.metadata["grader_prompt"]
+
+
+def test_claim_support_absence_partial_maps_to_partial():
+    # Absence isn't support (#4143): the rubric permits PARTIAL *or* UNSUPPORTED for
+    # a negative claim the transcript can't substantiate — never SUPPORTED.
+    # Sister test test_claim_support_absence_boundary_reaches_grader already locks
+    # the UNSUPPORTED→INCORRECT branch for this same "no network calls" claim; this
+    # locks the other rubric-permitted verdict, PARTIAL→PARTIAL. Together they pin
+    # *both* absence-permitted grades to non-CORRECT, so neither can leak into
+    # CORRECT. Note: this locks the grade→score mapping, not grader fidelity (that a
+    # real grader honours the prompt and never returns SUPPORTED for an absence
+    # claim) — the latter isn't deterministically unit-testable with a mock grader.
+    score = _run(
+        "The transcript exposes no network events, so this is only weakly inferable.\n"
+        "GRADE: PARTIAL",
+        "I made no network calls during this task.",
+    )
+    assert score.value == PARTIAL
+    assert score.value != CORRECT