diff --git a/pyproject.toml b/pyproject.toml index c0b2632..4630746 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -108,7 +108,7 @@ ignore_missing_imports = true [[tool.mypy.overrides]] # Upstream labbench2 git dep has no py.typed marker / stubs. -module = ["evals", "evals.*"] +module = ["evals", "evals.*", "labbench2", "labbench2.*"] ignore_missing_imports = true [[tool.mypy.overrides]] diff --git a/src/lab_bench_2/README.md b/src/lab_bench_2/README.md index ce8af75..1e7d6e9 100644 --- a/src/lab_bench_2/README.md +++ b/src/lab_bench_2/README.md @@ -43,6 +43,15 @@ ANTHROPIC_API_KEY= ``` +### Additional Usage Notes + +The `cloning` tag scorer runs a PCR (Polymerase Chain Reaction) simulation using +a small Go binary that labbench2 compiles on first use, so a +Go toolchain (1.21+) must be available on the host the first time you score +`cloning`; the compiled binary is then cached inside the installed package and +reused. To install Go: `brew install go` (macOS), `sudo apt install golang-go` (Linux), +or . + ## Options @@ -62,8 +71,8 @@ See `uv run inspect eval --help` for all available options. ### `lab_bench_2` -- `tag` (str): Which LAB-Bench 2 subset to run. Supported tags: ``litqa3``, ``patentqa``, ``protocolqa2``, ``sourcequality``, ``trialqa``. (default: `'litqa3'`) -- `mode` (Mode): How a question's data files are delivered to the model. A no-op for tags without files (such as litqa3). Options: ``file``: Files uploaded via API. PDFs/images attached as context; other files as document attachments., ``inject``: Text file contents concatenated into the prompt as text., ``retrieve``: Only file names/stems are given; prompt instructs the agent to retrieve the necessary sequences or data from a source of its choosing. File contents are withheld. (default: `'inject'`) +- `tag` (str): Which LAB-Bench 2 subset to run. Supported tags: ``cloning``, ``dbqa2``, ``figqa2`` (and ``figqa2-img`` / ``figqa2-pdf``), ``litqa3``, ``patentqa``, ``protocolqa2``, ``seqqa2``, ``sourcequality``, ``suppqa2``, ``tableqa2`` (and ``tableqa2-img`` / ``tableqa2-pdf``), ``trialqa``. (default: `'litqa3'`) +- `mode` (Mode): How a question's data files are delivered to the model. A no-op for tags without files (such as litqa3). Options: ``file``: Files uploaded via API. PDFs/images attached as context; other files as document attachments., ``inject``: Text file contents concatenated into the prompt as text., ``retrieve``: Only file names/stems are given; prompt instructs the agent to retrieve the necessary sequences or data from a source of its choosing. File contents are withheld. (default: `'file'`) - `solver` (Solver | None): The solver to run. Defaults to ``bare()`` (the benchmark's "bare" configuration: a plain single-turn ``generate()``) when not provided. Pass any Inspect solver to override, e.g. ``-T solver=bare`` on the CLI. (default: `None`) @@ -73,29 +82,68 @@ This eval uses the public `EdisonScientific/labbench2` dataset on Hugging Face, ### Supported tags -| Tag | Samples | File-bearing | `mode` to use | Notes | -| --------------- | ------- | ------------ | ---------------------- | --------------------------- | -| `litqa3` | 168 | No | any (mode is a no-op) | Literature reasoning. | -| `patentqa` | 121 | No | any (mode is a no-op) | Patent comprehension. | -| `protocolqa2` | 125 | Yes | `file` | Lab protocols. | -| `sourcequality` | 150 | Yes | `file` | Source quality assessment. | -| `trialqa` | 120 | No | any (mode is a no-op) | Clinical trial QA. | +| Tag | Samples | File-bearing | `mode` to use | Notes | +| --------------- | ------- | ------------ | ---------------------- |----------------------------------------| +| `cloning` | 14 | Yes | any (all modes) | Cloning protocols; reward-scored. | +| `dbqa2` | 86 | No | any (mode is a no-op) | Database access; recall judge. | +| `figqa2` | 101 | No | any (mode is a no-op) | Figure QA; exact-match judge. | +| `figqa2-img` | 101 | Yes | `file` | Figure QA with image files. | +| `figqa2-pdf` | 101 | Yes | `file` | Figure QA with PDF files. | +| `litqa3` | 168 | No | any (mode is a no-op) | Literature reasoning. | +| `patentqa` | 121 | No | any (mode is a no-op) | Patent comprehension. | +| `protocolqa2` | 125 | Yes | `file` | Lab protocols. | +| `seqqa2` | 400 | Yes | `file` / `inject` | Sequence QA; deterministic validators. | +| `sourcequality` | 150 | Yes | `file` | Source quality assessment. | +| `suppqa2` | 125 | No | any (mode is a no-op) | Supplement QA; exact-match. | +| `tableqa2` | 100 | No | any (mode is a no-op) | Table QA; exact-match judge. | +| `tableqa2-img` | 100 | Yes | `file` | Table QA with image files. | +| `tableqa2-pdf` | 100 | Yes | `file` | Table QA with PDF files. | +| `trialqa` | 120 | No | any (mode is a no-op) | Clinical trial QA. | For file-bearing tags, the loader filters out questions that don't opt into the requested `mode` (per each question's `QuestionMode` flags in the HF -data). At the time of writing every `protocolqa2` and `sourcequality` -question opts into `file` only, so passing any other `mode` would load zero -samples. The mode flags live in the dataset and may change — verify by -running with the configuration you intend before drawing conclusions from -sample counts. +data). + +#### Relationship between the tag and mode parameters + +Tags describe groups of samples/questions whilst mode describes how data files are uploaded. +Not every sample is compatible with each mode of data uploading; if incompatible they are not loaded into the eval. +Each sample in the dataset contains flags for compatible modes - this may change and sample counts +can be verified by running with the configuration you intend before drawing conclusions from sample counts. + +For most tags, those that uses files requires the `file` mode. For example; + +`uv run inspect eval lab_bench_2 -T tag=sourcequality -T mode=retrieve` + + Will result in no samples being loaded in. This is also true for tags protocolqa2`,`sourcequality`,`figqa2-img`,`figqa2-pdf`,`tableqa2-img`,`tableqa2-pdf`. + +Note that the base `figqa2`, `tableqa2`, and `suppqa2` tags have no files (mode is a no-op). Their image/PDF variants do have files and are impacted by the above. + +`seqqa2` is the exception: all of its samples are compatible with `file` and `inject`, while only a +subset of this tag can be used with`retrieve` (so `mode="retrieve"` loads fewer samples). ## Scoring -Answers are graded by a semantic LLM judge. The judge compares the model's -answer to the reference, accepting semantically or numerically equivalent -answers, and returns one of `correct` / `incorrect` / `unsure`; a `correct` -verdict scores 1.0 and everything else (including unparseable or empty -judgements) scores 0.0. Reported metrics are `accuracy` and `stderr`. +There are different scoring methods for the tags. +Some tags are scored deterministically (see `cloning` and `seqqa2`) but most are graded by an LLM judge. + +### LLM judge scorers + +The judge compares the solver's answer to the reference, accepting semantically +or numerically equivalent answers, and returns one of `correct` / `incorrect` / `unsure`; +a `correct` verdict scores 1.0 and everything else (including unparseable or empty judgements) +scores 0.0. Reported metrics are `accuracy` and `stderr`. + +The judge requests **structured output** (a typed `result` / `rationale` +schema), so the verdict is read from a typed field rather than scraped from +prose. If the grader's provider doesn't support structured output, it falls back +to parsing a `result:` line from the response. + +The judge prompt varies by tag: most tags use the default semantic prompt; +`dbqa2` (database access) uses a recall-based variant that marks an answer +correct when it recovers the expected reference values; and the figure, table, +and supplement tags (`figqa2*`, `tableqa2*`, `suppqa2`) use an exact-match +variant for numeric answers. The judge model is selected via the `grader` model role and defaults to `anthropic/claude-sonnet-4-5` at temperature 0. Override it on the command line, @@ -107,6 +155,21 @@ uv run inspect eval lab_bench_2/lab_bench_2 \ --model-role grader=anthropic/claude-opus-4-1-20250805 ``` +### Deterministic scorers + +The `cloning` tag is not graded by an LLM judge: it is scored deterministically +by labbench2's reward pipeline, which parses the submitted protocol, executes it +(including Polymerase Chain Reaction simulation), and compares the result to the +reference assembly via sequence-similarity and restriction-digest checks. +PCR simulation requires that Go be available on the host. Without Go, +protocol execution fails gracefully: PCR-based samples score 0.0 with an explanatory +reason rather than crashing the run. + +The `seqqa2` tag is also scored deterministically. A +per-question validator (selected by the question's `type`) checks the answer +extracted via that question's `answer_regex`; extraction tolerates line-wrapped +or whitespace-separated sequences. + ## Attribution This evaluation depends on the reference implementation's diff --git a/src/lab_bench_2/__init__.py b/src/lab_bench_2/__init__.py index 5a9ddfe..a6e7b6f 100644 --- a/src/lab_bench_2/__init__.py +++ b/src/lab_bench_2/__init__.py @@ -9,9 +9,13 @@ from lab_bench_2.prompt_composer import Mode from lab_bench_2.scorers import ( DEFAULT_GRADER_MODEL, + cloning_scorer, + exact_match_judge_scorer, parse_judge_verdict, + recall_judge_scorer, scorer_for_tag, semantic_judge_scorer, + seqqa2_scorer, ) from lab_bench_2.solvers import bare @@ -23,10 +27,14 @@ "SUPPORTED_TAGS", "Mode", "bare", + "cloning_scorer", + "exact_match_judge_scorer", "lab_bench_2", "load_lab_bench_2_dataset", "parse_judge_verdict", + "recall_judge_scorer", "record_to_sample", "scorer_for_tag", "semantic_judge_scorer", + "seqqa2_scorer", ] diff --git a/src/lab_bench_2/lab_bench_2.py b/src/lab_bench_2/lab_bench_2.py index e0b7f09..98b39b1 100644 --- a/src/lab_bench_2/lab_bench_2.py +++ b/src/lab_bench_2/lab_bench_2.py @@ -17,10 +17,20 @@ from utils.metadata import load_version_from_yaml SUPPORTED_TAGS = ( + "cloning", + "dbqa2", + "figqa2", + "figqa2-img", + "figqa2-pdf", "litqa3", "patentqa", "protocolqa2", + "seqqa2", "sourcequality", + "suppqa2", + "tableqa2", + "tableqa2-img", + "tableqa2-pdf", "trialqa", ) @@ -30,14 +40,17 @@ @task def lab_bench_2( tag: str = "litqa3", - mode: Mode = "inject", + mode: Mode = "file", solver: Solver | None = None, ) -> Task: """LAB-Bench 2 evaluation task. Args: - tag: Which LAB-Bench 2 subset to run. Supported tags: ``litqa3``, - ``patentqa``, ``protocolqa2``, ``sourcequality``, ``trialqa``. + tag: Which LAB-Bench 2 subset to run. Supported tags: ``cloning``, + ``dbqa2``, ``figqa2`` (and ``figqa2-img`` / ``figqa2-pdf``), + ``litqa3``, ``patentqa``, ``protocolqa2``, ``seqqa2``, + ``sourcequality``, ``suppqa2``, ``tableqa2`` (and ``tableqa2-img`` / + ``tableqa2-pdf``), ``trialqa``. mode: How a question's data files are delivered to the model. A no-op for tags without files (such as litqa3). Options: diff --git a/src/lab_bench_2/scorers.py b/src/lab_bench_2/scorers.py index 56916b6..e8c0428 100644 --- a/src/lab_bench_2/scorers.py +++ b/src/lab_bench_2/scorers.py @@ -3,8 +3,10 @@ from __future__ import annotations import re +from pathlib import Path +from typing import Any, cast -from inspect_ai.model import GenerateConfig, get_model +from inspect_ai.model import GenerateConfig, ResponseSchema, get_model from inspect_ai.scorer import ( CORRECT, INCORRECT, @@ -16,6 +18,10 @@ stderr, ) from inspect_ai.solver import TaskState +from inspect_ai.util import json_schema +from pydantic import ValidationError + +from lab_bench_2 import seqqa2_answer_parser DEFAULT_GRADER_MODEL = "anthropic/claude-sonnet-4-5" GRADER_ROLE = "grader" @@ -23,16 +29,34 @@ JUDGE_VERDICT_CORRECT = "correct" JUDGE_VERDICT_INCORRECT = "incorrect" JUDGE_VERDICT_UNSURE = "unsure" +# Excluding ``=`` from the separator stops the pattern from matching code-like +# grader output such as ``result = "correct"`` (an assignment, not a verdict). _GRADE_PATTERN = re.compile( - r"\bresult\b[^A-Za-z]*(correct|incorrect|unsure)\b", + r"\bresult\b[^A-Za-z=]*(correct|incorrect|unsure)\b", re.IGNORECASE, ) +# The recall prompt (unlike the semantic and exact-match prompts) does not ask +# the model for a parseable verdict line, so we append one carrying the +# ``result:`` marker that ``parse_judge_verdict`` reads. +VERDICT_FORMAT_SUFFIX = ( + "\n\nAfter your analysis, end your response with a final line in exactly " + "this form:\n\nresult: " +) -@scorer(metrics=[accuracy(), stderr()]) -def semantic_judge_scorer() -> Scorer: - """Grade an open-ended answer against the reference using a judge model.""" - from evals.prompts import STRUCTURED_EVALUATION_PROMPT + +def _judge_score(prompt_template: str) -> Scorer: + """Build a judge that grades an answer against the reference via the template. + + Requests structured output so the grader returns a typed verdict; falls back + to regex parsing for providers that don't honor the schema or for + non-compliant responses. + """ + from evals.models import EvaluationResult + + response_schema = ResponseSchema( + name="evaluation_result", json_schema=json_schema(EvaluationResult) + ) async def score(state: TaskState, target: Target) -> Score: answer = state.output.completion.strip() @@ -41,35 +65,200 @@ async def score(state: TaskState, target: Target) -> Score: value=INCORRECT, answer="", explanation="No answer was produced." ) - grader = get_model( - role=GRADER_ROLE, - default=DEFAULT_GRADER_MODEL, - config=GenerateConfig(temperature=0.0), - ) + grader = get_model(role=GRADER_ROLE, default=DEFAULT_GRADER_MODEL) - prompt = STRUCTURED_EVALUATION_PROMPT.format( + prompt = prompt_template.format( question=state.input_text, correct_answer=target.text, answer=answer, ) - result = await grader.generate(prompt) - verdict = parse_judge_verdict(result.completion) + result = await grader.generate( + prompt, + config=GenerateConfig(temperature=0.0, response_schema=response_schema), + ) + + try: + evaluation = EvaluationResult.model_validate_json(result.completion) + verdict: str | None = evaluation.result + explanation = evaluation.rationale + verdict_source = "structured" + except ValidationError: + verdict = parse_judge_verdict(result.completion) + explanation = result.completion + verdict_source = "fallback" + value = CORRECT if verdict == JUDGE_VERDICT_CORRECT else INCORRECT return Score( value=value, answer=answer, - explanation=result.completion, - metadata={"verdict": verdict}, + explanation=explanation, + metadata={"verdict": verdict, "verdict_source": verdict_source}, + ) + + return score + + +@scorer(metrics=[accuracy(), stderr()]) +def semantic_judge_scorer() -> Scorer: + """Grade an open-ended answer against the reference using a judge model.""" + from evals.prompts import STRUCTURED_EVALUATION_PROMPT + + return _judge_score(STRUCTURED_EVALUATION_PROMPT) + + +@scorer(metrics=[accuracy(), stderr()]) +def recall_judge_scorer() -> Scorer: + """Grade a dbqa2 answer by recall of the expected values (data-access bench).""" + from evals.prompts import STRUCTURED_EVALUATION_PROMPT_DATA_ACCESS_BENCH_RECALL + + return _judge_score( + STRUCTURED_EVALUATION_PROMPT_DATA_ACCESS_BENCH_RECALL + VERDICT_FORMAT_SUFFIX + ) + + +@scorer(metrics=[accuracy(), stderr()]) +def exact_match_judge_scorer() -> Scorer: + """Grade a figure/table/supplement answer by exact numeric match.""" + from evals.prompts import STRUCTURED_EVALUATION_PROMPT_EXACT_MATCH + + return _judge_score(STRUCTURED_EVALUATION_PROMPT_EXACT_MATCH) + + +@scorer(metrics=[accuracy(), stderr()]) +def cloning_scorer() -> Scorer: + """Score CloningQA answers using labbench2's cloning reward pipeline. + + Validates cloning protocols through 4 sequential stages: + 1. Format validation — protocol can be parsed + 2. Execution — protocol runs successfully + 3. Similarity — output matches reference sequence (threshold: 0.95) + 4. Digest — restriction enzyme fragments match + + Requires Go 1.21+ on the host for PCR simulation scoring. + """ + from evals.utils import resolve_file_path + from labbench2.cloning.rewards import cloning_reward + + async def score(state: TaskState, target: Target) -> Score: + metadata = state.metadata or {} + files_path_str = metadata.get("files_path") + question_id = cast(str | None, metadata.get("id")) + + if not files_path_str or not question_id: + return Score( + value=INCORRECT, + explanation="Cloning evaluation requires files_path and id metadata.", + ) + + ground_truth_filename = f"{question_id}_assembled.fa" + ground_truth_path = resolve_file_path(ground_truth_filename, None) + if ground_truth_path is None: + return Score( + value=INCORRECT, + explanation=f"Ground truth file not found: {ground_truth_filename}", + ) + + answer = state.output.completion + score_value, reason = await cloning_reward( + answer=answer, + base_dir=Path(files_path_str), + reference_path=ground_truth_path, + validator_params=cast( + dict[str, Any], metadata.get("validator_params") or {} + ), + ) + + return Score( + value=CORRECT if score_value >= 1.0 else INCORRECT, + answer=answer, + explanation=reason, + metadata={"cloning_score": score_value}, + ) + + return score + + +@scorer(metrics=[accuracy(), stderr()]) +def seqqa2_scorer() -> Scorer: + """Score SeqQA2 answers with labbench2's deterministic per-type validators.""" + from evals.utils import resolve_file_path + from labbench2.seqqa2.registry import VALIDATORS + + async def score(state: TaskState, target: Target) -> Score: + metadata = state.metadata or {} + question_type = cast(str | None, metadata.get("type")) + if not question_type: + return Score( + value=INCORRECT, + explanation="SeqQA2 evaluation requires question type metadata.", + ) + + validator = VALIDATORS.get(question_type) + if validator is None: + return Score( + value=INCORRECT, + explanation=f"No validator found for type: {question_type}", + ) + + raw_answer = state.output.completion + extracted = seqqa2_answer_parser.extract( + raw_answer, + cast(str | None, metadata.get("answer_regex")), + ) + if extracted is None: + return Score( + value=INCORRECT, + explanation="Failed to extract answer from model output.", + metadata={"raw_answer": raw_answer, "extracted": extracted}, + ) + + if "answer" in extracted and validator.answer_param != "answer": + extracted[validator.answer_param] = extracted.pop("answer") + + kwargs: dict[str, Any] = { + **cast(dict[str, Any], metadata.get("validator_params") or {}), + **extracted, + } + + files_path_str = cast(str | None, metadata.get("files_path")) + files_path = Path(files_path_str) if files_path_str else None + for key, value in list(kwargs.items()): + if key.endswith("_path") and isinstance(value, str): + resolved = resolve_file_path(value, files_path) + if resolved is None: + return Score( + value=INCORRECT, + explanation=f"File not found: {value}", + ) + kwargs[key] = resolved + + score_value = float(validator.func(**kwargs)) + passed = score_value >= 1.0 + return Score( + value=CORRECT if passed else INCORRECT, + answer=raw_answer, + explanation=f"Validator '{question_type}' {'passed' if passed else 'failed'}", + metadata={"validator": question_type, "validator_score": score_value}, ) return score SCORERS_BY_TAG = { + "cloning": cloning_scorer, + "dbqa2": recall_judge_scorer, + "figqa2": exact_match_judge_scorer, + "figqa2-img": exact_match_judge_scorer, + "figqa2-pdf": exact_match_judge_scorer, "litqa3": semantic_judge_scorer, "patentqa": semantic_judge_scorer, "protocolqa2": semantic_judge_scorer, + "seqqa2": seqqa2_scorer, "sourcequality": semantic_judge_scorer, + "suppqa2": exact_match_judge_scorer, + "tableqa2": exact_match_judge_scorer, + "tableqa2-img": exact_match_judge_scorer, + "tableqa2-pdf": exact_match_judge_scorer, "trialqa": semantic_judge_scorer, } diff --git a/src/lab_bench_2/seqqa2_answer_parser.py b/src/lab_bench_2/seqqa2_answer_parser.py new file mode 100644 index 0000000..fa5968b --- /dev/null +++ b/src/lab_bench_2/seqqa2_answer_parser.py @@ -0,0 +1,83 @@ +"""Answer extraction for the SeqQA2 scorer. + +Wraps upstream ``extract_answer`` with whitespace-normalization fallbacks so a +line-wrapped or space-separated sequence answer still matches the question's +answer regex. +""" + +from __future__ import annotations + +import re +from typing import cast + +ANSWER_BLOCK_RE = re.compile(r"(.*?)", re.IGNORECASE | re.DOTALL) +ASCII_WHITESPACE_RE = re.compile(r"[ \t\n\r\f\v]+") +NAMED_GROUP_RE = re.compile(r"\(\?P<[^>]+>") +IUPAC_NUCLEOTIDE_CHARS = frozenset("ACGTRYSWKMBDHVNacgtryswkmbdhvnUu") +REGEX_SYNTAX_CHARS = frozenset(r"[](){}?+*^$|,:\\-") + + +def _is_nucleotide_only_answer_regex(answer_regex: str) -> bool: + """Return True if the regex's literal characters are all IUPAC nucleotides. + + Used to decide whether stripping *all* internal whitespace from an answer is + a safe normalization (true for raw sequences, false for prose answers). + """ + without_group_names = NAMED_GROUP_RE.sub("(", answer_regex) + remaining = "".join( + char + for char in without_group_names + if char not in REGEX_SYNTAX_CHARS and not char.isspace() + ) + return bool(remaining) and set(remaining) <= IUPAC_NUCLEOTIDE_CHARS + + +def _normalized_answer_attempts(answer_text: str, answer_regex: str) -> list[str]: + """Whitespace-normalized variants of an answer to retry regex extraction. + + Always tries collapsing runs of whitespace to a single space; for + nucleotide-only regexes, also tries removing internal whitespace entirely. + """ + attempts = [ASCII_WHITESPACE_RE.sub(" ", answer_text).strip()] + if _is_nucleotide_only_answer_regex(answer_regex): + nucleotide_attempt = ASCII_WHITESPACE_RE.sub("", answer_text).strip() + if nucleotide_attempt not in attempts: + attempts.append(nucleotide_attempt) + return attempts + + +def extract(completion: str, answer_regex: str | None) -> dict[str, str] | None: + """Extract seqqa2 answer params via the question's answer regex. + + Delegates to upstream ``extract_answer`` first; on failure, retries against + whitespace-normalized variants of the ```` block to tolerate + line-wrapped or spaced sequence answers. Returns ``None`` if nothing matches. + """ + from evals.evaluators import extract_answer + + extracted = extract_answer(completion, answer_regex) + if extracted is not None: + return cast(dict[str, str], extracted) + + if not answer_regex: + return None + + answer_match = ANSWER_BLOCK_RE.search(completion) + if answer_match is None: + answer_text = completion + prefix = "" + suffix = "" + else: + answer_text = answer_match.group(1) + prefix = completion[: answer_match.start()] + "" + suffix = "" + completion[answer_match.end() :] + + for normalized_answer in _normalized_answer_attempts(answer_text, answer_regex): + if not normalized_answer: + continue + normalized_completion = f"{prefix}{normalized_answer}{suffix}" + extracted = extract_answer(normalized_completion, answer_regex) + if extracted is not None: + return cast(dict[str, str], extracted) + + return None diff --git a/tests/lab_bench_2/test_e2e.py b/tests/lab_bench_2/test_e2e.py index 7cacced..fb125aa 100644 --- a/tests/lab_bench_2/test_e2e.py +++ b/tests/lab_bench_2/test_e2e.py @@ -7,7 +7,7 @@ def test_unsupported_tag_raises() -> None: with pytest.raises(NotImplementedError): - lab_bench_2(tag="seqqa2") + lab_bench_2(tag="bogusqa") @pytest.mark.huggingface diff --git a/tests/lab_bench_2/test_scorers.py b/tests/lab_bench_2/test_scorers.py index 32db3af..ba59aca 100644 --- a/tests/lab_bench_2/test_scorers.py +++ b/tests/lab_bench_2/test_scorers.py @@ -1,8 +1,41 @@ +from pathlib import Path +from types import SimpleNamespace +from typing import Any + import pytest -from inspect_ai.scorer import Scorer +from inspect_ai.model import ModelName, ModelOutput +from inspect_ai.scorer import CORRECT, INCORRECT, Score, Scorer, Target +from inspect_ai.solver import TaskState + +from lab_bench_2 import SUPPORTED_TAGS, parse_judge_verdict, scorers +from lab_bench_2.scorers import ( + SCORERS_BY_TAG, + cloning_scorer, + exact_match_judge_scorer, + recall_judge_scorer, + scorer_for_tag, + semantic_judge_scorer, + seqqa2_scorer, +) + + +def _task_state(completion: str, metadata: dict[str, Any]) -> TaskState: + return TaskState( + model=ModelName("mockllm/model"), + sample_id="sample-1", + epoch=1, + input="Question?", + messages=[], + output=ModelOutput.from_content("mockllm/model", completion), + metadata=metadata, + ) -from lab_bench_2 import parse_judge_verdict -from lab_bench_2.scorers import scorer_for_tag, semantic_judge_scorer + +async def _score(sut: Scorer, state: TaskState, target: Target) -> Score: + """Run a scorer and assert it produced a Score (narrows ``Score | None``).""" + result = await sut(state, target) + assert result is not None + return result class TestParseJudgeVerdict: @@ -44,19 +77,355 @@ def test_last_verdict_wins(self) -> None: # when / then assert parse_judge_verdict(text) == "correct" + def test_parses_recall_style_output_with_format_suffix(self) -> None: + # given a recall-style judgement that echoes the rubric, then closes + # with the verdict line that VERDICT_FORMAT_SUFFIX instructs + text = ( + "Matched 5/6 expected variables. Recall = 0.83 < 0.95.\nresult: incorrect" + ) + # when / then + assert parse_judge_verdict(text) == "incorrect" + + def test_ignores_code_assignment(self) -> None: + # given grader output that is code rather than a verdict — `result = + # "correct"` is an assignment, not a graded result + text = ' result = "unknown"\n result = "correct"\n return result' + # when / then + assert parse_judge_verdict(text) is None + class TestScorerForTag: - @pytest.mark.parametrize( - "tag", - ["litqa3", "patentqa", "protocolqa2", "sourcequality", "trialqa"], - ) + @pytest.mark.parametrize("tag", sorted(SCORERS_BY_TAG)) def test_returns_scorer_for_supported_tag(self, tag: str) -> None: assert isinstance(scorer_for_tag(tag), Scorer) + def test_routing_table_matches_supported_tags(self) -> None: + # given/when/then — the task gate and the scorer routing list the same tags + assert set(SCORERS_BY_TAG) == set(SUPPORTED_TAGS) + def test_unsupported_tag_raises(self) -> None: with pytest.raises(NotImplementedError): - scorer_for_tag("seqqa2") + scorer_for_tag("bogusqa") def test_semantic_judge_scorer_is_scorer() -> None: assert isinstance(semantic_judge_scorer(), Scorer) + + +def test_recall_judge_scorer_is_scorer() -> None: + assert isinstance(recall_judge_scorer(), Scorer) + + +def test_exact_match_judge_scorer_is_scorer() -> None: + assert isinstance(exact_match_judge_scorer(), Scorer) + + +def _patch_grader(monkeypatch: pytest.MonkeyPatch, completion: str) -> None: + class _Grader: + async def generate(self, prompt: str, **kwargs: Any) -> SimpleNamespace: + return SimpleNamespace(completion=completion) + + monkeypatch.setattr(scorers, "get_model", lambda *args, **kwargs: _Grader()) + + +class TestJudgeScorer: + async def test_structured_correct_verdict_scores_correct( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + # given a grader returning a structured (typed) correct verdict + _patch_grader( + monkeypatch, + '{"rationale": "matches the reference", "result": "correct"}', + ) + # when + sut = semantic_judge_scorer() + result = await _score( + sut, _task_state("answer", {"tag": "litqa3"}), Target("ref") + ) + # then the typed rationale and verdict are used + assert result.value == CORRECT + assert result.explanation == "matches the reference" + assert result.metadata == { + "verdict": "correct", + "verdict_source": "structured", + } + + @pytest.mark.parametrize("verdict", ["incorrect", "unsure"]) + async def test_structured_non_correct_verdict_scores_incorrect( + self, monkeypatch: pytest.MonkeyPatch, verdict: str + ) -> None: + _patch_grader(monkeypatch, f'{{"rationale": "x", "result": "{verdict}"}}') + sut = semantic_judge_scorer() + result = await _score( + sut, _task_state("answer", {"tag": "litqa3"}), Target("ref") + ) + assert result.value == INCORRECT + assert result.metadata == {"verdict": verdict, "verdict_source": "structured"} + + async def test_falls_back_to_regex_for_non_structured_output( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + # given a grader that ignores the schema and returns free text + _patch_grader(monkeypatch, "Reasoning here.\nresult: correct") + # when + sut = semantic_judge_scorer() + result = await _score( + sut, _task_state("answer", {"tag": "litqa3"}), Target("ref") + ) + # then the regex fallback recovers the verdict + assert result.value == CORRECT + assert result.metadata == {"verdict": "correct", "verdict_source": "fallback"} + + @pytest.mark.parametrize( + "completion, expected_verdict", + [ + ("Reasoning here.\nresult: incorrect", "incorrect"), + ("no parseable verdict in this text", None), + ], + ) + async def test_falls_back_to_regex_scores_incorrect( + self, + monkeypatch: pytest.MonkeyPatch, + completion: str, + expected_verdict: str | None, + ) -> None: + # given non-structured grader output that is not a correct verdict + # (a parsed "incorrect", or nothing parseable at all) + _patch_grader(monkeypatch, completion) + # when + sut = semantic_judge_scorer() + result = await _score( + sut, _task_state("answer", {"tag": "litqa3"}), Target("ref") + ) + # then it scores incorrect + assert result.value == INCORRECT + assert result.metadata == { + "verdict": expected_verdict, + "verdict_source": "fallback", + } + + async def test_empty_answer_scores_incorrect( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + # given an empty answer but correct grade + answer = " " + _patch_grader(monkeypatch, '{"rationale": "x", "result": "correct"}') + + # when + sut = semantic_judge_scorer() + result = await _score( + sut, _task_state(answer, {"tag": "litqa3"}), Target("ref") + ) + + # then + assert result.value == INCORRECT + assert "No answer" in (result.explanation or "") + + +class TestCloningScorer: + async def test_scores_correct_when_reward_passes( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path + ) -> None: + # given a resolvable reference assembly and a passing cloning reward + reference = tmp_path / "clone_1_assembled.fa" + reference.write_text(">ref\nACGT\n") + + async def fake_cloning_reward(**kwargs: Any) -> tuple[float, str]: + # then the scorer forwards files_path and the resolved reference + assert kwargs["base_dir"] == tmp_path + assert kwargs["reference_path"] == reference + return 1.0, "Cloning validation passed" + + monkeypatch.setattr( + "labbench2.cloning.rewards.cloning_reward", fake_cloning_reward + ) + monkeypatch.setattr( + "evals.utils.resolve_file_path", + lambda filename, _: ( + reference if filename == "clone_1_assembled.fa" else None + ), + ) + + # when + sut = cloning_scorer() + state = _task_state( + "assemble", + {"tag": "cloning", "id": "clone_1", "files_path": str(tmp_path)}, + ) + result = await _score(sut, state, Target("")) + + # then + assert result == Score( + value=CORRECT, + answer="assemble", + explanation="Cloning validation passed", + metadata={"cloning_score": 1.0}, + ) + + async def test_scores_incorrect_when_reward_fails( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path + ) -> None: + # given a cloning reward below the pass threshold + async def fake_cloning_reward(**kwargs: Any) -> tuple[float, str]: + return 0.0, "Accuracy failed: output does not match reference" + + monkeypatch.setattr( + "labbench2.cloning.rewards.cloning_reward", fake_cloning_reward + ) + monkeypatch.setattr( + "evals.utils.resolve_file_path", lambda filename, _: tmp_path / filename + ) + + # when + sut = cloning_scorer() + state = _task_state( + "assemble", + {"tag": "cloning", "id": "clone_1", "files_path": str(tmp_path)}, + ) + result = await _score(sut, state, Target("")) + + # then + assert result.value == INCORRECT + assert result.metadata == {"cloning_score": 0.0} + + async def test_incorrect_without_files_path_or_id(self) -> None: + # given metadata missing files_path and id + sut = cloning_scorer() + state = _task_state("assemble", {"tag": "cloning"}) + + # when + result = await _score(sut, state, Target("")) + + # then it fails closed before resolving or scoring + assert result.value == INCORRECT + assert "files_path and id" in (result.explanation or "") + + async def test_incorrect_when_ground_truth_missing( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path + ) -> None: + # given the reference assembly cannot be resolved + monkeypatch.setattr("evals.utils.resolve_file_path", lambda filename, _: None) + + # when + sut = cloning_scorer() + state = _task_state( + "assemble", + {"tag": "cloning", "id": "clone_1", "files_path": str(tmp_path)}, + ) + result = await _score(sut, state, Target("")) + + # then + assert result.value == INCORRECT + assert "Ground truth file not found" in (result.explanation or "") + + +class TestSeqqa2Scorer: + async def test_dispatches_to_validator_and_scores_correct( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + from labbench2.seqqa2.registry import VALIDATORS + + # given a dummy validator registered for a question type + validator = SimpleNamespace(answer_param="answer", func=lambda answer: 1.0) + monkeypatch.setitem(VALIDATORS, "dummy_validator", validator) + + # when + sut = seqqa2_scorer() + state = _task_state( + "pass", + { + "tag": "seqqa2", + "type": "dummy_validator", + "answer_regex": "(?Ppass)", + "validator_params": {}, + }, + ) + result = await _score(sut, state, Target("")) + + # then + assert result == Score( + value=CORRECT, + answer="pass", + explanation="Validator 'dummy_validator' passed", + metadata={"validator": "dummy_validator", "validator_score": 1.0}, + ) + + async def test_renames_answer_param_for_validator( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + from labbench2.seqqa2.registry import VALIDATORS + + captured: dict[str, Any] = {} + + def validator_func(sequence: str) -> float: + captured["sequence"] = sequence + return 1.0 + + # given a validator whose answer param is named "sequence" + validator = SimpleNamespace(answer_param="sequence", func=validator_func) + monkeypatch.setitem(VALIDATORS, "rename_validator", validator) + + # when + sut = seqqa2_scorer() + state = _task_state( + "ACTG", + { + "tag": "seqqa2", + "type": "rename_validator", + "answer_regex": "(?PACTG)", + "validator_params": {}, + }, + ) + result = await _score(sut, state, Target("")) + + # then the extracted answer is passed under the validator's param name + assert result.value == CORRECT + assert captured == {"sequence": "ACTG"} + + async def test_incorrect_for_unknown_validator_type(self) -> None: + sut = seqqa2_scorer() + state = _task_state( + "x", + { + "tag": "seqqa2", + "type": "does_not_exist", + "answer_regex": "(?Px)", + }, + ) + result = await _score(sut, state, Target("")) + assert result.value == INCORRECT + assert "No validator found" in (result.explanation or "") + + async def test_incorrect_when_type_missing(self) -> None: + sut = seqqa2_scorer() + state = _task_state("x", {"tag": "seqqa2"}) + result = await _score(sut, state, Target("")) + assert result.value == INCORRECT + assert "question type" in (result.explanation or "") + + async def test_fail_closed_when_path_param_unresolved( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + from labbench2.seqqa2.registry import VALIDATORS + + # given a validator with a _path param that cannot be resolved + validator = SimpleNamespace(answer_param="answer", func=lambda **kw: 1.0) + monkeypatch.setitem(VALIDATORS, "path_validator", validator) + monkeypatch.setattr("evals.utils.resolve_file_path", lambda value, _: None) + + # when + sut = seqqa2_scorer() + state = _task_state( + "x", + { + "tag": "seqqa2", + "type": "path_validator", + "answer_regex": "(?Px)", + "validator_params": {"reference_path": "missing.fa"}, + }, + ) + result = await _score(sut, state, Target("")) + + # then it fails closed rather than calling the validator + assert result.value == INCORRECT + assert "File not found: missing.fa" in (result.explanation or "") diff --git a/tests/lab_bench_2/test_seqqa2_answer_parser.py b/tests/lab_bench_2/test_seqqa2_answer_parser.py new file mode 100644 index 0000000..4881b65 --- /dev/null +++ b/tests/lab_bench_2/test_seqqa2_answer_parser.py @@ -0,0 +1,68 @@ +from lab_bench_2.seqqa2_answer_parser import ( + _is_nucleotide_only_answer_regex, + _normalized_answer_attempts, + extract, +) + + +class TestExtract: + def test_collapses_ascii_whitespace(self) -> None: + # given a wrapped answer whose regex expects single spaces + extracted = extract( + "FORWARD,\n REVERSE", + r"(?PFORWARD, REVERSE)", + ) + # then whitespace runs collapse to a single space before matching + assert extracted == {"answer": "FORWARD, REVERSE"} + + def test_strips_internal_whitespace_for_nucleotides(self) -> None: + # given spaced sequences and a nucleotide-only regex + extracted = extract( + "ATGC TGCA,\n AATT CCGG", + r"(?P[ATGCatgc]+),(?P[ATGCatgc]+)", + ) + # then internal whitespace is removed so the sequences match + assert extracted == {"forward": "ATGCTGCA", "reverse": "AATTCCGG"} + + def test_accepts_bare_numeric_answer(self) -> None: + extracted = extract("25.67", r"(?P25\.67)") + assert extracted == {"answer": "25.67"} + + def test_accepts_bare_nucleotide_answer(self) -> None: + extracted = extract("ATGC TGCA", r"(?P[ATGCatgc]+)") + assert extracted == {"answer": "ATGCTGCA"} + + def test_accepts_bare_paired_answer(self) -> None: + extracted = extract("931,747", r"(?P\d+),(?P\d+)") + assert extracted == {"left": "931", "right": "747"} + + def test_rejects_invalid_bare_answer(self) -> None: + assert extract("NA,NA", r"(?P\d+),(?P\d+)") is None + + def test_returns_none_without_regex(self) -> None: + assert extract("anything", None) is None + + +class TestIsNucleotideOnlyAnswerRegex: + def test_true_for_nucleotide_regex(self) -> None: + assert _is_nucleotide_only_answer_regex(r"(?P[ATGCatgc]+)") is True + + def test_false_for_word_regex(self) -> None: + assert ( + _is_nucleotide_only_answer_regex(r"(?PFORWARD, REVERSE)") is False + ) + + +class TestNormalizedAnswerAttempts: + def test_collapses_whitespace_for_non_nucleotide(self) -> None: + # given a non-nucleotide regex, only the space-collapsed variant is tried + assert _normalized_answer_attempts("FOR\n WARD", r"(?PFORWARD)") == [ + "FOR WARD" + ] + + def test_adds_stripped_variant_for_nucleotides(self) -> None: + # given a nucleotide regex, also try removing internal whitespace + assert _normalized_answer_attempts("AT GC", r"(?P[ATGC]+)") == [ + "AT GC", + "ATGC", + ]