From d57f3d23065a623b4e4eb561839a650e2b1b9b5c Mon Sep 17 00:00:00 2001 From: Isabelle Phan Date: Thu, 28 May 2026 18:01:45 -0700 Subject: [PATCH 01/11] Add recall judge and enable dbqa2 Reuse the original's recall prompt, but since it has no parseable verdict instruction, so append VERDICT_FORMAT_SUFFIX to emit a `result:` line for parse_judge_verdict. --- src/lab_bench_2/README.md | 31 +++++++++++++++------------ src/lab_bench_2/__init__.py | 2 ++ src/lab_bench_2/lab_bench_2.py | 6 ++++-- src/lab_bench_2/scorers.py | 35 ++++++++++++++++++++++++++----- tests/lab_bench_2/test_scorers.py | 21 +++++++++++++++++-- 5 files changed, 73 insertions(+), 22 deletions(-) diff --git a/src/lab_bench_2/README.md b/src/lab_bench_2/README.md index ce8af75..74ee0c5 100644 --- a/src/lab_bench_2/README.md +++ b/src/lab_bench_2/README.md @@ -62,7 +62,7 @@ See `uv run inspect eval --help` for all available options. ### `lab_bench_2` -- `tag` (str): Which LAB-Bench 2 subset to run. Supported tags: ``litqa3``, ``patentqa``, ``protocolqa2``, ``sourcequality``, ``trialqa``. (default: `'litqa3'`) +- `tag` (str): Which LAB-Bench 2 subset to run. Supported tags: ``dbqa2``, ``litqa3``, ``patentqa``, ``protocolqa2``, ``sourcequality``, ``trialqa``. (default: `'litqa3'`) - `mode` (Mode): How a question's data files are delivered to the model. A no-op for tags without files (such as litqa3). Options: ``file``: Files uploaded via API. PDFs/images attached as context; other files as document attachments., ``inject``: Text file contents concatenated into the prompt as text., ``retrieve``: Only file names/stems are given; prompt instructs the agent to retrieve the necessary sequences or data from a source of its choosing. File contents are withheld. (default: `'inject'`) - `solver` (Solver | None): The solver to run. Defaults to ``bare()`` (the benchmark's "bare" configuration: a plain single-turn ``generate()``) when not provided. Pass any Inspect solver to override, e.g. ``-T solver=bare`` on the CLI. (default: `None`) @@ -73,13 +73,14 @@ This eval uses the public `EdisonScientific/labbench2` dataset on Hugging Face, ### Supported tags -| Tag | Samples | File-bearing | `mode` to use | Notes | -| --------------- | ------- | ------------ | ---------------------- | --------------------------- | -| `litqa3` | 168 | No | any (mode is a no-op) | Literature reasoning. | -| `patentqa` | 121 | No | any (mode is a no-op) | Patent comprehension. | -| `protocolqa2` | 125 | Yes | `file` | Lab protocols. | -| `sourcequality` | 150 | Yes | `file` | Source quality assessment. | -| `trialqa` | 120 | No | any (mode is a no-op) | Clinical trial QA. | +| Tag | Samples | File-bearing | `mode` to use | Notes | +| --------------- | ------- | ------------ | ---------------------- |--------------------------------| +| `dbqa2` | 86 | No | any (mode is a no-op) | Database access; recall judge. | +| `litqa3` | 168 | No | any (mode is a no-op) | Literature reasoning. | +| `patentqa` | 121 | No | any (mode is a no-op) | Patent comprehension. | +| `protocolqa2` | 125 | Yes | `file` | Lab protocols. | +| `sourcequality` | 150 | Yes | `file` | Source quality assessment. | +| `trialqa` | 120 | No | any (mode is a no-op) | Clinical trial QA. | For file-bearing tags, the loader filters out questions that don't opt into the requested `mode` (per each question's `QuestionMode` flags in the HF @@ -91,11 +92,15 @@ sample counts. ## Scoring -Answers are graded by a semantic LLM judge. The judge compares the model's -answer to the reference, accepting semantically or numerically equivalent -answers, and returns one of `correct` / `incorrect` / `unsure`; a `correct` -verdict scores 1.0 and everything else (including unparseable or empty -judgements) scores 0.0. Reported metrics are `accuracy` and `stderr`. +Answers are graded by an LLM judge. The judge compares the model's answer to +the reference, accepting semantically or numerically equivalent answers, and +returns one of `correct` / `incorrect` / `unsure`; a `correct` verdict scores +1.0 and everything else (including unparseable or empty judgements) scores 0.0. +Reported metrics are `accuracy` and `stderr`. + +The judge prompt varies by tag: most tags use the default semantic prompt, while +`dbqa2` (database access) uses a recall-based variant that marks an answer +correct when it recovers the expected reference values. The judge model is selected via the `grader` model role and defaults to `anthropic/claude-sonnet-4-5` at temperature 0. Override it on the command line, diff --git a/src/lab_bench_2/__init__.py b/src/lab_bench_2/__init__.py index 5a9ddfe..3139b97 100644 --- a/src/lab_bench_2/__init__.py +++ b/src/lab_bench_2/__init__.py @@ -10,6 +10,7 @@ from lab_bench_2.scorers import ( DEFAULT_GRADER_MODEL, parse_judge_verdict, + recall_judge_scorer, scorer_for_tag, semantic_judge_scorer, ) @@ -26,6 +27,7 @@ "lab_bench_2", "load_lab_bench_2_dataset", "parse_judge_verdict", + "recall_judge_scorer", "record_to_sample", "scorer_for_tag", "semantic_judge_scorer", diff --git a/src/lab_bench_2/lab_bench_2.py b/src/lab_bench_2/lab_bench_2.py index e0b7f09..ad89071 100644 --- a/src/lab_bench_2/lab_bench_2.py +++ b/src/lab_bench_2/lab_bench_2.py @@ -17,6 +17,7 @@ from utils.metadata import load_version_from_yaml SUPPORTED_TAGS = ( + "dbqa2", "litqa3", "patentqa", "protocolqa2", @@ -36,8 +37,9 @@ def lab_bench_2( """LAB-Bench 2 evaluation task. Args: - tag: Which LAB-Bench 2 subset to run. Supported tags: ``litqa3``, - ``patentqa``, ``protocolqa2``, ``sourcequality``, ``trialqa``. + tag: Which LAB-Bench 2 subset to run. Supported tags: ``dbqa2``, + ``litqa3``, ``patentqa``, ``protocolqa2``, ``sourcequality``, + ``trialqa``. mode: How a question's data files are delivered to the model. A no-op for tags without files (such as litqa3). Options: diff --git a/src/lab_bench_2/scorers.py b/src/lab_bench_2/scorers.py index 56916b6..1874ee9 100644 --- a/src/lab_bench_2/scorers.py +++ b/src/lab_bench_2/scorers.py @@ -28,11 +28,17 @@ re.IGNORECASE, ) +# The recall prompt (unlike the semantic and exact-match prompts) does not ask +# the model for a parseable verdict line, so we append one carrying the +# ``result:`` marker that ``parse_judge_verdict`` reads. +VERDICT_FORMAT_SUFFIX = ( + "\n\nAfter your analysis, end your response with a final line in exactly " + "this form:\n\nresult: " +) -@scorer(metrics=[accuracy(), stderr()]) -def semantic_judge_scorer() -> Scorer: - """Grade an open-ended answer against the reference using a judge model.""" - from evals.prompts import STRUCTURED_EVALUATION_PROMPT + +def _judge_score(prompt_template: str) -> Scorer: + """Build a judge that grades an answer against the reference via the template.""" async def score(state: TaskState, target: Target) -> Score: answer = state.output.completion.strip() @@ -47,7 +53,7 @@ async def score(state: TaskState, target: Target) -> Score: config=GenerateConfig(temperature=0.0), ) - prompt = STRUCTURED_EVALUATION_PROMPT.format( + prompt = prompt_template.format( question=state.input_text, correct_answer=target.text, answer=answer, @@ -65,7 +71,26 @@ async def score(state: TaskState, target: Target) -> Score: return score +@scorer(metrics=[accuracy(), stderr()]) +def semantic_judge_scorer() -> Scorer: + """Grade an open-ended answer against the reference using a judge model.""" + from evals.prompts import STRUCTURED_EVALUATION_PROMPT + + return _judge_score(STRUCTURED_EVALUATION_PROMPT) + + +@scorer(metrics=[accuracy(), stderr()]) +def recall_judge_scorer() -> Scorer: + """Grade a dbqa2 answer by recall of the expected values (data-access bench).""" + from evals.prompts import STRUCTURED_EVALUATION_PROMPT_DATA_ACCESS_BENCH_RECALL + + return _judge_score( + STRUCTURED_EVALUATION_PROMPT_DATA_ACCESS_BENCH_RECALL + VERDICT_FORMAT_SUFFIX + ) + + SCORERS_BY_TAG = { + "dbqa2": recall_judge_scorer, "litqa3": semantic_judge_scorer, "patentqa": semantic_judge_scorer, "protocolqa2": semantic_judge_scorer, diff --git a/tests/lab_bench_2/test_scorers.py b/tests/lab_bench_2/test_scorers.py index 32db3af..f7889dc 100644 --- a/tests/lab_bench_2/test_scorers.py +++ b/tests/lab_bench_2/test_scorers.py @@ -2,7 +2,11 @@ from inspect_ai.scorer import Scorer from lab_bench_2 import parse_judge_verdict -from lab_bench_2.scorers import scorer_for_tag, semantic_judge_scorer +from lab_bench_2.scorers import ( + recall_judge_scorer, + scorer_for_tag, + semantic_judge_scorer, +) class TestParseJudgeVerdict: @@ -44,11 +48,20 @@ def test_last_verdict_wins(self) -> None: # when / then assert parse_judge_verdict(text) == "correct" + def test_parses_recall_style_output_with_format_suffix(self) -> None: + # given a recall-style judgement that echoes the rubric, then closes + # with the verdict line that VERDICT_FORMAT_SUFFIX instructs + text = ( + "Matched 5/6 expected variables. Recall = 0.83 < 0.95.\nresult: incorrect" + ) + # when / then + assert parse_judge_verdict(text) == "incorrect" + class TestScorerForTag: @pytest.mark.parametrize( "tag", - ["litqa3", "patentqa", "protocolqa2", "sourcequality", "trialqa"], + ["dbqa2", "litqa3", "patentqa", "protocolqa2", "sourcequality", "trialqa"], ) def test_returns_scorer_for_supported_tag(self, tag: str) -> None: assert isinstance(scorer_for_tag(tag), Scorer) @@ -60,3 +73,7 @@ def test_unsupported_tag_raises(self) -> None: def test_semantic_judge_scorer_is_scorer() -> None: assert isinstance(semantic_judge_scorer(), Scorer) + + +def test_recall_judge_scorer_is_scorer() -> None: + assert isinstance(recall_judge_scorer(), Scorer) From 8164c0d4b4aedad11acea2103fab4a5a7b2f1d52 Mon Sep 17 00:00:00 2001 From: Isabelle Phan Date: Thu, 28 May 2026 18:15:13 -0700 Subject: [PATCH 02/11] Add exact-match judge and enable figure/table/supplement tags Route figqa2*/tableqa2*/suppqa2 to an exact-match LLM judge using STRUCTURED_EVALUATION_PROMPT_EXACT_MATCH Enable the seven configs in SUPPORTED_TAGS. The base figqa2/tableqa2/suppqa2 configs are text-only (mode is a no-op); the -img/-pdf variants are file-bearing and support only `file` mode. --- src/lab_bench_2/README.md | 27 +++++++++++++++++++-------- src/lab_bench_2/__init__.py | 2 ++ src/lab_bench_2/lab_bench_2.py | 12 ++++++++++-- src/lab_bench_2/scorers.py | 15 +++++++++++++++ tests/lab_bench_2/test_scorers.py | 17 ++++++++++++----- 5 files changed, 58 insertions(+), 15 deletions(-) diff --git a/src/lab_bench_2/README.md b/src/lab_bench_2/README.md index 74ee0c5..5d1879a 100644 --- a/src/lab_bench_2/README.md +++ b/src/lab_bench_2/README.md @@ -62,7 +62,7 @@ See `uv run inspect eval --help` for all available options. ### `lab_bench_2` -- `tag` (str): Which LAB-Bench 2 subset to run. Supported tags: ``dbqa2``, ``litqa3``, ``patentqa``, ``protocolqa2``, ``sourcequality``, ``trialqa``. (default: `'litqa3'`) +- `tag` (str): Which LAB-Bench 2 subset to run. Supported tags: ``dbqa2``, ``figqa2`` (and ``figqa2-img`` / ``figqa2-pdf``), ``litqa3``, ``patentqa``, ``protocolqa2``, ``sourcequality``, ``suppqa2``, ``tableqa2`` (and ``tableqa2-img`` / ``tableqa2-pdf``), ``trialqa``. (default: `'litqa3'`) - `mode` (Mode): How a question's data files are delivered to the model. A no-op for tags without files (such as litqa3). Options: ``file``: Files uploaded via API. PDFs/images attached as context; other files as document attachments., ``inject``: Text file contents concatenated into the prompt as text., ``retrieve``: Only file names/stems are given; prompt instructs the agent to retrieve the necessary sequences or data from a source of its choosing. File contents are withheld. (default: `'inject'`) - `solver` (Solver | None): The solver to run. Defaults to ``bare()`` (the benchmark's "bare" configuration: a plain single-turn ``generate()``) when not provided. Pass any Inspect solver to override, e.g. ``-T solver=bare`` on the CLI. (default: `None`) @@ -76,19 +76,28 @@ This eval uses the public `EdisonScientific/labbench2` dataset on Hugging Face, | Tag | Samples | File-bearing | `mode` to use | Notes | | --------------- | ------- | ------------ | ---------------------- |--------------------------------| | `dbqa2` | 86 | No | any (mode is a no-op) | Database access; recall judge. | +| `figqa2` | 101 | No | any (mode is a no-op) | Figure QA; exact-match judge. | +| `figqa2-img` | 101 | Yes | `file` | Figure QA with image files. | +| `figqa2-pdf` | 101 | Yes | `file` | Figure QA with PDF files. | | `litqa3` | 168 | No | any (mode is a no-op) | Literature reasoning. | | `patentqa` | 121 | No | any (mode is a no-op) | Patent comprehension. | | `protocolqa2` | 125 | Yes | `file` | Lab protocols. | | `sourcequality` | 150 | Yes | `file` | Source quality assessment. | +| `suppqa2` | 125 | No | any (mode is a no-op) | Supplement QA; exact-match. | +| `tableqa2` | 100 | No | any (mode is a no-op) | Table QA; exact-match judge. | +| `tableqa2-img` | 100 | Yes | `file` | Table QA with image files. | +| `tableqa2-pdf` | 100 | Yes | `file` | Table QA with PDF files. | | `trialqa` | 120 | No | any (mode is a no-op) | Clinical trial QA. | For file-bearing tags, the loader filters out questions that don't opt into the requested `mode` (per each question's `QuestionMode` flags in the HF -data). At the time of writing every `protocolqa2` and `sourcequality` -question opts into `file` only, so passing any other `mode` would load zero -samples. The mode flags live in the dataset and may change — verify by -running with the configuration you intend before drawing conclusions from -sample counts. +data). At the time of writing every file-bearing tag (`protocolqa2`, +`sourcequality`, `figqa2-img`, `figqa2-pdf`, `tableqa2-img`, `tableqa2-pdf`) +opts into `file` only, so passing any other `mode` would load zero samples. +Note that the base `figqa2`, `tableqa2`, and `suppqa2` configs carry no files +(mode is a no-op); the image/PDF variants are the file-bearing ones. The mode +flags live in the dataset and may change — verify by running with the +configuration you intend before drawing conclusions from sample counts. ## Scoring @@ -98,9 +107,11 @@ returns one of `correct` / `incorrect` / `unsure`; a `correct` verdict scores 1.0 and everything else (including unparseable or empty judgements) scores 0.0. Reported metrics are `accuracy` and `stderr`. -The judge prompt varies by tag: most tags use the default semantic prompt, while +The judge prompt varies by tag: most tags use the default semantic prompt; `dbqa2` (database access) uses a recall-based variant that marks an answer -correct when it recovers the expected reference values. +correct when it recovers the expected reference values; and the figure, table, +and supplement tags (`figqa2*`, `tableqa2*`, `suppqa2`) use an exact-match +variant for numeric answers. The judge model is selected via the `grader` model role and defaults to `anthropic/claude-sonnet-4-5` at temperature 0. Override it on the command line, diff --git a/src/lab_bench_2/__init__.py b/src/lab_bench_2/__init__.py index 3139b97..fc5b2d8 100644 --- a/src/lab_bench_2/__init__.py +++ b/src/lab_bench_2/__init__.py @@ -9,6 +9,7 @@ from lab_bench_2.prompt_composer import Mode from lab_bench_2.scorers import ( DEFAULT_GRADER_MODEL, + exact_match_judge_scorer, parse_judge_verdict, recall_judge_scorer, scorer_for_tag, @@ -24,6 +25,7 @@ "SUPPORTED_TAGS", "Mode", "bare", + "exact_match_judge_scorer", "lab_bench_2", "load_lab_bench_2_dataset", "parse_judge_verdict", diff --git a/src/lab_bench_2/lab_bench_2.py b/src/lab_bench_2/lab_bench_2.py index ad89071..25e3dc8 100644 --- a/src/lab_bench_2/lab_bench_2.py +++ b/src/lab_bench_2/lab_bench_2.py @@ -18,10 +18,17 @@ SUPPORTED_TAGS = ( "dbqa2", + "figqa2", + "figqa2-img", + "figqa2-pdf", "litqa3", "patentqa", "protocolqa2", "sourcequality", + "suppqa2", + "tableqa2", + "tableqa2-img", + "tableqa2-pdf", "trialqa", ) @@ -38,8 +45,9 @@ def lab_bench_2( Args: tag: Which LAB-Bench 2 subset to run. Supported tags: ``dbqa2``, - ``litqa3``, ``patentqa``, ``protocolqa2``, ``sourcequality``, - ``trialqa``. + ``figqa2`` (and ``figqa2-img`` / ``figqa2-pdf``), ``litqa3``, + ``patentqa``, ``protocolqa2``, ``sourcequality``, ``suppqa2``, + ``tableqa2`` (and ``tableqa2-img`` / ``tableqa2-pdf``), ``trialqa``. mode: How a question's data files are delivered to the model. A no-op for tags without files (such as litqa3). Options: diff --git a/src/lab_bench_2/scorers.py b/src/lab_bench_2/scorers.py index 1874ee9..3f6b9df 100644 --- a/src/lab_bench_2/scorers.py +++ b/src/lab_bench_2/scorers.py @@ -89,12 +89,27 @@ def recall_judge_scorer() -> Scorer: ) +@scorer(metrics=[accuracy(), stderr()]) +def exact_match_judge_scorer() -> Scorer: + """Grade a figure/table/supplement answer by exact numeric match.""" + from evals.prompts import STRUCTURED_EVALUATION_PROMPT_EXACT_MATCH + + return _judge_score(STRUCTURED_EVALUATION_PROMPT_EXACT_MATCH) + + SCORERS_BY_TAG = { "dbqa2": recall_judge_scorer, + "figqa2": exact_match_judge_scorer, + "figqa2-img": exact_match_judge_scorer, + "figqa2-pdf": exact_match_judge_scorer, "litqa3": semantic_judge_scorer, "patentqa": semantic_judge_scorer, "protocolqa2": semantic_judge_scorer, "sourcequality": semantic_judge_scorer, + "suppqa2": exact_match_judge_scorer, + "tableqa2": exact_match_judge_scorer, + "tableqa2-img": exact_match_judge_scorer, + "tableqa2-pdf": exact_match_judge_scorer, "trialqa": semantic_judge_scorer, } diff --git a/tests/lab_bench_2/test_scorers.py b/tests/lab_bench_2/test_scorers.py index f7889dc..d022c9d 100644 --- a/tests/lab_bench_2/test_scorers.py +++ b/tests/lab_bench_2/test_scorers.py @@ -1,8 +1,10 @@ import pytest from inspect_ai.scorer import Scorer -from lab_bench_2 import parse_judge_verdict +from lab_bench_2 import SUPPORTED_TAGS, parse_judge_verdict from lab_bench_2.scorers import ( + SCORERS_BY_TAG, + exact_match_judge_scorer, recall_judge_scorer, scorer_for_tag, semantic_judge_scorer, @@ -59,13 +61,14 @@ def test_parses_recall_style_output_with_format_suffix(self) -> None: class TestScorerForTag: - @pytest.mark.parametrize( - "tag", - ["dbqa2", "litqa3", "patentqa", "protocolqa2", "sourcequality", "trialqa"], - ) + @pytest.mark.parametrize("tag", sorted(SCORERS_BY_TAG)) def test_returns_scorer_for_supported_tag(self, tag: str) -> None: assert isinstance(scorer_for_tag(tag), Scorer) + def test_routing_table_matches_supported_tags(self) -> None: + # given/when/then — the task gate and the scorer routing list the same tags + assert set(SCORERS_BY_TAG) == set(SUPPORTED_TAGS) + def test_unsupported_tag_raises(self) -> None: with pytest.raises(NotImplementedError): scorer_for_tag("seqqa2") @@ -77,3 +80,7 @@ def test_semantic_judge_scorer_is_scorer() -> None: def test_recall_judge_scorer_is_scorer() -> None: assert isinstance(recall_judge_scorer(), Scorer) + + +def test_exact_match_judge_scorer_is_scorer() -> None: + assert isinstance(exact_match_judge_scorer(), Scorer) From 4f335abbb444baa2b8b35ec0178c305e124cfa93 Mon Sep 17 00:00:00 2001 From: Isabelle Phan Date: Thu, 28 May 2026 20:49:44 -0700 Subject: [PATCH 03/11] Port cloning scorer and enable the cloning tag Code lifted from UKGovernmentBEIS/inspect_evals#1307 Add cloning_scorer, scoring CloningQA protocols via labbench2's cloning_reward pipeline. Call upstream cloning_reward directly (no local fork). Enable cloning in SUPPORTED_TAGS (14 samples, file-bearing). Extend the mypy override to cover the labbench2 package. Co-Authored-By: Lewis Tunstall --- pyproject.toml | 2 +- src/lab_bench_2/README.md | 44 ++++++++++++++++---------- src/lab_bench_2/__init__.py | 2 ++ src/lab_bench_2/lab_bench_2.py | 10 +++--- src/lab_bench_2/scorers.py | 57 ++++++++++++++++++++++++++++++++++ 5 files changed, 94 insertions(+), 21 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index c0b2632..4630746 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -108,7 +108,7 @@ ignore_missing_imports = true [[tool.mypy.overrides]] # Upstream labbench2 git dep has no py.typed marker / stubs. -module = ["evals", "evals.*"] +module = ["evals", "evals.*", "labbench2", "labbench2.*"] ignore_missing_imports = true [[tool.mypy.overrides]] diff --git a/src/lab_bench_2/README.md b/src/lab_bench_2/README.md index 5d1879a..3c9eb91 100644 --- a/src/lab_bench_2/README.md +++ b/src/lab_bench_2/README.md @@ -62,7 +62,7 @@ See `uv run inspect eval --help` for all available options. ### `lab_bench_2` -- `tag` (str): Which LAB-Bench 2 subset to run. Supported tags: ``dbqa2``, ``figqa2`` (and ``figqa2-img`` / ``figqa2-pdf``), ``litqa3``, ``patentqa``, ``protocolqa2``, ``sourcequality``, ``suppqa2``, ``tableqa2`` (and ``tableqa2-img`` / ``tableqa2-pdf``), ``trialqa``. (default: `'litqa3'`) +- `tag` (str): Which LAB-Bench 2 subset to run. Supported tags: ``cloning``, ``dbqa2``, ``figqa2`` (and ``figqa2-img`` / ``figqa2-pdf``), ``litqa3``, ``patentqa``, ``protocolqa2``, ``sourcequality``, ``suppqa2``, ``tableqa2`` (and ``tableqa2-img`` / ``tableqa2-pdf``), ``trialqa``. (default: `'litqa3'`) - `mode` (Mode): How a question's data files are delivered to the model. A no-op for tags without files (such as litqa3). Options: ``file``: Files uploaded via API. PDFs/images attached as context; other files as document attachments., ``inject``: Text file contents concatenated into the prompt as text., ``retrieve``: Only file names/stems are given; prompt instructs the agent to retrieve the necessary sequences or data from a source of its choosing. File contents are withheld. (default: `'inject'`) - `solver` (Solver | None): The solver to run. Defaults to ``bare()`` (the benchmark's "bare" configuration: a plain single-turn ``generate()``) when not provided. Pass any Inspect solver to override, e.g. ``-T solver=bare`` on the CLI. (default: `None`) @@ -73,21 +73,22 @@ This eval uses the public `EdisonScientific/labbench2` dataset on Hugging Face, ### Supported tags -| Tag | Samples | File-bearing | `mode` to use | Notes | -| --------------- | ------- | ------------ | ---------------------- |--------------------------------| -| `dbqa2` | 86 | No | any (mode is a no-op) | Database access; recall judge. | -| `figqa2` | 101 | No | any (mode is a no-op) | Figure QA; exact-match judge. | -| `figqa2-img` | 101 | Yes | `file` | Figure QA with image files. | -| `figqa2-pdf` | 101 | Yes | `file` | Figure QA with PDF files. | -| `litqa3` | 168 | No | any (mode is a no-op) | Literature reasoning. | -| `patentqa` | 121 | No | any (mode is a no-op) | Patent comprehension. | -| `protocolqa2` | 125 | Yes | `file` | Lab protocols. | -| `sourcequality` | 150 | Yes | `file` | Source quality assessment. | -| `suppqa2` | 125 | No | any (mode is a no-op) | Supplement QA; exact-match. | -| `tableqa2` | 100 | No | any (mode is a no-op) | Table QA; exact-match judge. | -| `tableqa2-img` | 100 | Yes | `file` | Table QA with image files. | -| `tableqa2-pdf` | 100 | Yes | `file` | Table QA with PDF files. | -| `trialqa` | 120 | No | any (mode is a no-op) | Clinical trial QA. | +| Tag | Samples | File-bearing | `mode` to use | Notes | +| --------------- | ------- | ------------ | ---------------------- |-----------------------------------| +| `cloning` | 14 | Yes | any (all modes) | Cloning protocols; reward-scored. | +| `dbqa2` | 86 | No | any (mode is a no-op) | Database access; recall judge. | +| `figqa2` | 101 | No | any (mode is a no-op) | Figure QA; exact-match judge. | +| `figqa2-img` | 101 | Yes | `file` | Figure QA with image files. | +| `figqa2-pdf` | 101 | Yes | `file` | Figure QA with PDF files. | +| `litqa3` | 168 | No | any (mode is a no-op) | Literature reasoning. | +| `patentqa` | 121 | No | any (mode is a no-op) | Patent comprehension. | +| `protocolqa2` | 125 | Yes | `file` | Lab protocols. | +| `sourcequality` | 150 | Yes | `file` | Source quality assessment. | +| `suppqa2` | 125 | No | any (mode is a no-op) | Supplement QA; exact-match. | +| `tableqa2` | 100 | No | any (mode is a no-op) | Table QA; exact-match judge. | +| `tableqa2-img` | 100 | Yes | `file` | Table QA with image files. | +| `tableqa2-pdf` | 100 | Yes | `file` | Table QA with PDF files. | +| `trialqa` | 120 | No | any (mode is a no-op) | Clinical trial QA. | For file-bearing tags, the loader filters out questions that don't opt into the requested `mode` (per each question's `QuestionMode` flags in the HF @@ -113,6 +114,17 @@ correct when it recovers the expected reference values; and the figure, table, and supplement tags (`figqa2*`, `tableqa2*`, `suppqa2`) use an exact-match variant for numeric answers. +The `cloning` tag is not graded by an LLM judge: it is scored deterministically +by labbench2's reward pipeline, which parses the submitted protocol, executes it +(including PCR simulation), and compares the result to the reference assembly via +sequence-similarity and restriction-digest checks. +PCR simulation runs a small Go binary that labbench2 compiles on first use, so a +Go toolchain (1.21+) must be available on the host the first time you score +`cloning`; the compiled binary is then cached inside the installed package and +reused. To install Go: `brew install go` (macOS), `sudo apt install golang-go` (Linux), +or . Without Go, protocol execution fails gracefully: PCR-based +samples score 0.0 with an explanatory reason rather than crashing the run. + The judge model is selected via the `grader` model role and defaults to `anthropic/claude-sonnet-4-5` at temperature 0. Override it on the command line, for example: diff --git a/src/lab_bench_2/__init__.py b/src/lab_bench_2/__init__.py index fc5b2d8..cb085a1 100644 --- a/src/lab_bench_2/__init__.py +++ b/src/lab_bench_2/__init__.py @@ -9,6 +9,7 @@ from lab_bench_2.prompt_composer import Mode from lab_bench_2.scorers import ( DEFAULT_GRADER_MODEL, + cloning_scorer, exact_match_judge_scorer, parse_judge_verdict, recall_judge_scorer, @@ -25,6 +26,7 @@ "SUPPORTED_TAGS", "Mode", "bare", + "cloning_scorer", "exact_match_judge_scorer", "lab_bench_2", "load_lab_bench_2_dataset", diff --git a/src/lab_bench_2/lab_bench_2.py b/src/lab_bench_2/lab_bench_2.py index 25e3dc8..3053e7e 100644 --- a/src/lab_bench_2/lab_bench_2.py +++ b/src/lab_bench_2/lab_bench_2.py @@ -17,6 +17,7 @@ from utils.metadata import load_version_from_yaml SUPPORTED_TAGS = ( + "cloning", "dbqa2", "figqa2", "figqa2-img", @@ -44,10 +45,11 @@ def lab_bench_2( """LAB-Bench 2 evaluation task. Args: - tag: Which LAB-Bench 2 subset to run. Supported tags: ``dbqa2``, - ``figqa2`` (and ``figqa2-img`` / ``figqa2-pdf``), ``litqa3``, - ``patentqa``, ``protocolqa2``, ``sourcequality``, ``suppqa2``, - ``tableqa2`` (and ``tableqa2-img`` / ``tableqa2-pdf``), ``trialqa``. + tag: Which LAB-Bench 2 subset to run. Supported tags: ``cloning``, + ``dbqa2``, ``figqa2`` (and ``figqa2-img`` / ``figqa2-pdf``), + ``litqa3``, ``patentqa``, ``protocolqa2``, ``sourcequality``, + ``suppqa2``, ``tableqa2`` (and ``tableqa2-img`` / ``tableqa2-pdf``), + ``trialqa``. mode: How a question's data files are delivered to the model. A no-op for tags without files (such as litqa3). Options: diff --git a/src/lab_bench_2/scorers.py b/src/lab_bench_2/scorers.py index 3f6b9df..aa97e1c 100644 --- a/src/lab_bench_2/scorers.py +++ b/src/lab_bench_2/scorers.py @@ -3,6 +3,8 @@ from __future__ import annotations import re +from pathlib import Path +from typing import Any, cast from inspect_ai.model import GenerateConfig, get_model from inspect_ai.scorer import ( @@ -97,7 +99,62 @@ def exact_match_judge_scorer() -> Scorer: return _judge_score(STRUCTURED_EVALUATION_PROMPT_EXACT_MATCH) +@scorer(metrics=[accuracy(), stderr()]) +def cloning_scorer() -> Scorer: + """Score CloningQA answers using labbench2's cloning reward pipeline. + + Validates cloning protocols through 4 sequential stages: + 1. Format validation — protocol can be parsed + 2. Execution — protocol runs successfully + 3. Similarity — output matches reference sequence (threshold: 0.95) + 4. Digest — restriction enzyme fragments match + + Requires Go 1.21+ on the host for PCR simulation scoring. + """ + from evals.utils import resolve_file_path + from labbench2.cloning.rewards import cloning_reward + + async def score(state: TaskState, target: Target) -> Score: + metadata = state.metadata or {} + files_path_str = metadata.get("files_path") + question_id = cast(str | None, metadata.get("id")) + + if not files_path_str or not question_id: + return Score( + value=0.0, + explanation="Cloning evaluation requires files_path and id metadata.", + metadata={"route": "cloning"}, + ) + + ground_truth_filename = f"{question_id}_assembled.fa" + ground_truth_path = resolve_file_path(ground_truth_filename, None) + if ground_truth_path is None: + return Score( + value=0.0, + explanation=f"Ground truth file not found: {ground_truth_filename}", + metadata={"route": "cloning"}, + ) + + score_value, reason = await cloning_reward( + answer=state.output.completion, + base_dir=Path(files_path_str), + reference_path=ground_truth_path, + validator_params=cast( + dict[str, Any] | None, metadata.get("validator_params") or None + ), + ) + + return Score( + value=float(score_value), + explanation=reason, + metadata={"route": "cloning"}, + ) + + return score + + SCORERS_BY_TAG = { + "cloning": cloning_scorer, "dbqa2": recall_judge_scorer, "figqa2": exact_match_judge_scorer, "figqa2-img": exact_match_judge_scorer, From 5f10e171812c43cffdb09e492e9eccfd55ea14d2 Mon Sep 17 00:00:00 2001 From: Isabelle Phan Date: Thu, 28 May 2026 21:09:06 -0700 Subject: [PATCH 04/11] Tidy cloning scorer and add unit tests Map the cloning reward to CORRECT/INCORRECT (was a raw float), record the numeric reward under metadata["cloning_score"], and drop the vestigial "route" metadata. Pass validator_params as a plain dict defaulting to {}, matching the loader's parsed shape. --- src/lab_bench_2/scorers.py | 12 ++-- tests/lab_bench_2/test_scorers.py | 116 +++++++++++++++++++++++++++++- 2 files changed, 120 insertions(+), 8 deletions(-) diff --git a/src/lab_bench_2/scorers.py b/src/lab_bench_2/scorers.py index aa97e1c..c0e2540 100644 --- a/src/lab_bench_2/scorers.py +++ b/src/lab_bench_2/scorers.py @@ -121,18 +121,16 @@ async def score(state: TaskState, target: Target) -> Score: if not files_path_str or not question_id: return Score( - value=0.0, + value=INCORRECT, explanation="Cloning evaluation requires files_path and id metadata.", - metadata={"route": "cloning"}, ) ground_truth_filename = f"{question_id}_assembled.fa" ground_truth_path = resolve_file_path(ground_truth_filename, None) if ground_truth_path is None: return Score( - value=0.0, + value=INCORRECT, explanation=f"Ground truth file not found: {ground_truth_filename}", - metadata={"route": "cloning"}, ) score_value, reason = await cloning_reward( @@ -140,14 +138,14 @@ async def score(state: TaskState, target: Target) -> Score: base_dir=Path(files_path_str), reference_path=ground_truth_path, validator_params=cast( - dict[str, Any] | None, metadata.get("validator_params") or None + dict[str, Any], metadata.get("validator_params") or {} ), ) return Score( - value=float(score_value), + value=CORRECT if score_value >= 1.0 else INCORRECT, explanation=reason, - metadata={"route": "cloning"}, + metadata={"cloning_score": score_value}, ) return score diff --git a/tests/lab_bench_2/test_scorers.py b/tests/lab_bench_2/test_scorers.py index d022c9d..0898b98 100644 --- a/tests/lab_bench_2/test_scorers.py +++ b/tests/lab_bench_2/test_scorers.py @@ -1,9 +1,15 @@ +from pathlib import Path +from typing import Any + import pytest -from inspect_ai.scorer import Scorer +from inspect_ai.model import ModelOutput +from inspect_ai.scorer import CORRECT, INCORRECT, Score, Scorer, Target +from inspect_ai.solver import TaskState from lab_bench_2 import SUPPORTED_TAGS, parse_judge_verdict from lab_bench_2.scorers import ( SCORERS_BY_TAG, + cloning_scorer, exact_match_judge_scorer, recall_judge_scorer, scorer_for_tag, @@ -11,6 +17,18 @@ ) +def _task_state(completion: str, metadata: dict[str, Any]) -> TaskState: + return TaskState( + model="mockllm/model", + sample_id="sample-1", + epoch=1, + input="Question?", + messages=[], + output=ModelOutput.from_content("mockllm/model", completion), + metadata=metadata, + ) + + class TestParseJudgeVerdict: @pytest.mark.parametrize( "verdict", @@ -84,3 +102,99 @@ def test_recall_judge_scorer_is_scorer() -> None: def test_exact_match_judge_scorer_is_scorer() -> None: assert isinstance(exact_match_judge_scorer(), Scorer) + + +class TestCloningScorer: + async def test_scores_correct_when_reward_passes( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path + ) -> None: + # given a resolvable reference assembly and a passing cloning reward + reference = tmp_path / "clone_1_assembled.fa" + reference.write_text(">ref\nACGT\n") + + async def fake_cloning_reward(**kwargs: Any) -> tuple[float, str]: + # then the scorer forwards files_path and the resolved reference + assert kwargs["base_dir"] == tmp_path + assert kwargs["reference_path"] == reference + return 1.0, "Cloning validation passed" + + monkeypatch.setattr( + "labbench2.cloning.rewards.cloning_reward", fake_cloning_reward + ) + monkeypatch.setattr( + "evals.utils.resolve_file_path", + lambda filename, _: ( + reference if filename == "clone_1_assembled.fa" else None + ), + ) + + # when + sut = cloning_scorer() + state = _task_state( + "assemble", + {"tag": "cloning", "id": "clone_1", "files_path": str(tmp_path)}, + ) + result = await sut(state, Target("")) + + # then + assert result == Score( + value=CORRECT, + explanation="Cloning validation passed", + metadata={"cloning_score": 1.0}, + ) + + async def test_scores_incorrect_when_reward_fails( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path + ) -> None: + # given a cloning reward below the pass threshold + async def fake_cloning_reward(**kwargs: Any) -> tuple[float, str]: + return 0.0, "Accuracy failed: output does not match reference" + + monkeypatch.setattr( + "labbench2.cloning.rewards.cloning_reward", fake_cloning_reward + ) + monkeypatch.setattr( + "evals.utils.resolve_file_path", lambda filename, _: tmp_path / filename + ) + + # when + sut = cloning_scorer() + state = _task_state( + "assemble", + {"tag": "cloning", "id": "clone_1", "files_path": str(tmp_path)}, + ) + result = await sut(state, Target("")) + + # then + assert result.value == INCORRECT + assert result.metadata == {"cloning_score": 0.0} + + async def test_incorrect_without_files_path_or_id(self) -> None: + # given metadata missing files_path and id + sut = cloning_scorer() + state = _task_state("assemble", {"tag": "cloning"}) + + # when + result = await sut(state, Target("")) + + # then it fails closed before resolving or scoring + assert result.value == INCORRECT + assert "files_path and id" in (result.explanation or "") + + async def test_incorrect_when_ground_truth_missing( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path + ) -> None: + # given the reference assembly cannot be resolved + monkeypatch.setattr("evals.utils.resolve_file_path", lambda filename, _: None) + + # when + sut = cloning_scorer() + state = _task_state( + "assemble", + {"tag": "cloning", "id": "clone_1", "files_path": str(tmp_path)}, + ) + result = await sut(state, Target("")) + + # then + assert result.value == INCORRECT + assert "Ground truth file not found" in (result.explanation or "") From fecb9f3d7a3c8bda4293ebe149a9a546689060c1 Mon Sep 17 00:00:00 2001 From: Isabelle Phan Date: Thu, 28 May 2026 21:26:52 -0700 Subject: [PATCH 05/11] Don't parse verdicts from code-like grader output parse_judge_verdict matched `result = "correct"` inside a grader's code-dump response scoring it CORRECT. Exclude `=` from the separator so assignments no longer match; every supported verdict format (colon, arrow, markdown) still parses. --- src/lab_bench_2/scorers.py | 4 +++- tests/lab_bench_2/test_scorers.py | 7 +++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/lab_bench_2/scorers.py b/src/lab_bench_2/scorers.py index c0e2540..f74aec1 100644 --- a/src/lab_bench_2/scorers.py +++ b/src/lab_bench_2/scorers.py @@ -25,8 +25,10 @@ JUDGE_VERDICT_CORRECT = "correct" JUDGE_VERDICT_INCORRECT = "incorrect" JUDGE_VERDICT_UNSURE = "unsure" +# Excluding ``=`` from the separator stops the pattern from matching code-like +# grader output such as ``result = "correct"`` (an assignment, not a verdict). _GRADE_PATTERN = re.compile( - r"\bresult\b[^A-Za-z]*(correct|incorrect|unsure)\b", + r"\bresult\b[^A-Za-z=]*(correct|incorrect|unsure)\b", re.IGNORECASE, ) diff --git a/tests/lab_bench_2/test_scorers.py b/tests/lab_bench_2/test_scorers.py index 0898b98..2b3f869 100644 --- a/tests/lab_bench_2/test_scorers.py +++ b/tests/lab_bench_2/test_scorers.py @@ -77,6 +77,13 @@ def test_parses_recall_style_output_with_format_suffix(self) -> None: # when / then assert parse_judge_verdict(text) == "incorrect" + def test_ignores_code_assignment(self) -> None: + # given grader output that is code rather than a verdict — `result = + # "correct"` is an assignment, not a graded result + text = ' result = "unknown"\n result = "correct"\n return result' + # when / then + assert parse_judge_verdict(text) is None + class TestScorerForTag: @pytest.mark.parametrize("tag", sorted(SCORERS_BY_TAG)) From efe825abd656adeadd5b78e28e5bc63c61b17fbd Mon Sep 17 00:00:00 2001 From: Isabelle Phan Date: Thu, 28 May 2026 21:45:59 -0700 Subject: [PATCH 06/11] Port seqqa2 scorer and enable the seqqa2 tag Code lifted from UKGovernmentBEIS/inspect_evals#1307 Add seqqa2_scorer, dispatching to labbench2's per-type VALIDATORS. Co-Authored-By: Lewis Tunstall --- src/lab_bench_2/README.md | 44 ++++++---- src/lab_bench_2/__init__.py | 2 + src/lab_bench_2/lab_bench_2.py | 7 +- src/lab_bench_2/scorers.py | 137 ++++++++++++++++++++++++++++++ tests/lab_bench_2/test_e2e.py | 2 +- tests/lab_bench_2/test_scorers.py | 2 +- 6 files changed, 171 insertions(+), 23 deletions(-) diff --git a/src/lab_bench_2/README.md b/src/lab_bench_2/README.md index 3c9eb91..fa93a1c 100644 --- a/src/lab_bench_2/README.md +++ b/src/lab_bench_2/README.md @@ -62,7 +62,7 @@ See `uv run inspect eval --help` for all available options. ### `lab_bench_2` -- `tag` (str): Which LAB-Bench 2 subset to run. Supported tags: ``cloning``, ``dbqa2``, ``figqa2`` (and ``figqa2-img`` / ``figqa2-pdf``), ``litqa3``, ``patentqa``, ``protocolqa2``, ``sourcequality``, ``suppqa2``, ``tableqa2`` (and ``tableqa2-img`` / ``tableqa2-pdf``), ``trialqa``. (default: `'litqa3'`) +- `tag` (str): Which LAB-Bench 2 subset to run. Supported tags: ``cloning``, ``dbqa2``, ``figqa2`` (and ``figqa2-img`` / ``figqa2-pdf``), ``litqa3``, ``patentqa``, ``protocolqa2``, ``seqqa2``, ``sourcequality``, ``suppqa2``, ``tableqa2`` (and ``tableqa2-img`` / ``tableqa2-pdf``), ``trialqa``. (default: `'litqa3'`) - `mode` (Mode): How a question's data files are delivered to the model. A no-op for tags without files (such as litqa3). Options: ``file``: Files uploaded via API. PDFs/images attached as context; other files as document attachments., ``inject``: Text file contents concatenated into the prompt as text., ``retrieve``: Only file names/stems are given; prompt instructs the agent to retrieve the necessary sequences or data from a source of its choosing. File contents are withheld. (default: `'inject'`) - `solver` (Solver | None): The solver to run. Defaults to ``bare()`` (the benchmark's "bare" configuration: a plain single-turn ``generate()``) when not provided. Pass any Inspect solver to override, e.g. ``-T solver=bare`` on the CLI. (default: `None`) @@ -73,22 +73,23 @@ This eval uses the public `EdisonScientific/labbench2` dataset on Hugging Face, ### Supported tags -| Tag | Samples | File-bearing | `mode` to use | Notes | -| --------------- | ------- | ------------ | ---------------------- |-----------------------------------| -| `cloning` | 14 | Yes | any (all modes) | Cloning protocols; reward-scored. | -| `dbqa2` | 86 | No | any (mode is a no-op) | Database access; recall judge. | -| `figqa2` | 101 | No | any (mode is a no-op) | Figure QA; exact-match judge. | -| `figqa2-img` | 101 | Yes | `file` | Figure QA with image files. | -| `figqa2-pdf` | 101 | Yes | `file` | Figure QA with PDF files. | -| `litqa3` | 168 | No | any (mode is a no-op) | Literature reasoning. | -| `patentqa` | 121 | No | any (mode is a no-op) | Patent comprehension. | -| `protocolqa2` | 125 | Yes | `file` | Lab protocols. | -| `sourcequality` | 150 | Yes | `file` | Source quality assessment. | -| `suppqa2` | 125 | No | any (mode is a no-op) | Supplement QA; exact-match. | -| `tableqa2` | 100 | No | any (mode is a no-op) | Table QA; exact-match judge. | -| `tableqa2-img` | 100 | Yes | `file` | Table QA with image files. | -| `tableqa2-pdf` | 100 | Yes | `file` | Table QA with PDF files. | -| `trialqa` | 120 | No | any (mode is a no-op) | Clinical trial QA. | +| Tag | Samples | File-bearing | `mode` to use | Notes | +| --------------- | ------- | ------------ | ---------------------- |----------------------------------------| +| `cloning` | 14 | Yes | any (all modes) | Cloning protocols; reward-scored. | +| `dbqa2` | 86 | No | any (mode is a no-op) | Database access; recall judge. | +| `figqa2` | 101 | No | any (mode is a no-op) | Figure QA; exact-match judge. | +| `figqa2-img` | 101 | Yes | `file` | Figure QA with image files. | +| `figqa2-pdf` | 101 | Yes | `file` | Figure QA with PDF files. | +| `litqa3` | 168 | No | any (mode is a no-op) | Literature reasoning. | +| `patentqa` | 121 | No | any (mode is a no-op) | Patent comprehension. | +| `protocolqa2` | 125 | Yes | `file` | Lab protocols. | +| `seqqa2` | 400 | Yes | `file` / `inject` | Sequence QA; deterministic validators. | +| `sourcequality` | 150 | Yes | `file` | Source quality assessment. | +| `suppqa2` | 125 | No | any (mode is a no-op) | Supplement QA; exact-match. | +| `tableqa2` | 100 | No | any (mode is a no-op) | Table QA; exact-match judge. | +| `tableqa2-img` | 100 | Yes | `file` | Table QA with image files. | +| `tableqa2-pdf` | 100 | Yes | `file` | Table QA with PDF files. | +| `trialqa` | 120 | No | any (mode is a no-op) | Clinical trial QA. | For file-bearing tags, the loader filters out questions that don't opt into the requested `mode` (per each question's `QuestionMode` flags in the HF @@ -96,7 +97,9 @@ data). At the time of writing every file-bearing tag (`protocolqa2`, `sourcequality`, `figqa2-img`, `figqa2-pdf`, `tableqa2-img`, `tableqa2-pdf`) opts into `file` only, so passing any other `mode` would load zero samples. Note that the base `figqa2`, `tableqa2`, and `suppqa2` configs carry no files -(mode is a no-op); the image/PDF variants are the file-bearing ones. The mode +(mode is a no-op); the image/PDF variants are the file-bearing ones. `seqqa2` is +the exception: all of its questions opt into `file` and `inject`, while only a +subset opt into `retrieve` (so `mode="retrieve"` loads fewer samples). The mode flags live in the dataset and may change — verify by running with the configuration you intend before drawing conclusions from sample counts. @@ -125,6 +128,11 @@ reused. To install Go: `brew install go` (macOS), `sudo apt install golang-go` ( or . Without Go, protocol execution fails gracefully: PCR-based samples score 0.0 with an explanatory reason rather than crashing the run. +The `seqqa2` tag is likewise scored deterministically — never by an LLM judge. A +per-question validator (selected by the question's `type`) checks the answer +extracted via that question's `answer_regex`; extraction tolerates line-wrapped +or whitespace-separated sequences. + The judge model is selected via the `grader` model role and defaults to `anthropic/claude-sonnet-4-5` at temperature 0. Override it on the command line, for example: diff --git a/src/lab_bench_2/__init__.py b/src/lab_bench_2/__init__.py index cb085a1..a6e7b6f 100644 --- a/src/lab_bench_2/__init__.py +++ b/src/lab_bench_2/__init__.py @@ -15,6 +15,7 @@ recall_judge_scorer, scorer_for_tag, semantic_judge_scorer, + seqqa2_scorer, ) from lab_bench_2.solvers import bare @@ -35,4 +36,5 @@ "record_to_sample", "scorer_for_tag", "semantic_judge_scorer", + "seqqa2_scorer", ] diff --git a/src/lab_bench_2/lab_bench_2.py b/src/lab_bench_2/lab_bench_2.py index 3053e7e..06abb9e 100644 --- a/src/lab_bench_2/lab_bench_2.py +++ b/src/lab_bench_2/lab_bench_2.py @@ -25,6 +25,7 @@ "litqa3", "patentqa", "protocolqa2", + "seqqa2", "sourcequality", "suppqa2", "tableqa2", @@ -47,9 +48,9 @@ def lab_bench_2( Args: tag: Which LAB-Bench 2 subset to run. Supported tags: ``cloning``, ``dbqa2``, ``figqa2`` (and ``figqa2-img`` / ``figqa2-pdf``), - ``litqa3``, ``patentqa``, ``protocolqa2``, ``sourcequality``, - ``suppqa2``, ``tableqa2`` (and ``tableqa2-img`` / ``tableqa2-pdf``), - ``trialqa``. + ``litqa3``, ``patentqa``, ``protocolqa2``, ``seqqa2``, + ``sourcequality``, ``suppqa2``, ``tableqa2`` (and ``tableqa2-img`` / + ``tableqa2-pdf``), ``trialqa``. mode: How a question's data files are delivered to the model. A no-op for tags without files (such as litqa3). Options: diff --git a/src/lab_bench_2/scorers.py b/src/lab_bench_2/scorers.py index f74aec1..7709364 100644 --- a/src/lab_bench_2/scorers.py +++ b/src/lab_bench_2/scorers.py @@ -153,6 +153,142 @@ async def score(state: TaskState, target: Target) -> Score: return score +ANSWER_BLOCK_RE = re.compile(r"(.*?)", re.IGNORECASE | re.DOTALL) +ASCII_WHITESPACE_RE = re.compile(r"[ \t\n\r\f\v]+") +NAMED_GROUP_RE = re.compile(r"\(\?P<[^>]+>") +IUPAC_NUCLEOTIDE_CHARS = frozenset("ACGTRYSWKMBDHVNacgtryswkmbdhvnUu") +REGEX_SYNTAX_CHARS = frozenset(r"[](){}?+*^$|,:\\-") + + +def is_nucleotide_only_answer_regex(answer_regex: str) -> bool: + without_group_names = NAMED_GROUP_RE.sub("(", answer_regex) + remaining = "".join( + char + for char in without_group_names + if char not in REGEX_SYNTAX_CHARS and not char.isspace() + ) + return bool(remaining) and set(remaining) <= IUPAC_NUCLEOTIDE_CHARS + + +def normalized_answer_attempts(answer_text: str, answer_regex: str) -> list[str]: + attempts = [ASCII_WHITESPACE_RE.sub(" ", answer_text).strip()] + if is_nucleotide_only_answer_regex(answer_regex): + nucleotide_attempt = ASCII_WHITESPACE_RE.sub("", answer_text).strip() + if nucleotide_attempt not in attempts: + attempts.append(nucleotide_attempt) + return attempts + + +def extract_seqqa2_answer( + completion: str, answer_regex: str | None +) -> dict[str, str] | None: + """Extract seqqa2 answer params via the question's answer regex. + + Delegates to upstream ``extract_answer`` first; on failure, retries against + whitespace-normalized variants of the ```` block to tolerate + line-wrapped or spaced sequence answers. + """ + from evals.evaluators import extract_answer + + extracted = extract_answer(completion, answer_regex) + if extracted is not None: + return cast(dict[str, str], extracted) + + if not answer_regex: + return None + + answer_match = ANSWER_BLOCK_RE.search(completion) + if answer_match is None: + answer_text = completion + prefix = "" + suffix = "" + else: + answer_text = answer_match.group(1) + prefix = completion[: answer_match.start()] + "" + suffix = "" + completion[answer_match.end() :] + + for normalized_answer in normalized_answer_attempts(answer_text, answer_regex): + if not normalized_answer: + continue + normalized_completion = f"{prefix}{normalized_answer}{suffix}" + extracted = extract_answer(normalized_completion, answer_regex) + if extracted is not None: + return cast(dict[str, str], extracted) + + return None + + +@scorer(metrics=[accuracy(), stderr()]) +def seqqa2_scorer() -> Scorer: + """Score SeqQA2 answers with labbench2's deterministic per-type validators.""" + from evals.utils import resolve_file_path + from labbench2.seqqa2.registry import VALIDATORS + + async def score(state: TaskState, target: Target) -> Score: + metadata = state.metadata or {} + question_type = cast(str | None, metadata.get("type")) + if not question_type: + return Score( + value=0.0, + explanation="SeqQA2 evaluation requires question type metadata.", + metadata={"route": "seqqa2"}, + ) + + validator = VALIDATORS.get(question_type) + if validator is None: + return Score( + value=0.0, + explanation=f"No validator found for type: {question_type}", + metadata={"route": "seqqa2"}, + ) + + extracted = extract_seqqa2_answer( + state.output.completion, + cast(str | None, metadata.get("answer_regex")), + ) + if extracted is None: + return Score( + value=0.0, + explanation="Failed to extract answer from model output.", + metadata={"route": "seqqa2", "validator": question_type}, + ) + + if "answer" in extracted and validator.answer_param != "answer": + extracted[validator.answer_param] = extracted.pop("answer") + + kwargs: dict[str, Any] = { + **cast(dict[str, Any], metadata.get("validator_params") or {}), + **extracted, + } + + files_path_str = cast(str | None, metadata.get("files_path")) + files_path = Path(files_path_str) if files_path_str else None + for key, value in list(kwargs.items()): + if key.endswith("_path") and isinstance(value, str): + resolved = resolve_file_path(value, files_path) + if resolved is None: + return Score( + value=0.0, + explanation=f"File not found: {value}", + metadata={"route": "seqqa2", "validator": question_type}, + ) + kwargs[key] = resolved + + score_value = float(validator.func(**kwargs)) + explanation = ( + f"Validator '{question_type}' passed" + if score_value == 1.0 + else f"Validator '{question_type}' failed" + ) + return Score( + value=score_value, + explanation=explanation, + metadata={"route": "seqqa2", "validator": question_type}, + ) + + return score + + SCORERS_BY_TAG = { "cloning": cloning_scorer, "dbqa2": recall_judge_scorer, @@ -162,6 +298,7 @@ async def score(state: TaskState, target: Target) -> Score: "litqa3": semantic_judge_scorer, "patentqa": semantic_judge_scorer, "protocolqa2": semantic_judge_scorer, + "seqqa2": seqqa2_scorer, "sourcequality": semantic_judge_scorer, "suppqa2": exact_match_judge_scorer, "tableqa2": exact_match_judge_scorer, diff --git a/tests/lab_bench_2/test_e2e.py b/tests/lab_bench_2/test_e2e.py index 7cacced..fb125aa 100644 --- a/tests/lab_bench_2/test_e2e.py +++ b/tests/lab_bench_2/test_e2e.py @@ -7,7 +7,7 @@ def test_unsupported_tag_raises() -> None: with pytest.raises(NotImplementedError): - lab_bench_2(tag="seqqa2") + lab_bench_2(tag="bogusqa") @pytest.mark.huggingface diff --git a/tests/lab_bench_2/test_scorers.py b/tests/lab_bench_2/test_scorers.py index 2b3f869..b8ec18d 100644 --- a/tests/lab_bench_2/test_scorers.py +++ b/tests/lab_bench_2/test_scorers.py @@ -96,7 +96,7 @@ def test_routing_table_matches_supported_tags(self) -> None: def test_unsupported_tag_raises(self) -> None: with pytest.raises(NotImplementedError): - scorer_for_tag("seqqa2") + scorer_for_tag("bogusqa") def test_semantic_judge_scorer_is_scorer() -> None: From a5e4e062a6dca9d531587b0d9ca5f192653e92a2 Mon Sep 17 00:00:00 2001 From: Isabelle Phan Date: Thu, 28 May 2026 21:57:46 -0700 Subject: [PATCH 07/11] Tidy seqqa2 scorer, extract answer helpers, add tests --- src/lab_bench_2/scorers.py | 93 ++------------ src/lab_bench_2/seqqa2_answer_parser.py | 83 +++++++++++++ tests/lab_bench_2/test_scorers.py | 113 ++++++++++++++++++ .../lab_bench_2/test_seqqa2_answer_parser.py | 68 +++++++++++ 4 files changed, 275 insertions(+), 82 deletions(-) create mode 100644 src/lab_bench_2/seqqa2_answer_parser.py create mode 100644 tests/lab_bench_2/test_seqqa2_answer_parser.py diff --git a/src/lab_bench_2/scorers.py b/src/lab_bench_2/scorers.py index 7709364..f428412 100644 --- a/src/lab_bench_2/scorers.py +++ b/src/lab_bench_2/scorers.py @@ -19,6 +19,8 @@ ) from inspect_ai.solver import TaskState +from lab_bench_2 import seqqa2_answer_parser + DEFAULT_GRADER_MODEL = "anthropic/claude-sonnet-4-5" GRADER_ROLE = "grader" @@ -153,71 +155,6 @@ async def score(state: TaskState, target: Target) -> Score: return score -ANSWER_BLOCK_RE = re.compile(r"(.*?)", re.IGNORECASE | re.DOTALL) -ASCII_WHITESPACE_RE = re.compile(r"[ \t\n\r\f\v]+") -NAMED_GROUP_RE = re.compile(r"\(\?P<[^>]+>") -IUPAC_NUCLEOTIDE_CHARS = frozenset("ACGTRYSWKMBDHVNacgtryswkmbdhvnUu") -REGEX_SYNTAX_CHARS = frozenset(r"[](){}?+*^$|,:\\-") - - -def is_nucleotide_only_answer_regex(answer_regex: str) -> bool: - without_group_names = NAMED_GROUP_RE.sub("(", answer_regex) - remaining = "".join( - char - for char in without_group_names - if char not in REGEX_SYNTAX_CHARS and not char.isspace() - ) - return bool(remaining) and set(remaining) <= IUPAC_NUCLEOTIDE_CHARS - - -def normalized_answer_attempts(answer_text: str, answer_regex: str) -> list[str]: - attempts = [ASCII_WHITESPACE_RE.sub(" ", answer_text).strip()] - if is_nucleotide_only_answer_regex(answer_regex): - nucleotide_attempt = ASCII_WHITESPACE_RE.sub("", answer_text).strip() - if nucleotide_attempt not in attempts: - attempts.append(nucleotide_attempt) - return attempts - - -def extract_seqqa2_answer( - completion: str, answer_regex: str | None -) -> dict[str, str] | None: - """Extract seqqa2 answer params via the question's answer regex. - - Delegates to upstream ``extract_answer`` first; on failure, retries against - whitespace-normalized variants of the ```` block to tolerate - line-wrapped or spaced sequence answers. - """ - from evals.evaluators import extract_answer - - extracted = extract_answer(completion, answer_regex) - if extracted is not None: - return cast(dict[str, str], extracted) - - if not answer_regex: - return None - - answer_match = ANSWER_BLOCK_RE.search(completion) - if answer_match is None: - answer_text = completion - prefix = "" - suffix = "" - else: - answer_text = answer_match.group(1) - prefix = completion[: answer_match.start()] + "" - suffix = "" + completion[answer_match.end() :] - - for normalized_answer in normalized_answer_attempts(answer_text, answer_regex): - if not normalized_answer: - continue - normalized_completion = f"{prefix}{normalized_answer}{suffix}" - extracted = extract_answer(normalized_completion, answer_regex) - if extracted is not None: - return cast(dict[str, str], extracted) - - return None - - @scorer(metrics=[accuracy(), stderr()]) def seqqa2_scorer() -> Scorer: """Score SeqQA2 answers with labbench2's deterministic per-type validators.""" @@ -229,28 +166,25 @@ async def score(state: TaskState, target: Target) -> Score: question_type = cast(str | None, metadata.get("type")) if not question_type: return Score( - value=0.0, + value=INCORRECT, explanation="SeqQA2 evaluation requires question type metadata.", - metadata={"route": "seqqa2"}, ) validator = VALIDATORS.get(question_type) if validator is None: return Score( - value=0.0, + value=INCORRECT, explanation=f"No validator found for type: {question_type}", - metadata={"route": "seqqa2"}, ) - extracted = extract_seqqa2_answer( + extracted = seqqa2_answer_parser.extract( state.output.completion, cast(str | None, metadata.get("answer_regex")), ) if extracted is None: return Score( - value=0.0, + value=INCORRECT, explanation="Failed to extract answer from model output.", - metadata={"route": "seqqa2", "validator": question_type}, ) if "answer" in extracted and validator.answer_param != "answer": @@ -268,22 +202,17 @@ async def score(state: TaskState, target: Target) -> Score: resolved = resolve_file_path(value, files_path) if resolved is None: return Score( - value=0.0, + value=INCORRECT, explanation=f"File not found: {value}", - metadata={"route": "seqqa2", "validator": question_type}, ) kwargs[key] = resolved score_value = float(validator.func(**kwargs)) - explanation = ( - f"Validator '{question_type}' passed" - if score_value == 1.0 - else f"Validator '{question_type}' failed" - ) + passed = score_value >= 1.0 return Score( - value=score_value, - explanation=explanation, - metadata={"route": "seqqa2", "validator": question_type}, + value=CORRECT if passed else INCORRECT, + explanation=f"Validator '{question_type}' {'passed' if passed else 'failed'}", + metadata={"validator": question_type, "validator_score": score_value}, ) return score diff --git a/src/lab_bench_2/seqqa2_answer_parser.py b/src/lab_bench_2/seqqa2_answer_parser.py new file mode 100644 index 0000000..fa5968b --- /dev/null +++ b/src/lab_bench_2/seqqa2_answer_parser.py @@ -0,0 +1,83 @@ +"""Answer extraction for the SeqQA2 scorer. + +Wraps upstream ``extract_answer`` with whitespace-normalization fallbacks so a +line-wrapped or space-separated sequence answer still matches the question's +answer regex. +""" + +from __future__ import annotations + +import re +from typing import cast + +ANSWER_BLOCK_RE = re.compile(r"(.*?)", re.IGNORECASE | re.DOTALL) +ASCII_WHITESPACE_RE = re.compile(r"[ \t\n\r\f\v]+") +NAMED_GROUP_RE = re.compile(r"\(\?P<[^>]+>") +IUPAC_NUCLEOTIDE_CHARS = frozenset("ACGTRYSWKMBDHVNacgtryswkmbdhvnUu") +REGEX_SYNTAX_CHARS = frozenset(r"[](){}?+*^$|,:\\-") + + +def _is_nucleotide_only_answer_regex(answer_regex: str) -> bool: + """Return True if the regex's literal characters are all IUPAC nucleotides. + + Used to decide whether stripping *all* internal whitespace from an answer is + a safe normalization (true for raw sequences, false for prose answers). + """ + without_group_names = NAMED_GROUP_RE.sub("(", answer_regex) + remaining = "".join( + char + for char in without_group_names + if char not in REGEX_SYNTAX_CHARS and not char.isspace() + ) + return bool(remaining) and set(remaining) <= IUPAC_NUCLEOTIDE_CHARS + + +def _normalized_answer_attempts(answer_text: str, answer_regex: str) -> list[str]: + """Whitespace-normalized variants of an answer to retry regex extraction. + + Always tries collapsing runs of whitespace to a single space; for + nucleotide-only regexes, also tries removing internal whitespace entirely. + """ + attempts = [ASCII_WHITESPACE_RE.sub(" ", answer_text).strip()] + if _is_nucleotide_only_answer_regex(answer_regex): + nucleotide_attempt = ASCII_WHITESPACE_RE.sub("", answer_text).strip() + if nucleotide_attempt not in attempts: + attempts.append(nucleotide_attempt) + return attempts + + +def extract(completion: str, answer_regex: str | None) -> dict[str, str] | None: + """Extract seqqa2 answer params via the question's answer regex. + + Delegates to upstream ``extract_answer`` first; on failure, retries against + whitespace-normalized variants of the ```` block to tolerate + line-wrapped or spaced sequence answers. Returns ``None`` if nothing matches. + """ + from evals.evaluators import extract_answer + + extracted = extract_answer(completion, answer_regex) + if extracted is not None: + return cast(dict[str, str], extracted) + + if not answer_regex: + return None + + answer_match = ANSWER_BLOCK_RE.search(completion) + if answer_match is None: + answer_text = completion + prefix = "" + suffix = "" + else: + answer_text = answer_match.group(1) + prefix = completion[: answer_match.start()] + "" + suffix = "" + completion[answer_match.end() :] + + for normalized_answer in _normalized_answer_attempts(answer_text, answer_regex): + if not normalized_answer: + continue + normalized_completion = f"{prefix}{normalized_answer}{suffix}" + extracted = extract_answer(normalized_completion, answer_regex) + if extracted is not None: + return cast(dict[str, str], extracted) + + return None diff --git a/tests/lab_bench_2/test_scorers.py b/tests/lab_bench_2/test_scorers.py index b8ec18d..b279596 100644 --- a/tests/lab_bench_2/test_scorers.py +++ b/tests/lab_bench_2/test_scorers.py @@ -1,4 +1,5 @@ from pathlib import Path +from types import SimpleNamespace from typing import Any import pytest @@ -14,6 +15,7 @@ recall_judge_scorer, scorer_for_tag, semantic_judge_scorer, + seqqa2_scorer, ) @@ -205,3 +207,114 @@ async def test_incorrect_when_ground_truth_missing( # then assert result.value == INCORRECT assert "Ground truth file not found" in (result.explanation or "") + + +class TestSeqqa2Scorer: + async def test_dispatches_to_validator_and_scores_correct( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + from labbench2.seqqa2.registry import VALIDATORS + + # given a dummy validator registered for a question type + validator = SimpleNamespace(answer_param="answer", func=lambda answer: 1.0) + monkeypatch.setitem(VALIDATORS, "dummy_validator", validator) + + # when + sut = seqqa2_scorer() + state = _task_state( + "pass", + { + "tag": "seqqa2", + "type": "dummy_validator", + "answer_regex": "(?Ppass)", + "validator_params": {}, + }, + ) + result = await sut(state, Target("")) + + # then + assert result == Score( + value=CORRECT, + explanation="Validator 'dummy_validator' passed", + metadata={"validator": "dummy_validator", "validator_score": 1.0}, + ) + + async def test_renames_answer_param_for_validator( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + from labbench2.seqqa2.registry import VALIDATORS + + captured: dict[str, Any] = {} + + def validator_func(sequence: str) -> float: + captured["sequence"] = sequence + return 1.0 + + # given a validator whose answer param is named "sequence" + validator = SimpleNamespace(answer_param="sequence", func=validator_func) + monkeypatch.setitem(VALIDATORS, "rename_validator", validator) + + # when + sut = seqqa2_scorer() + state = _task_state( + "ACTG", + { + "tag": "seqqa2", + "type": "rename_validator", + "answer_regex": "(?PACTG)", + "validator_params": {}, + }, + ) + result = await sut(state, Target("")) + + # then the extracted answer is passed under the validator's param name + assert result.value == CORRECT + assert captured == {"sequence": "ACTG"} + + async def test_incorrect_for_unknown_validator_type(self) -> None: + sut = seqqa2_scorer() + state = _task_state( + "x", + { + "tag": "seqqa2", + "type": "does_not_exist", + "answer_regex": "(?Px)", + }, + ) + result = await sut(state, Target("")) + assert result.value == INCORRECT + assert "No validator found" in (result.explanation or "") + + async def test_incorrect_when_type_missing(self) -> None: + sut = seqqa2_scorer() + state = _task_state("x", {"tag": "seqqa2"}) + result = await sut(state, Target("")) + assert result.value == INCORRECT + assert "question type" in (result.explanation or "") + + async def test_fail_closed_when_path_param_unresolved( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + from labbench2.seqqa2.registry import VALIDATORS + + # given a validator with a _path param that cannot be resolved + validator = SimpleNamespace(answer_param="answer", func=lambda **kw: 1.0) + monkeypatch.setitem(VALIDATORS, "path_validator", validator) + monkeypatch.setattr("evals.utils.resolve_file_path", lambda value, _: None) + + # when + sut = seqqa2_scorer() + state = _task_state( + "x", + { + "tag": "seqqa2", + "type": "path_validator", + "answer_regex": "(?Px)", + "validator_params": {"reference_path": "missing.fa"}, + }, + ) + result = await sut(state, Target("")) + + # then it fails closed rather than calling the validator + assert result.value == INCORRECT + assert "File not found: missing.fa" in (result.explanation or "") diff --git a/tests/lab_bench_2/test_seqqa2_answer_parser.py b/tests/lab_bench_2/test_seqqa2_answer_parser.py new file mode 100644 index 0000000..4881b65 --- /dev/null +++ b/tests/lab_bench_2/test_seqqa2_answer_parser.py @@ -0,0 +1,68 @@ +from lab_bench_2.seqqa2_answer_parser import ( + _is_nucleotide_only_answer_regex, + _normalized_answer_attempts, + extract, +) + + +class TestExtract: + def test_collapses_ascii_whitespace(self) -> None: + # given a wrapped answer whose regex expects single spaces + extracted = extract( + "FORWARD,\n REVERSE", + r"(?PFORWARD, REVERSE)", + ) + # then whitespace runs collapse to a single space before matching + assert extracted == {"answer": "FORWARD, REVERSE"} + + def test_strips_internal_whitespace_for_nucleotides(self) -> None: + # given spaced sequences and a nucleotide-only regex + extracted = extract( + "ATGC TGCA,\n AATT CCGG", + r"(?P[ATGCatgc]+),(?P[ATGCatgc]+)", + ) + # then internal whitespace is removed so the sequences match + assert extracted == {"forward": "ATGCTGCA", "reverse": "AATTCCGG"} + + def test_accepts_bare_numeric_answer(self) -> None: + extracted = extract("25.67", r"(?P25\.67)") + assert extracted == {"answer": "25.67"} + + def test_accepts_bare_nucleotide_answer(self) -> None: + extracted = extract("ATGC TGCA", r"(?P[ATGCatgc]+)") + assert extracted == {"answer": "ATGCTGCA"} + + def test_accepts_bare_paired_answer(self) -> None: + extracted = extract("931,747", r"(?P\d+),(?P\d+)") + assert extracted == {"left": "931", "right": "747"} + + def test_rejects_invalid_bare_answer(self) -> None: + assert extract("NA,NA", r"(?P\d+),(?P\d+)") is None + + def test_returns_none_without_regex(self) -> None: + assert extract("anything", None) is None + + +class TestIsNucleotideOnlyAnswerRegex: + def test_true_for_nucleotide_regex(self) -> None: + assert _is_nucleotide_only_answer_regex(r"(?P[ATGCatgc]+)") is True + + def test_false_for_word_regex(self) -> None: + assert ( + _is_nucleotide_only_answer_regex(r"(?PFORWARD, REVERSE)") is False + ) + + +class TestNormalizedAnswerAttempts: + def test_collapses_whitespace_for_non_nucleotide(self) -> None: + # given a non-nucleotide regex, only the space-collapsed variant is tried + assert _normalized_answer_attempts("FOR\n WARD", r"(?PFORWARD)") == [ + "FOR WARD" + ] + + def test_adds_stripped_variant_for_nucleotides(self) -> None: + # given a nucleotide regex, also try removing internal whitespace + assert _normalized_answer_attempts("AT GC", r"(?P[ATGC]+)") == [ + "AT GC", + "ATGC", + ] From cd7f6007549e956fcef5fc943ec3b618ef422eec Mon Sep 17 00:00:00 2001 From: Isabelle Phan Date: Fri, 29 May 2026 11:19:41 -0700 Subject: [PATCH 08/11] Ask structured-output from LLM judge with regex fallback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Request structured output from the grader, reading the verdict from a typed field instead of regex-scraping the completion — mirroring upstream's LLMJudgeEvaluator. Fall back to parse_judge_verdict when a provider doesn't honor the schema or returns non-compliant output. --- src/lab_bench_2/README.md | 5 ++ src/lab_bench_2/scorers.py | 45 +++++++--- tests/lab_bench_2/test_scorers.py | 133 +++++++++++++++++++++++++++--- 3 files changed, 160 insertions(+), 23 deletions(-) diff --git a/src/lab_bench_2/README.md b/src/lab_bench_2/README.md index fa93a1c..fd8e3a6 100644 --- a/src/lab_bench_2/README.md +++ b/src/lab_bench_2/README.md @@ -111,6 +111,11 @@ returns one of `correct` / `incorrect` / `unsure`; a `correct` verdict scores 1.0 and everything else (including unparseable or empty judgements) scores 0.0. Reported metrics are `accuracy` and `stderr`. +The judge requests **structured output** (a typed `result` / `rationale` +schema), so the verdict is read from a typed field rather than scraped from +prose. If the grader's provider doesn't support structured output, it falls back +to parsing a `result:` line from the response. + The judge prompt varies by tag: most tags use the default semantic prompt; `dbqa2` (database access) uses a recall-based variant that marks an answer correct when it recovers the expected reference values; and the figure, table, diff --git a/src/lab_bench_2/scorers.py b/src/lab_bench_2/scorers.py index f428412..ab2e6b5 100644 --- a/src/lab_bench_2/scorers.py +++ b/src/lab_bench_2/scorers.py @@ -6,7 +6,7 @@ from pathlib import Path from typing import Any, cast -from inspect_ai.model import GenerateConfig, get_model +from inspect_ai.model import GenerateConfig, ResponseSchema, get_model from inspect_ai.scorer import ( CORRECT, INCORRECT, @@ -18,6 +18,8 @@ stderr, ) from inspect_ai.solver import TaskState +from inspect_ai.util import json_schema +from pydantic import ValidationError from lab_bench_2 import seqqa2_answer_parser @@ -44,7 +46,17 @@ def _judge_score(prompt_template: str) -> Scorer: - """Build a judge that grades an answer against the reference via the template.""" + """Build a judge that grades an answer against the reference via the template. + + Requests structured output so the grader returns a typed verdict; falls back + to regex parsing for providers that don't honor the schema or for + non-compliant responses. + """ + from evals.models import EvaluationResult + + response_schema = ResponseSchema( + name="evaluation_result", json_schema=json_schema(EvaluationResult) + ) async def score(state: TaskState, target: Target) -> Score: answer = state.output.completion.strip() @@ -53,25 +65,36 @@ async def score(state: TaskState, target: Target) -> Score: value=INCORRECT, answer="", explanation="No answer was produced." ) - grader = get_model( - role=GRADER_ROLE, - default=DEFAULT_GRADER_MODEL, - config=GenerateConfig(temperature=0.0), - ) + grader = get_model(role=GRADER_ROLE, default=DEFAULT_GRADER_MODEL) prompt = prompt_template.format( question=state.input_text, correct_answer=target.text, answer=answer, ) - result = await grader.generate(prompt) - verdict = parse_judge_verdict(result.completion) + # Pass config on the generate() call (not get_model): a role-bound + # grader is returned as-is by get_model, dropping any config given there. + result = await grader.generate( + prompt, + config=GenerateConfig(temperature=0.0, response_schema=response_schema), + ) + + try: + evaluation = EvaluationResult.model_validate_json(result.completion) + verdict: str | None = evaluation.result + explanation = evaluation.rationale + verdict_source = "structured" + except ValidationError: + verdict = parse_judge_verdict(result.completion) + explanation = result.completion + verdict_source = "fallback" + value = CORRECT if verdict == JUDGE_VERDICT_CORRECT else INCORRECT return Score( value=value, answer=answer, - explanation=result.completion, - metadata={"verdict": verdict}, + explanation=explanation, + metadata={"verdict": verdict, "verdict_source": verdict_source}, ) return score diff --git a/tests/lab_bench_2/test_scorers.py b/tests/lab_bench_2/test_scorers.py index b279596..55b273b 100644 --- a/tests/lab_bench_2/test_scorers.py +++ b/tests/lab_bench_2/test_scorers.py @@ -3,11 +3,11 @@ from typing import Any import pytest -from inspect_ai.model import ModelOutput +from inspect_ai.model import ModelName, ModelOutput from inspect_ai.scorer import CORRECT, INCORRECT, Score, Scorer, Target from inspect_ai.solver import TaskState -from lab_bench_2 import SUPPORTED_TAGS, parse_judge_verdict +from lab_bench_2 import SUPPORTED_TAGS, parse_judge_verdict, scorers from lab_bench_2.scorers import ( SCORERS_BY_TAG, cloning_scorer, @@ -21,7 +21,7 @@ def _task_state(completion: str, metadata: dict[str, Any]) -> TaskState: return TaskState( - model="mockllm/model", + model=ModelName("mockllm/model"), sample_id="sample-1", epoch=1, input="Question?", @@ -31,6 +31,13 @@ def _task_state(completion: str, metadata: dict[str, Any]) -> TaskState: ) +async def _score(sut: Scorer, state: TaskState, target: Target) -> Score: + """Run a scorer and assert it produced a Score (narrows ``Score | None``).""" + result = await sut(state, target) + assert result is not None + return result + + class TestParseJudgeVerdict: @pytest.mark.parametrize( "verdict", @@ -113,6 +120,108 @@ def test_exact_match_judge_scorer_is_scorer() -> None: assert isinstance(exact_match_judge_scorer(), Scorer) +def _patch_grader(monkeypatch: pytest.MonkeyPatch, completion: str) -> None: + class _Grader: + async def generate(self, prompt: str, **kwargs: Any) -> SimpleNamespace: + return SimpleNamespace(completion=completion) + + monkeypatch.setattr(scorers, "get_model", lambda *args, **kwargs: _Grader()) + + +class TestJudgeScorer: + async def test_structured_correct_verdict_scores_correct( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + # given a grader returning a structured (typed) correct verdict + _patch_grader( + monkeypatch, + '{"rationale": "matches the reference", "result": "correct"}', + ) + # when + sut = semantic_judge_scorer() + result = await _score( + sut, _task_state("answer", {"tag": "litqa3"}), Target("ref") + ) + # then the typed rationale and verdict are used + assert result.value == CORRECT + assert result.explanation == "matches the reference" + assert result.metadata == { + "verdict": "correct", + "verdict_source": "structured", + } + + @pytest.mark.parametrize("verdict", ["incorrect", "unsure"]) + async def test_structured_non_correct_verdict_scores_incorrect( + self, monkeypatch: pytest.MonkeyPatch, verdict: str + ) -> None: + _patch_grader(monkeypatch, f'{{"rationale": "x", "result": "{verdict}"}}') + sut = semantic_judge_scorer() + result = await _score( + sut, _task_state("answer", {"tag": "litqa3"}), Target("ref") + ) + assert result.value == INCORRECT + assert result.metadata == {"verdict": verdict, "verdict_source": "structured"} + + async def test_falls_back_to_regex_for_non_structured_output( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + # given a grader that ignores the schema and returns free text + _patch_grader(monkeypatch, "Reasoning here.\nresult: correct") + # when + sut = semantic_judge_scorer() + result = await _score( + sut, _task_state("answer", {"tag": "litqa3"}), Target("ref") + ) + # then the regex fallback recovers the verdict + assert result.value == CORRECT + assert result.metadata == {"verdict": "correct", "verdict_source": "fallback"} + + @pytest.mark.parametrize( + "completion, expected_verdict", + [ + ("Reasoning here.\nresult: incorrect", "incorrect"), + ("no parseable verdict in this text", None), + ], + ) + async def test_falls_back_to_regex_scores_incorrect( + self, + monkeypatch: pytest.MonkeyPatch, + completion: str, + expected_verdict: str | None, + ) -> None: + # given non-structured grader output that is not a correct verdict + # (a parsed "incorrect", or nothing parseable at all) + _patch_grader(monkeypatch, completion) + # when + sut = semantic_judge_scorer() + result = await _score( + sut, _task_state("answer", {"tag": "litqa3"}), Target("ref") + ) + # then it scores incorrect + assert result.value == INCORRECT + assert result.metadata == { + "verdict": expected_verdict, + "verdict_source": "fallback", + } + + async def test_empty_answer_scores_incorrect( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + # given an empty answer but correct grade + answer = " " + _patch_grader(monkeypatch, '{"rationale": "x", "result": "correct"}') + + # when + sut = semantic_judge_scorer() + result = await _score( + sut, _task_state(answer, {"tag": "litqa3"}), Target("ref") + ) + + # then + assert result.value == INCORRECT + assert "No answer" in (result.explanation or "") + + class TestCloningScorer: async def test_scores_correct_when_reward_passes( self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path @@ -143,7 +252,7 @@ async def fake_cloning_reward(**kwargs: Any) -> tuple[float, str]: "assemble", {"tag": "cloning", "id": "clone_1", "files_path": str(tmp_path)}, ) - result = await sut(state, Target("")) + result = await _score(sut, state, Target("")) # then assert result == Score( @@ -172,7 +281,7 @@ async def fake_cloning_reward(**kwargs: Any) -> tuple[float, str]: "assemble", {"tag": "cloning", "id": "clone_1", "files_path": str(tmp_path)}, ) - result = await sut(state, Target("")) + result = await _score(sut, state, Target("")) # then assert result.value == INCORRECT @@ -184,7 +293,7 @@ async def test_incorrect_without_files_path_or_id(self) -> None: state = _task_state("assemble", {"tag": "cloning"}) # when - result = await sut(state, Target("")) + result = await _score(sut, state, Target("")) # then it fails closed before resolving or scoring assert result.value == INCORRECT @@ -202,7 +311,7 @@ async def test_incorrect_when_ground_truth_missing( "assemble", {"tag": "cloning", "id": "clone_1", "files_path": str(tmp_path)}, ) - result = await sut(state, Target("")) + result = await _score(sut, state, Target("")) # then assert result.value == INCORRECT @@ -230,7 +339,7 @@ async def test_dispatches_to_validator_and_scores_correct( "validator_params": {}, }, ) - result = await sut(state, Target("")) + result = await _score(sut, state, Target("")) # then assert result == Score( @@ -265,7 +374,7 @@ def validator_func(sequence: str) -> float: "validator_params": {}, }, ) - result = await sut(state, Target("")) + result = await _score(sut, state, Target("")) # then the extracted answer is passed under the validator's param name assert result.value == CORRECT @@ -281,14 +390,14 @@ async def test_incorrect_for_unknown_validator_type(self) -> None: "answer_regex": "(?Px)", }, ) - result = await sut(state, Target("")) + result = await _score(sut, state, Target("")) assert result.value == INCORRECT assert "No validator found" in (result.explanation or "") async def test_incorrect_when_type_missing(self) -> None: sut = seqqa2_scorer() state = _task_state("x", {"tag": "seqqa2"}) - result = await sut(state, Target("")) + result = await _score(sut, state, Target("")) assert result.value == INCORRECT assert "question type" in (result.explanation or "") @@ -313,7 +422,7 @@ async def test_fail_closed_when_path_param_unresolved( "validator_params": {"reference_path": "missing.fa"}, }, ) - result = await sut(state, Target("")) + result = await _score(sut, state, Target("")) # then it fails closed rather than calling the validator assert result.value == INCORRECT From a1694d9446d3f5147bbacca59ec75a8b882f1293 Mon Sep 17 00:00:00 2001 From: Isabelle Phan Date: Fri, 29 May 2026 14:27:27 -0700 Subject: [PATCH 09/11] Pass answer in the cloning scorer output --- src/lab_bench_2/scorers.py | 8 ++++++-- tests/lab_bench_2/test_scorers.py | 2 ++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/lab_bench_2/scorers.py b/src/lab_bench_2/scorers.py index ab2e6b5..99f06ac 100644 --- a/src/lab_bench_2/scorers.py +++ b/src/lab_bench_2/scorers.py @@ -160,8 +160,9 @@ async def score(state: TaskState, target: Target) -> Score: explanation=f"Ground truth file not found: {ground_truth_filename}", ) + answer = state.output.completion score_value, reason = await cloning_reward( - answer=state.output.completion, + answer=answer, base_dir=Path(files_path_str), reference_path=ground_truth_path, validator_params=cast( @@ -171,6 +172,7 @@ async def score(state: TaskState, target: Target) -> Score: return Score( value=CORRECT if score_value >= 1.0 else INCORRECT, + answer=answer, explanation=reason, metadata={"cloning_score": score_value}, ) @@ -200,8 +202,9 @@ async def score(state: TaskState, target: Target) -> Score: explanation=f"No validator found for type: {question_type}", ) + raw_answer = state.output.completion extracted = seqqa2_answer_parser.extract( - state.output.completion, + raw_answer, cast(str | None, metadata.get("answer_regex")), ) if extracted is None: @@ -234,6 +237,7 @@ async def score(state: TaskState, target: Target) -> Score: passed = score_value >= 1.0 return Score( value=CORRECT if passed else INCORRECT, + answer=raw_answer, explanation=f"Validator '{question_type}' {'passed' if passed else 'failed'}", metadata={"validator": question_type, "validator_score": score_value}, ) diff --git a/tests/lab_bench_2/test_scorers.py b/tests/lab_bench_2/test_scorers.py index 55b273b..ba59aca 100644 --- a/tests/lab_bench_2/test_scorers.py +++ b/tests/lab_bench_2/test_scorers.py @@ -257,6 +257,7 @@ async def fake_cloning_reward(**kwargs: Any) -> tuple[float, str]: # then assert result == Score( value=CORRECT, + answer="assemble", explanation="Cloning validation passed", metadata={"cloning_score": 1.0}, ) @@ -344,6 +345,7 @@ async def test_dispatches_to_validator_and_scores_correct( # then assert result == Score( value=CORRECT, + answer="pass", explanation="Validator 'dummy_validator' passed", metadata={"validator": "dummy_validator", "validator_score": 1.0}, ) From 224c4faeb0bdfb0bfb8c00222268501c1c968c09 Mon Sep 17 00:00:00 2001 From: Isabelle Phan Date: Mon, 1 Jun 2026 15:06:12 -0700 Subject: [PATCH 10/11] Apply README suggestions from code review Co-authored-by: Tania <120768997+ItsTania@users.noreply.github.com> --- src/lab_bench_2/README.md | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/src/lab_bench_2/README.md b/src/lab_bench_2/README.md index fd8e3a6..b11e267 100644 --- a/src/lab_bench_2/README.md +++ b/src/lab_bench_2/README.md @@ -93,19 +93,28 @@ This eval uses the public `EdisonScientific/labbench2` dataset on Hugging Face, For file-bearing tags, the loader filters out questions that don't opt into the requested `mode` (per each question's `QuestionMode` flags in the HF -data). At the time of writing every file-bearing tag (`protocolqa2`, -`sourcequality`, `figqa2-img`, `figqa2-pdf`, `tableqa2-img`, `tableqa2-pdf`) -opts into `file` only, so passing any other `mode` would load zero samples. -Note that the base `figqa2`, `tableqa2`, and `suppqa2` configs carry no files -(mode is a no-op); the image/PDF variants are the file-bearing ones. `seqqa2` is -the exception: all of its questions opt into `file` and `inject`, while only a -subset opt into `retrieve` (so `mode="retrieve"` loads fewer samples). The mode -flags live in the dataset and may change — verify by running with the -configuration you intend before drawing conclusions from sample counts. +data). + +** Relationship between the tag and mode parameters** + +Tags describe groups of samples/questions whilst mode describes how data files are uploaded. Not every sample is compatible with each mode of data uploading; if incompatible they are not loaded into the eval. +Each sample in the dataset contains flags for compatible modes - this may change and sample counts can be verified by running with the configuration you intend before drawing conclusions from sample counts. + +For most tags, those that uses files requires the `file` mode. For example; + +`uv run inspect eval lab_bench_2/lab_bench_2 -T tag=sourcequality -T mode=retrieve` + + Will result in no samples being loaded in. This is also true for tags protocolqa2`,`sourcequality`, `figqa2-img`, `figqa2-pdf`, `tableqa2-img`, `tableqa2-pdf`. + +Note that the base `figqa2`, `tableqa2`, and `suppqa2` tags have no files (mode is a no-op). Their image/PDF variants do have files and are impacted by the above. + +`seqqa2` is the exception: all of its samples are compatible with `file` and `inject`, while only a +subset of this tag can be used with`retrieve` (so `mode="retrieve"` loads fewer samples). ## Scoring -Answers are graded by an LLM judge. The judge compares the model's answer to +There are different scoring methods for the tags. +Some tags are scored deterministically (see x and y) but most are graded by an LLM judge. The judge compares the solver's answer to the reference, accepting semantically or numerically equivalent answers, and returns one of `correct` / `incorrect` / `unsure`; a `correct` verdict scores 1.0 and everything else (including unparseable or empty judgements) scores 0.0. @@ -133,7 +142,7 @@ reused. To install Go: `brew install go` (macOS), `sudo apt install golang-go` ( or . Without Go, protocol execution fails gracefully: PCR-based samples score 0.0 with an explanatory reason rather than crashing the run. -The `seqqa2` tag is likewise scored deterministically — never by an LLM judge. A +The `seqqa2` tag is also scored deterministically. A per-question validator (selected by the question's `type`) checks the answer extracted via that question's `answer_regex`; extraction tolerates line-wrapped or whitespace-separated sequences. From bf2d35772309419b2c8386f83d03c9ba39ae2957 Mon Sep 17 00:00:00 2001 From: Isabelle Phan Date: Mon, 1 Jun 2026 16:06:51 -0700 Subject: [PATCH 11/11] Update readme and add more debugging to scorer --- src/lab_bench_2/README.md | 75 ++++++++++++++++++++-------------- src/lab_bench_2/lab_bench_2.py | 2 +- src/lab_bench_2/scorers.py | 3 +- 3 files changed, 46 insertions(+), 34 deletions(-) diff --git a/src/lab_bench_2/README.md b/src/lab_bench_2/README.md index b11e267..1e7d6e9 100644 --- a/src/lab_bench_2/README.md +++ b/src/lab_bench_2/README.md @@ -43,6 +43,15 @@ ANTHROPIC_API_KEY= ``` +### Additional Usage Notes + +The `cloning` tag scorer runs a PCR (Polymerase Chain Reaction) simulation using +a small Go binary that labbench2 compiles on first use, so a +Go toolchain (1.21+) must be available on the host the first time you score +`cloning`; the compiled binary is then cached inside the installed package and +reused. To install Go: `brew install go` (macOS), `sudo apt install golang-go` (Linux), +or . + ## Options @@ -63,7 +72,7 @@ See `uv run inspect eval --help` for all available options. ### `lab_bench_2` - `tag` (str): Which LAB-Bench 2 subset to run. Supported tags: ``cloning``, ``dbqa2``, ``figqa2`` (and ``figqa2-img`` / ``figqa2-pdf``), ``litqa3``, ``patentqa``, ``protocolqa2``, ``seqqa2``, ``sourcequality``, ``suppqa2``, ``tableqa2`` (and ``tableqa2-img`` / ``tableqa2-pdf``), ``trialqa``. (default: `'litqa3'`) -- `mode` (Mode): How a question's data files are delivered to the model. A no-op for tags without files (such as litqa3). Options: ``file``: Files uploaded via API. PDFs/images attached as context; other files as document attachments., ``inject``: Text file contents concatenated into the prompt as text., ``retrieve``: Only file names/stems are given; prompt instructs the agent to retrieve the necessary sequences or data from a source of its choosing. File contents are withheld. (default: `'inject'`) +- `mode` (Mode): How a question's data files are delivered to the model. A no-op for tags without files (such as litqa3). Options: ``file``: Files uploaded via API. PDFs/images attached as context; other files as document attachments., ``inject``: Text file contents concatenated into the prompt as text., ``retrieve``: Only file names/stems are given; prompt instructs the agent to retrieve the necessary sequences or data from a source of its choosing. File contents are withheld. (default: `'file'`) - `solver` (Solver | None): The solver to run. Defaults to ``bare()`` (the benchmark's "bare" configuration: a plain single-turn ``generate()``) when not provided. Pass any Inspect solver to override, e.g. ``-T solver=bare`` on the CLI. (default: `None`) @@ -93,32 +102,37 @@ This eval uses the public `EdisonScientific/labbench2` dataset on Hugging Face, For file-bearing tags, the loader filters out questions that don't opt into the requested `mode` (per each question's `QuestionMode` flags in the HF -data). +data). + +#### Relationship between the tag and mode parameters -** Relationship between the tag and mode parameters** +Tags describe groups of samples/questions whilst mode describes how data files are uploaded. +Not every sample is compatible with each mode of data uploading; if incompatible they are not loaded into the eval. +Each sample in the dataset contains flags for compatible modes - this may change and sample counts +can be verified by running with the configuration you intend before drawing conclusions from sample counts. -Tags describe groups of samples/questions whilst mode describes how data files are uploaded. Not every sample is compatible with each mode of data uploading; if incompatible they are not loaded into the eval. -Each sample in the dataset contains flags for compatible modes - this may change and sample counts can be verified by running with the configuration you intend before drawing conclusions from sample counts. +For most tags, those that uses files requires the `file` mode. For example; -For most tags, those that uses files requires the `file` mode. For example; +`uv run inspect eval lab_bench_2 -T tag=sourcequality -T mode=retrieve` -`uv run inspect eval lab_bench_2/lab_bench_2 -T tag=sourcequality -T mode=retrieve` + Will result in no samples being loaded in. This is also true for tags protocolqa2`,`sourcequality`,`figqa2-img`,`figqa2-pdf`,`tableqa2-img`,`tableqa2-pdf`. - Will result in no samples being loaded in. This is also true for tags protocolqa2`,`sourcequality`, `figqa2-img`, `figqa2-pdf`, `tableqa2-img`, `tableqa2-pdf`. - Note that the base `figqa2`, `tableqa2`, and `suppqa2` tags have no files (mode is a no-op). Their image/PDF variants do have files and are impacted by the above. `seqqa2` is the exception: all of its samples are compatible with `file` and `inject`, while only a -subset of this tag can be used with`retrieve` (so `mode="retrieve"` loads fewer samples). +subset of this tag can be used with`retrieve` (so `mode="retrieve"` loads fewer samples). ## Scoring There are different scoring methods for the tags. -Some tags are scored deterministically (see x and y) but most are graded by an LLM judge. The judge compares the solver's answer to -the reference, accepting semantically or numerically equivalent answers, and -returns one of `correct` / `incorrect` / `unsure`; a `correct` verdict scores -1.0 and everything else (including unparseable or empty judgements) scores 0.0. -Reported metrics are `accuracy` and `stderr`. +Some tags are scored deterministically (see `cloning` and `seqqa2`) but most are graded by an LLM judge. + +### LLM judge scorers + +The judge compares the solver's answer to the reference, accepting semantically +or numerically equivalent answers, and returns one of `correct` / `incorrect` / `unsure`; +a `correct` verdict scores 1.0 and everything else (including unparseable or empty judgements) +scores 0.0. Reported metrics are `accuracy` and `stderr`. The judge requests **structured output** (a typed `result` / `rationale` schema), so the verdict is read from a typed field rather than scraped from @@ -131,22 +145,6 @@ correct when it recovers the expected reference values; and the figure, table, and supplement tags (`figqa2*`, `tableqa2*`, `suppqa2`) use an exact-match variant for numeric answers. -The `cloning` tag is not graded by an LLM judge: it is scored deterministically -by labbench2's reward pipeline, which parses the submitted protocol, executes it -(including PCR simulation), and compares the result to the reference assembly via -sequence-similarity and restriction-digest checks. -PCR simulation runs a small Go binary that labbench2 compiles on first use, so a -Go toolchain (1.21+) must be available on the host the first time you score -`cloning`; the compiled binary is then cached inside the installed package and -reused. To install Go: `brew install go` (macOS), `sudo apt install golang-go` (Linux), -or . Without Go, protocol execution fails gracefully: PCR-based -samples score 0.0 with an explanatory reason rather than crashing the run. - -The `seqqa2` tag is also scored deterministically. A -per-question validator (selected by the question's `type`) checks the answer -extracted via that question's `answer_regex`; extraction tolerates line-wrapped -or whitespace-separated sequences. - The judge model is selected via the `grader` model role and defaults to `anthropic/claude-sonnet-4-5` at temperature 0. Override it on the command line, for example: @@ -157,6 +155,21 @@ uv run inspect eval lab_bench_2/lab_bench_2 \ --model-role grader=anthropic/claude-opus-4-1-20250805 ``` +### Deterministic scorers + +The `cloning` tag is not graded by an LLM judge: it is scored deterministically +by labbench2's reward pipeline, which parses the submitted protocol, executes it +(including Polymerase Chain Reaction simulation), and compares the result to the +reference assembly via sequence-similarity and restriction-digest checks. +PCR simulation requires that Go be available on the host. Without Go, +protocol execution fails gracefully: PCR-based samples score 0.0 with an explanatory +reason rather than crashing the run. + +The `seqqa2` tag is also scored deterministically. A +per-question validator (selected by the question's `type`) checks the answer +extracted via that question's `answer_regex`; extraction tolerates line-wrapped +or whitespace-separated sequences. + ## Attribution This evaluation depends on the reference implementation's diff --git a/src/lab_bench_2/lab_bench_2.py b/src/lab_bench_2/lab_bench_2.py index 06abb9e..98b39b1 100644 --- a/src/lab_bench_2/lab_bench_2.py +++ b/src/lab_bench_2/lab_bench_2.py @@ -40,7 +40,7 @@ @task def lab_bench_2( tag: str = "litqa3", - mode: Mode = "inject", + mode: Mode = "file", solver: Solver | None = None, ) -> Task: """LAB-Bench 2 evaluation task. diff --git a/src/lab_bench_2/scorers.py b/src/lab_bench_2/scorers.py index 99f06ac..e8c0428 100644 --- a/src/lab_bench_2/scorers.py +++ b/src/lab_bench_2/scorers.py @@ -72,8 +72,6 @@ async def score(state: TaskState, target: Target) -> Score: correct_answer=target.text, answer=answer, ) - # Pass config on the generate() call (not get_model): a role-bound - # grader is returned as-is by get_model, dropping any config given there. result = await grader.generate( prompt, config=GenerateConfig(temperature=0.0, response_schema=response_schema), @@ -211,6 +209,7 @@ async def score(state: TaskState, target: Target) -> Score: return Score( value=INCORRECT, explanation="Failed to extract answer from model output.", + metadata={"raw_answer": raw_answer, "extracted": extracted}, ) if "answer" in extracted and validator.answer_param != "answer":