Add all scorers for LABBench2 [3/n]

pyproject.toml

@@ -108,7 +108,7 @@ ignore_missing_imports = true
 
 [[tool.mypy.overrides]]
 # Upstream labbench2 git dep has no py.typed marker / stubs.
-module = ["evals", "evals.*"]
+module = ["evals", "evals.*", "labbench2", "labbench2.*"]
 ignore_missing_imports = true
 
 [[tool.mypy.overrides]]

src/lab_bench_2/README.md

@@ -43,6 +43,15 @@ ANTHROPIC_API_KEY=<anthropic-api-key>
 ```
 <!-- /Usage: Automatically Generated -->
 
+### Additional Usage Notes
+
+The `cloning` tag scorer runs a PCR (Polymerase Chain Reaction) simulation using
+a small Go binary that labbench2 compiles on first use, so a
+Go toolchain (1.21+) must be available on the host the first time you score
+`cloning`; the compiled binary is then cached inside the installed package and
+reused. To install Go: `brew install go` (macOS), `sudo apt install golang-go` (Linux),
+or <https://go.dev/dl/>.
+
 <!-- Options: Automatically Generated -->
 ## Options
 
@@ -62,8 +71,8 @@ See `uv run inspect eval --help` for all available options.
 
 ### `lab_bench_2`
 
-- `tag` (str): Which LAB-Bench 2 subset to run. Supported tags: ``litqa3``, ``patentqa``, ``protocolqa2``, ``sourcequality``, ``trialqa``. (default: `'litqa3'`)
-- `mode` (Mode): How a question's data files are delivered to the model. A no-op for tags without files (such as litqa3). Options: ``file``: Files uploaded via API. PDFs/images attached as context; other files as document attachments., ``inject``: Text file contents concatenated into the prompt as text., ``retrieve``: Only file names/stems are given; prompt instructs the agent to retrieve the necessary sequences or data from a source of its choosing. File contents are withheld. (default: `'inject'`)
+- `tag` (str): Which LAB-Bench 2 subset to run. Supported tags: ``cloning``, ``dbqa2``, ``figqa2`` (and ``figqa2-img`` / ``figqa2-pdf``), ``litqa3``, ``patentqa``, ``protocolqa2``, ``seqqa2``, ``sourcequality``, ``suppqa2``, ``tableqa2`` (and ``tableqa2-img`` / ``tableqa2-pdf``), ``trialqa``. (default: `'litqa3'`)
+- `mode` (Mode): How a question's data files are delivered to the model. A no-op for tags without files (such as litqa3). Options: ``file``: Files uploaded via API. PDFs/images attached as context; other files as document attachments., ``inject``: Text file contents concatenated into the prompt as text., ``retrieve``: Only file names/stems are given; prompt instructs the agent to retrieve the necessary sequences or data from a source of its choosing. File contents are withheld. (default: `'file'`)
 - `solver` (Solver | None): The solver to run. Defaults to ``bare()`` (the benchmark's "bare" configuration: a plain single-turn ``generate()``) when not provided. Pass any Inspect solver to override, e.g. ``-T solver=bare`` on the CLI. (default: `None`)
 <!-- /Parameters: Automatically Generated -->
 
@@ -73,29 +82,68 @@ This eval uses the public `EdisonScientific/labbench2` dataset on Hugging Face,
 
 ### Supported tags
 
-| Tag             | Samples | File-bearing | `mode` to use          | Notes                       |
-| --------------- | ------- | ------------ | ---------------------- | --------------------------- |
-| `litqa3`        | 168     | No           | any (mode is a no-op)  | Literature reasoning.       |
-| `patentqa`      | 121     | No           | any (mode is a no-op)  | Patent comprehension.       |
-| `protocolqa2`   | 125     | Yes          | `file`                 | Lab protocols.              |
-| `sourcequality` | 150     | Yes          | `file`                 | Source quality assessment.  |
-| `trialqa`       | 120     | No           | any (mode is a no-op)  | Clinical trial QA.          |
+| Tag             | Samples | File-bearing | `mode` to use          | Notes                                  |
+| --------------- | ------- | ------------ | ---------------------- |----------------------------------------|
+| `cloning`       | 14      | Yes          | any (all modes)        | Cloning protocols; reward-scored.      |
+| `dbqa2`         | 86      | No           | any (mode is a no-op)  | Database access; recall judge.         |
+| `figqa2`        | 101     | No           | any (mode is a no-op)  | Figure QA; exact-match judge.          |
+| `figqa2-img`    | 101     | Yes          | `file`                 | Figure QA with image files.            |
+| `figqa2-pdf`    | 101     | Yes          | `file`                 | Figure QA with PDF files.              |
+| `litqa3`        | 168     | No           | any (mode is a no-op)  | Literature reasoning.                  |
+| `patentqa`      | 121     | No           | any (mode is a no-op)  | Patent comprehension.                  |
+| `protocolqa2`   | 125     | Yes          | `file`                 | Lab protocols.                         |
+| `seqqa2`        | 400     | Yes          | `file` / `inject`      | Sequence QA; deterministic validators. |
+| `sourcequality` | 150     | Yes          | `file`                 | Source quality assessment.             |
+| `suppqa2`       | 125     | No           | any (mode is a no-op)  | Supplement QA; exact-match.            |
+| `tableqa2`      | 100     | No           | any (mode is a no-op)  | Table QA; exact-match judge.           |
+| `tableqa2-img`  | 100     | Yes          | `file`                 | Table QA with image files.             |
+| `tableqa2-pdf`  | 100     | Yes          | `file`                 | Table QA with PDF files.               |
+| `trialqa`       | 120     | No           | any (mode is a no-op)  | Clinical trial QA.                     |
 
 For file-bearing tags, the loader filters out questions that don't opt into
 the requested `mode` (per each question's `QuestionMode` flags in the HF
-data). At the time of writing every `protocolqa2` and `sourcequality`
-question opts into `file` only, so passing any other `mode` would load zero
-samples. The mode flags live in the dataset and may change — verify by
-running with the configuration you intend before drawing conclusions from
-sample counts.
+data).
+
+#### Relationship between the tag and mode parameters
+
+Tags describe groups of samples/questions whilst mode describes how data files are uploaded.
+Not every sample is compatible with each mode of data uploading; if incompatible they are not loaded into the eval.
+Each sample in the dataset contains flags for compatible modes - this may change and sample counts
+can be verified by running with the configuration you intend before drawing conclusions from sample counts.
+
+For most tags, those that uses files requires the `file` mode. For example;
+
+`uv run inspect eval lab_bench_2 -T tag=sourcequality -T mode=retrieve`
+
+ Will result in no samples being loaded in. This is also true for tags protocolqa2`,`sourcequality`,`figqa2-img`,`figqa2-pdf`,`tableqa2-img`,`tableqa2-pdf`.
+
+Note that the base `figqa2`, `tableqa2`, and `suppqa2` tags have no files (mode is a no-op). Their image/PDF variants do have files and are impacted by the above.
+
+`seqqa2` is the exception: all of its samples are compatible with `file` and `inject`, while only a
+subset of this tag can be used with`retrieve` (so `mode="retrieve"` loads fewer samples).
 
 ## Scoring
 
-Answers are graded by a semantic LLM judge. The judge compares the model's
-answer to the reference, accepting semantically or numerically equivalent
-answers, and returns one of `correct` / `incorrect` / `unsure`; a `correct`
-verdict scores 1.0 and everything else (including unparseable or empty
-judgements) scores 0.0. Reported metrics are `accuracy` and `stderr`.
+There are different scoring methods for the tags.
+Some tags are scored deterministically (see `cloning` and `seqqa2`) but most are graded by an LLM judge.
+
+### LLM judge scorers
+
+The judge compares the solver's answer to the reference, accepting semantically
+or numerically equivalent answers, and returns one of `correct` / `incorrect` / `unsure`;
+a `correct` verdict scores 1.0 and everything else (including unparseable or empty judgements)
+scores 0.0. Reported metrics are `accuracy` and `stderr`.
+
+The judge requests **structured output** (a typed `result` / `rationale`
+schema), so the verdict is read from a typed field rather than scraped from
+prose. If the grader's provider doesn't support structured output, it falls back
+to parsing a `result:` line from the response.
+
+The judge prompt varies by tag: most tags use the default semantic prompt;
+`dbqa2` (database access) uses a recall-based variant that marks an answer
+correct when it recovers the expected reference values; and the figure, table,
+and supplement tags (`figqa2*`, `tableqa2*`, `suppqa2`) use an exact-match
+variant for numeric answers.
 
 The judge model is selected via the `grader` model role and defaults to
 `anthropic/claude-sonnet-4-5` at temperature 0. Override it on the command line,
@@ -107,6 +155,21 @@ uv run inspect eval lab_bench_2/lab_bench_2 \
   --model-role grader=anthropic/claude-opus-4-1-20250805
 ```
 
+### Deterministic scorers
+
+The `cloning` tag is not graded by an LLM judge: it is scored deterministically
+by labbench2's reward pipeline, which parses the submitted protocol, executes it
+(including Polymerase Chain Reaction simulation), and compares the result to the
+reference assembly via sequence-similarity and restriction-digest checks.
+PCR simulation requires that Go be available on the host. Without Go,
+protocol execution fails gracefully: PCR-based samples score 0.0 with an explanatory
+reason rather than crashing the run.
+
+The `seqqa2` tag is also scored deterministically. A
+per-question validator (selected by the question's `type`) checks the answer
+extracted via that question's `answer_regex`; extraction tolerates line-wrapped
+or whitespace-separated sequences.
+
 ## Attribution
 
 This evaluation depends on the reference implementation's

src/lab_bench_2/__init__.py

@@ -9,9 +9,13 @@
 from lab_bench_2.prompt_composer import Mode
 from lab_bench_2.scorers import (
     DEFAULT_GRADER_MODEL,
+    cloning_scorer,
+    exact_match_judge_scorer,
     parse_judge_verdict,
+    recall_judge_scorer,
     scorer_for_tag,
     semantic_judge_scorer,
+    seqqa2_scorer,
 )
 from lab_bench_2.solvers import bare
 
@@ -23,10 +27,14 @@
     "SUPPORTED_TAGS",
     "Mode",
     "bare",
+    "cloning_scorer",
+    "exact_match_judge_scorer",
     "lab_bench_2",
     "load_lab_bench_2_dataset",
     "parse_judge_verdict",
+    "recall_judge_scorer",
     "record_to_sample",
     "scorer_for_tag",
     "semantic_judge_scorer",
+    "seqqa2_scorer",
 ]

src/lab_bench_2/lab_bench_2.py

@@ -17,10 +17,20 @@
 from utils.metadata import load_version_from_yaml
 
 SUPPORTED_TAGS = (
+    "cloning",
+    "dbqa2",
+    "figqa2",
+    "figqa2-img",
+    "figqa2-pdf",
     "litqa3",
     "patentqa",
     "protocolqa2",
+    "seqqa2",
     "sourcequality",
+    "suppqa2",
+    "tableqa2",
+    "tableqa2-img",
+    "tableqa2-pdf",
     "trialqa",
 )
 
@@ -30,14 +40,17 @@
 @task
 def lab_bench_2(
     tag: str = "litqa3",
-    mode: Mode = "inject",
+    mode: Mode = "file",
     solver: Solver | None = None,
 ) -> Task:
     """LAB-Bench 2 evaluation task.
 
     Args:
-        tag: Which LAB-Bench 2 subset to run. Supported tags: ``litqa3``,
-            ``patentqa``, ``protocolqa2``, ``sourcequality``, ``trialqa``.
+        tag: Which LAB-Bench 2 subset to run. Supported tags: ``cloning``,
+            ``dbqa2``, ``figqa2`` (and ``figqa2-img`` / ``figqa2-pdf``),
+            ``litqa3``, ``patentqa``, ``protocolqa2``, ``seqqa2``,
+            ``sourcequality``, ``suppqa2``, ``tableqa2`` (and ``tableqa2-img`` /
+            ``tableqa2-pdf``), ``trialqa``.
         mode: How a question's data files are delivered to the model. A no-op
             for tags without files (such as litqa3). Options: