Add simplest lab_bench_2 case [1/n]

pyproject.toml

@@ -10,7 +10,6 @@ fallback_version = "0.0.0+unknown"
 
 [tool.setuptools.packages.find]
 where = ["src"]
-include = ["examples*", "utils*"]
 
 [tool.setuptools.package-data]
 "*" = ["eval.yaml"]
@@ -91,7 +90,7 @@ warn_redundant_casts = true
 warn_unused_configs = true
 
 [[tool.mypy.overrides]]
-module = ["examples.*", "utils.*"]
+module = ["examples.*", "utils.*", "lab_bench_2.*"]
 warn_return_any = true
 disallow_untyped_defs = true
 disallow_any_generics = true
@@ -107,16 +106,21 @@ disable_error_code = "unused-ignore"
 module = ["datasets", "datasets.*"]
 ignore_missing_imports = true
 
+[[tool.mypy.overrides]]
+# Upstream labbench2 git dep has no py.typed marker / stubs.
+module = ["evals", "evals.*"]
+ignore_missing_imports = true
+
 [[tool.mypy.overrides]]
 # Test files import these helpers via runtime sys.path manipulation, so
 # mypy can't resolve them statically. Treat them as untyped.
 module = ["add_readme_section", "generate_readmes"]
 ignore_missing_imports = true
 
 [project]
-name = "inspect-eval-template"
-description = "Template for Inspect AI evaluations"
-authors = [{ name = "Your Name" }]
+name = "lab-bench"
+description = "Benchmark of biology research tasks"
+authors = [{ name = "Isabelle Phan" }]
 readme = "README.md"
 requires-python = ">=3.11"
 license = "MIT"
@@ -134,11 +138,10 @@ classifiers = [
 dependencies = [
     "inspect_ai>=0.3.158",
     "pyyaml>=5.1.0",
-    # USER: Add your eval's dependencies here
+    "datasets>=4.8.5",
 ]
 
 [project.urls]
-# USER: Update these URLs
 "Source Code" = "https://github.com/your-org/your-eval"
 "Issue Tracker" = "https://github.com/your-org/your-eval/issues"
 
@@ -151,11 +154,20 @@ dependencies = [
 # should export your @task functions.
 # Note: examples are NOT registered -- only your evaluations under src/.
 [project.entry-points.inspect_ai]
+lab_bench_2 = "lab_bench_2"
+
+[project.optional-dependencies]
+# Reference implementation's deterministic scoring code (CC BY-SA 4.0), scoped to
+# this eval rather than core. Pulled as a dependency rather than vendored; source
+# + pin in [tool.uv.sources]. See src/lab_bench_2/README.md.
+lab_bench_2 = ["labbench2"]
+test = []
 
 [dependency-groups]
 dev = [
     "inspect-scout>=0.3.0",  # llm_scanner API used by agent_artefacts/trajectory_analysis/inspect_scout/
     "mypy",
+    "openai>=1.99.7",
     "pre-commit",
     "pytest",
     "pytest-asyncio",
@@ -167,10 +179,16 @@ dev = [
     "types-PyYAML",
 ]
 
+test_py311_or_lower = []
+test_py312_or_higher = []
+
 [tool.check-wheel-contents]
 ignore = ["W005", "W009"]
 
 [tool.uv]
 environments = [
     "python_version >= '3.11'",
 ]
+
+[tool.uv.sources]
+labbench2 = { git = "https://github.com/EdisonScientific/labbench2.git", rev = "c028ecdcf144b55ffcd92b68be45081df5628c20" }

src/lab_bench_2/.noautolint

@@ -0,0 +1,12 @@
+# external_dependencies: false positive. `utils` is the repo's first-party helper
+# package (src/utils, same distribution), used via the documented
+# load_version_from_yaml pattern (see src/examples/gpqa). It is not an external
+# dependency, but the check resolves it to the installed distribution and flags it.
+external_dependencies
+
+# tests_init: inapplicable to this template's layout. Eval packages live at the top
+# level (src/lab_bench_2), so a tests/lab_bench_2/__init__.py makes pytest's prepend
+# importer treat the test directory as the `lab_bench_2` package and shadow the
+# source package, breaking collection. The template's own example evals
+# (tests/examples/*) likewise omit __init__.py.
+tests_init

src/lab_bench_2/README.md

@@ -0,0 +1,105 @@
+# LABBench2
+
+LABBench2 evaluates language models and research agents on life-science tasks spanning literature reasoning, database access, figures, tables, protocols, source quality, sequence analysis, cloning, patents, and clinical trials.
+
+<!-- Contributors: Automatically Generated -->
+Contributed by [@iphan](https://github.com/iphan)
+<!-- /Contributors: Automatically Generated -->
+
+<!-- Usage: Automatically Generated -->
+## Usage
+
+First, install dependencies:
+
+```bash
+uv sync
+```
+
+Then run evaluations:
+
+```bash
+uv run inspect eval lab_bench_2/lab_bench_2 --model openai/gpt-5-nano
+```
+
+You can also import tasks as Python objects:
+
+```python
+from inspect_ai import eval
+from lab_bench_2 import lab_bench_2
+eval(lab_bench_2)
+```
+
+After running evaluations, view logs with:
+
+```bash
+uv run inspect view
+```
+
+If you don't want to specify `--model` each time, create a `.env` file:
+
+```bash
+INSPECT_EVAL_MODEL=anthropic/claude-opus-4-1-20250805
+ANTHROPIC_API_KEY=<anthropic-api-key>
+```
+<!-- /Usage: Automatically Generated -->
+
+<!-- Options: Automatically Generated -->
+## Options
+
+You can control a variety of options from the command line. For example:
+
+```bash
+uv run inspect eval lab_bench_2/lab_bench_2 --limit 10
+uv run inspect eval lab_bench_2/lab_bench_2 --max-connections 10
+uv run inspect eval lab_bench_2/lab_bench_2 --temperature 0.5
+```
+
+See `uv run inspect eval --help` for all available options.
+<!-- /Options: Automatically Generated -->
+
+<!-- Parameters: Automatically Generated -->
+## Parameters
+
+### `lab_bench_2`
+
+- `tag` (str): Which LAB-Bench 2 subset to run. Phase 1 supports only "litqa3". (default: `'litqa3'`)
+- `mode` (Mode): How a question's data files are delivered to the model. A no-op for tags without files (such as litqa3). Options: ``file``: Files uploaded via API. Smart routing: PDFs/images → context; other files → provider-side filesystem/container when code execution is enabled, else context., ``inject``: Text file contents concatenated into the prompt as text., ``retrieve``: Only file names/stems are given; prompt instructs the agent to retrieve the necessary sequences or data from a source of its choosing. File contents are withheld. (default: `'inject'`)
+- `solver` (Solver | None): The solver to run. Defaults to ``bare()`` (the benchmark's "bare" configuration: a plain single-turn ``generate()``) when not provided. Pass any Inspect solver to override, e.g. ``-T solver=bare`` on the CLI. (default: `None`)
+<!-- /Parameters: Automatically Generated -->
+
+## Dataset
+
+This eval uses the public `EdisonScientific/labbench2` dataset on Hugging Face, pinned to a specific commit for reproducibility.
+
+## Scoring
+
+Answers are graded by a semantic LLM judge. The judge compares the model's
+answer to the reference, accepting semantically or numerically equivalent
+answers, and returns one of `correct` / `incorrect` / `unsure`; a `correct`
+verdict scores 1.0 and everything else (including unparseable or empty
+judgements) scores 0.0. Reported metrics are `accuracy` and `stderr`.
+
+The judge model is selected via the `grader` model role and defaults to
+`anthropic/claude-sonnet-4-5` at temperature 0. Override it on the command line,
+for example:
+
+```bash
+uv run inspect eval lab_bench_2/lab_bench_2 \
+  --model openai/gpt-5-nano \
+  --model-role grader=anthropic/claude-opus-4-1-20250805
+```
+
+## Attribution
+
+This evaluation depends on the reference implementation's
+[`labbench2`](https://github.com/EdisonScientific/labbench2) package for its
+deterministic scientific scoring functions (used by the sequence and cloning
+tags). That code is licensed **CC BY-SA 4.0**. We pull it in as a dependency —
+pinned to a specific commit in `pyproject.toml` (`[tool.uv.sources]`) — rather
+than vendoring (copying) the code into this repository.
+
+## Evaluation Report
+
+TODO: A brief summary of results for your evaluation implementation compared against a standard set of existing results.
+
+## Changelog

src/lab_bench_2/__init__.py

@@ -0,0 +1,31 @@
+from lab_bench_2.dataset import (
+    LAB_BENCH_2_DATASET_PATH,
+    LAB_BENCH_2_DATASET_REVISION,
+    LAB_BENCH_2_DATASET_SPLIT,
+    load_lab_bench_2_dataset,
+    record_to_sample,
+)
+from lab_bench_2.lab_bench_2 import SUPPORTED_TAGS, Mode, lab_bench_2
+from lab_bench_2.scorers import (
+    DEFAULT_GRADER_MODEL,
+    parse_judge_verdict,
+    scorer_for_tag,
+    semantic_judge_scorer,
+)
+from lab_bench_2.solvers import bare
+
+__all__ = [
+    "DEFAULT_GRADER_MODEL",
+    "LAB_BENCH_2_DATASET_PATH",
+    "LAB_BENCH_2_DATASET_REVISION",
+    "LAB_BENCH_2_DATASET_SPLIT",
+    "SUPPORTED_TAGS",
+    "Mode",
+    "bare",
+    "lab_bench_2",
+    "load_lab_bench_2_dataset",
+    "parse_judge_verdict",
+    "record_to_sample",
+    "scorer_for_tag",
+    "semantic_judge_scorer",
+]

src/lab_bench_2/dataset.py

@@ -0,0 +1,52 @@
+"""Dataset loading for the LAB-Bench 2 evaluation.
+
+The questions live in a single (gated) HuggingFace dataset broken up by tags.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from inspect_ai.dataset import Dataset, Sample, hf_dataset
+
+LAB_BENCH_2_DATASET_PATH = "EdisonScientific/labbench2"
+LAB_BENCH_2_DATASET_REVISION = "27d12d72af24e3f70db8a99df63e567366cbdb80"
+LAB_BENCH_2_DATASET_SPLIT = "train"
+
+
+def record_to_sample(record: dict[str, Any]) -> Sample:
+    """Map a raw LAB-Bench 2 record to an Inspect Sample.
+
+    Used for file-less tags (e.g. litqa3): the question becomes the input and
+    the dataset's `ideal` field becomes the target the judge grades against.
+    """
+    question = record["question"]
+    prompt_suffix = record.get("prompt_suffix") or ""
+    input_text = f"{question}\n\n{prompt_suffix}".strip() if prompt_suffix else question
+    return Sample(
+        input=input_text,
+        target=record["ideal"],
+        id=record["id"],
+        metadata={
+            "tag": record.get("tag"),
+            "type": record.get("type") or None,
+            "sources": record.get("sources") or [],
+        },
+    )
+
+
+def load_lab_bench_2_dataset(tag: str, limit: int | None = None) -> Dataset:
+    """Load a single LAB-Bench 2 tag, pinned to a fixed dataset revision.
+
+    Args:
+        tag: The dataset config to load (e.g. "litqa3").
+        limit: Optional cap on the number of samples loaded.
+    """
+    return hf_dataset(
+        path=LAB_BENCH_2_DATASET_PATH,
+        name=tag,
+        split=LAB_BENCH_2_DATASET_SPLIT,
+        revision=LAB_BENCH_2_DATASET_REVISION,
+        sample_fields=record_to_sample,
+        limit=limit,
+    )

src/lab_bench_2/eval.yaml

@@ -0,0 +1,17 @@
+title: "LABBench2: An Improved Benchmark for AI Systems Performing Biology Research"
+description: |
+  LABBench2 is a benchmark for AI Systems Performing Biology Research. 
+  It evaluates language models and research agents on life-science tasks spanning 
+  literature reasoning, database access, figures, tables, protocols, source quality, 
+  sequence analysis, cloning, patents, and clinical trials.
+
+arxiv: https://arxiv.org/abs/2604.09554
+group: Reasoning
+contributors:
+  - iphan
+version: "1-A"
+tasks:
+  - name: lab_bench_2
+    dataset_samples: 168
+metadata:
+  requires_internet: true

src/lab_bench_2/lab_bench_2.py

@@ -0,0 +1,61 @@
+"""LAB-Bench 2: a benchmark of biology research tasks (arXiv:2604.09554).
+
+A single parameterized task selects a dataset `tag` and a file-delivery `mode`,
+and accepts an optional `solver` (defaulting to the benchmark's "bare"
+single-turn `generate()`).
+"""
+
+from __future__ import annotations
+
+from typing import Literal
+
+from inspect_ai import Task, task
+from inspect_ai.solver import Solver
+
+from lab_bench_2.dataset import load_lab_bench_2_dataset
+from lab_bench_2.scorers import scorer_for_tag
+from lab_bench_2.solvers import bare
+from utils.metadata import load_version_from_yaml
+
+Mode = Literal["file", "inject", "retrieve"]
+
+SUPPORTED_TAGS = ("litqa3",)
+
+EVAL_VERSION = load_version_from_yaml("lab_bench_2")
+
+
+@task
+def lab_bench_2(
+    tag: str = "litqa3",
+    mode: Mode = "inject",
+    solver: Solver | None = None,
+) -> Task:
+    """LAB-Bench 2 evaluation task.
+
+    Args:
+        tag: Which LAB-Bench 2 subset to run. Phase 1 supports only "litqa3".
+        mode: How a question's data files are delivered to the model. A no-op
+            for tags without files (such as litqa3). Options:
+
+            - ``file``: Files uploaded via API. Smart routing: PDFs/images →
+              context; other files → provider-side filesystem/container when
+              code execution is enabled, else context.
+            - ``inject``: Text file contents concatenated into the prompt as
+              text.
+            - ``retrieve``: Only file names/stems are given; prompt instructs
+              the agent to retrieve the necessary sequences or data from a
+              source of its choosing. File contents are withheld.
+        solver: The solver to run. Defaults to ``bare()`` (the benchmark's "bare"
+            configuration: a plain single-turn ``generate()``) when not provided.
+            Pass any Inspect solver to override, e.g. ``-T solver=bare`` on the CLI.
+    """
+    if tag not in SUPPORTED_TAGS:
+        raise NotImplementedError(
+            f"tag={tag!r} is not implemented yet; supported tags: {list(SUPPORTED_TAGS)}."
+        )
+    return Task(
+        dataset=load_lab_bench_2_dataset(tag),
+        solver=solver or bare(),
+        scorer=scorer_for_tag(tag),
+        version=EVAL_VERSION,
+    )