From c7d49be6860905569d7924e8ae919de6e6349e47 Mon Sep 17 00:00:00 2001
From: Isabelle Phan <nliphan@gmail.com>
Date: Fri, 29 May 2026 18:56:06 -0700
Subject: [PATCH 1/9] Add agentic "tools" solver for LAB-Bench 2

Add the benchmark's agentic configuration: a `tools()` solver giving
the model provider-native, server-side WebSearch and CodeExecution
---
 src/lab_bench_2/README.md         | 36 ++++++++++++++++++-
 src/lab_bench_2/__init__.py       |  6 +++-
 src/lab_bench_2/lab_bench_2.py    | 16 +++++----
 src/lab_bench_2/solvers.py        | 58 ++++++++++++++++++++++++++++++-
 tests/lab_bench_2/test_e2e.py     | 15 ++++++++
 tests/lab_bench_2/test_solvers.py | 23 +++++++++++-
 6 files changed, 143 insertions(+), 11 deletions(-)

diff --git a/src/lab_bench_2/README.md b/src/lab_bench_2/README.md
index 1e7d6e9..171742f 100644
--- a/src/lab_bench_2/README.md
+++ b/src/lab_bench_2/README.md
@@ -73,9 +73,43 @@ See `uv run inspect eval --help` for all available options.
 
 - `tag` (str): Which LAB-Bench 2 subset to run. Supported tags: ``cloning``, ``dbqa2``, ``figqa2`` (and ``figqa2-img`` / ``figqa2-pdf``), ``litqa3``, ``patentqa``, ``protocolqa2``, ``seqqa2``, ``sourcequality``, ``suppqa2``, ``tableqa2`` (and ``tableqa2-img`` / ``tableqa2-pdf``), ``trialqa``. (default: `'litqa3'`)
 - `mode` (Mode): How a question's data files are delivered to the model. A no-op for tags without files (such as litqa3). Options: ``file``: Files uploaded via API. PDFs/images attached as context; other files as document attachments., ``inject``: Text file contents concatenated into the prompt as text., ``retrieve``: Only file names/stems are given; prompt instructs the agent to retrieve the necessary sequences or data from a source of its choosing. File contents are withheld. (default: `'file'`)
-- `solver` (Solver | None): The solver to run. Defaults to ``bare()`` (the benchmark's "bare" configuration: a plain single-turn ``generate()``) when not provided. Pass any Inspect solver to override, e.g. ``-T solver=bare`` on the CLI. (default: `None`)
+- `solver` (SolverType): The solver to run. Options: ``bare``: a plain single-turn `generate()`., ``tools``: the server-side agentic configuration. The model is given provider-native, **server-side** tools — WebSearch and CodeExecution — and runs Inspect's tool-use loop (default: `'bare'`)
 <!-- /Parameters: Automatically Generated -->
 
+## Solvers
+
+The benchmark runs each model in two configurations, selected via the `solver`
+parameter:
+
+- **`bare`** (default): a plain single-turn `generate()` — no tools.
+- **`tools`**: the server-side agentic configuration. The model is given provider-native,
+  **server-side** tools — WebSearch and CodeExecution — and runs Inspect's
+  tool-use loop, which drives each provider's server-side tool round-trips. The
+  internal provider is auto-selected for the active model, so no external search
+  keys or local sandbox are required.
+
+Reasoning effort is set with Inspect's built-in `--reasoning-effort` flag (it
+applies to the model under test only, not the grader). The paper's "tools,high"
+case is:
+
+```bash
+uv run inspect eval lab_bench_2/lab_bench_2 -T tag=litqa3 -T solver=tools --reasoning-effort high
+```
+
+### WebFetch is omitted
+
+The reference benchmark's tool set also includes a **WebFetch** tool (Anthropic
+`web_fetch`, Google `url_context`) for retrieving the full content of a specific
+URL. It is **omitted here because Inspect has no native `web_fetch` wrapper** in
+any release, and no mechanism to pass a raw provider-native tool through. The
+capability exists at the providers — it simply is not surfaced through Inspect.
+Revisit if Inspect adds a web-fetch tool.
+
+One consequence: under `solver=tools`, `mode="retrieve"` (where the model is given
+only file names and must obtain the data itself) is degraded, since the model
+cannot reliably fetch a specific record's full content. Prefer `inject` or `file`
+with the agentic configuration.
+
 ## Dataset
 
 This eval uses the public `EdisonScientific/labbench2` dataset on Hugging Face, pinned to a specific commit for reproducibility.
diff --git a/src/lab_bench_2/__init__.py b/src/lab_bench_2/__init__.py
index a6e7b6f..7fb4124 100644
--- a/src/lab_bench_2/__init__.py
+++ b/src/lab_bench_2/__init__.py
@@ -17,7 +17,7 @@
     semantic_judge_scorer,
     seqqa2_scorer,
 )
-from lab_bench_2.solvers import bare
+from lab_bench_2.solvers import bare, native_tools, tools
 
 __all__ = [
     "DEFAULT_GRADER_MODEL",
@@ -26,15 +26,19 @@
     "LAB_BENCH_2_DATASET_SPLIT",
     "SUPPORTED_TAGS",
     "Mode",
+    "SolverType",
     "bare",
     "cloning_scorer",
     "exact_match_judge_scorer",
     "lab_bench_2",
     "load_lab_bench_2_dataset",
+    "native_tools",
     "parse_judge_verdict",
     "recall_judge_scorer",
     "record_to_sample",
     "scorer_for_tag",
     "semantic_judge_scorer",
     "seqqa2_scorer",
+    "solver_for_type",
+    "tools",
 ]
diff --git a/src/lab_bench_2/lab_bench_2.py b/src/lab_bench_2/lab_bench_2.py
index 98b39b1..f1eec86 100644
--- a/src/lab_bench_2/lab_bench_2.py
+++ b/src/lab_bench_2/lab_bench_2.py
@@ -8,12 +8,11 @@
 from __future__ import annotations
 
 from inspect_ai import Task, task
-from inspect_ai.solver import Solver
 
 from lab_bench_2.dataset import load_lab_bench_2_dataset
 from lab_bench_2.prompt_composer import Mode
 from lab_bench_2.scorers import scorer_for_tag
-from lab_bench_2.solvers import bare
+from lab_bench_2.solvers import SolverType, solver_for_type
 from utils.metadata import load_version_from_yaml
 
 SUPPORTED_TAGS = (
@@ -41,7 +40,7 @@
 def lab_bench_2(
     tag: str = "litqa3",
     mode: Mode = "file",
-    solver: Solver | None = None,
+    solver: SolverType = "bare",
 ) -> Task:
     """LAB-Bench 2 evaluation task.
 
@@ -61,9 +60,12 @@ def lab_bench_2(
             - ``retrieve``: Only file names/stems are given; prompt instructs
               the agent to retrieve the necessary sequences or data from a
               source of its choosing. File contents are withheld.
-        solver: The solver to run. Defaults to ``bare()`` (the benchmark's "bare"
-            configuration: a plain single-turn ``generate()``) when not provided.
-            Pass any Inspect solver to override, e.g. ``-T solver=bare`` on the CLI.
+        solver: The solver to run. Options:
+
+            - ``bare``: a plain single-turn `generate()`.
+            - ``tools``: the server-side agentic configuration. The model
+            is given provider-native, **server-side** tools — WebSearch and
+            CodeExecution — and runs Inspect's tool-use loop
     """
     if tag not in SUPPORTED_TAGS:
         raise NotImplementedError(
@@ -71,7 +73,7 @@ def lab_bench_2(
         )
     return Task(
         dataset=load_lab_bench_2_dataset(tag=tag, mode=mode),
-        solver=solver or bare(),
+        solver=solver_for_type(solver),
         scorer=scorer_for_tag(tag),
         version=EVAL_VERSION,
     )
diff --git a/src/lab_bench_2/solvers.py b/src/lab_bench_2/solvers.py
index 784e4ad..44bd356 100644
--- a/src/lab_bench_2/solvers.py
+++ b/src/lab_bench_2/solvers.py
@@ -6,10 +6,66 @@
 
 from __future__ import annotations
 
-from inspect_ai.solver import Solver, generate, solver
+from typing import Literal
+
+from inspect_ai.solver import Solver, chain, generate, solver, use_tools
+from inspect_ai.tool import Tool, code_execution, web_search
+
+SolverType = Literal["bare", "tools"]
 
 
 @solver
 def bare() -> Solver:
     """The benchmark's "bare" configuration: a plain single-turn ``generate()``."""
     return generate()
+
+
+def native_tools() -> list[Tool]:
+    """Provider-native, server-side tools for the agentic configuration.
+
+    ``web_search()`` and ``code_execution()`` run on the model provider's
+    servers; Inspect auto-selects the internal provider that matches the active
+    model. ``providers={"python": False}`` disables the sandboxed ``python()``
+    fallback so no local sandbox is required.
+
+    TODO Add web_fetch when Inspect adds support for it.
+    The paper uses a 3rd tool WebFetch, it is omitted here due to lack of
+    support in Inspect.
+    """
+    return [
+        web_search(),
+        code_execution(
+            providers={"python": False}
+        ),  # disable python fallback code_interpreter
+    ]
+
+
+@solver
+def tools() -> Solver:
+    """The benchmark's agentic ("tools") configuration.
+
+    Gives the model the provider-native tools from :func:`native_tools` and runs
+    Inspect's tool-use loop, which drives each provider's server-side tool
+    round-trips.
+    """
+    return chain(
+        use_tools(native_tools()),
+        generate(tool_calls="loop"),
+    )
+
+
+SOLVERS_BY_TYPE = {
+    "bare": bare,
+    "tools": tools,
+}
+
+
+def solver_for_type(solver_type: SolverType) -> Solver:
+    """Return the solver for a type, or raise if the type is not yet implemented."""
+    factory = SOLVERS_BY_TYPE.get(solver_type)
+    if factory is None:
+        raise NotImplementedError(
+            f"No solver implemented for type={solver_type!r}; "
+            f"supported types: {sorted(SOLVERS_BY_TYPE)}."
+        )
+    return factory()
diff --git a/tests/lab_bench_2/test_e2e.py b/tests/lab_bench_2/test_e2e.py
index fb125aa..2c1b796 100644
--- a/tests/lab_bench_2/test_e2e.py
+++ b/tests/lab_bench_2/test_e2e.py
@@ -10,6 +10,21 @@ def test_unsupported_tag_raises() -> None:
         lab_bench_2(tag="bogusqa")
 
 
+@pytest.mark.huggingface
+@pytest.mark.dataset_download
+def test_litqa3_tools_e2e() -> None:
+    # given the litqa3 task under the agentic (tools) solver, with a mock grader
+    # when
+    [log] = eval(
+        tasks=lab_bench_2(tag="litqa3", solver="tools"),
+        model="mockllm/model",
+        model_roles={"grader": "mockllm/model"},
+        limit=1,
+    )
+    # then the agentic configuration runs end to end
+    assert log.status == "success"
+
+
 @pytest.mark.huggingface
 @pytest.mark.dataset_download
 def test_litqa3_bare_e2e() -> None:
diff --git a/tests/lab_bench_2/test_solvers.py b/tests/lab_bench_2/test_solvers.py
index 67fe17a..d3b21fd 100644
--- a/tests/lab_bench_2/test_solvers.py
+++ b/tests/lab_bench_2/test_solvers.py
@@ -1,7 +1,28 @@
 from inspect_ai.solver import Solver
+from inspect_ai.tool import Tool, ToolDef
 
-from lab_bench_2.solvers import bare
+from lab_bench_2.solvers import bare, native_tools, tools
 
 
 def test_bare_returns_solver() -> None:
     assert isinstance(bare(), Solver)
+
+
+def test_tools_returns_solver() -> None:
+    assert isinstance(tools(), Solver)
+
+
+class TestNativeTools:
+    def test_returns_provider_native_search_and_code_execution(self) -> None:
+        # given / when
+        result = native_tools()
+        # then the two provider-native, server-side tools are present
+        names = {ToolDef(t).name for t in result}
+        assert all(isinstance(t, Tool) for t in result)
+        assert names == {"web_search", "code_execution"}
+
+    def test_omits_web_fetch(self) -> None:
+        # given Inspect has no native web_fetch wrapper (documented omission)
+        # when / then no fetch tool leaks into the agentic tool set
+        names = {ToolDef(t).name for t in native_tools()}
+        assert not any("fetch" in name for name in names)

From f911570e42e9372a1d1990890cb8d6a47f079db9 Mon Sep 17 00:00:00 2001
From: Isabelle Phan <nliphan@gmail.com>
Date: Mon, 1 Jun 2026 17:53:49 -0700
Subject: [PATCH 2/9] Refactor solvers.py into a solvers package

---
 src/lab_bench_2/solvers/__init__.py           | 22 +++++++++++++++
 src/lab_bench_2/solvers/registry.py           | 27 ++++++++++++++++++
 src/lab_bench_2/{ => solvers}/solvers.py      | 25 ++---------------
 tests/lab_bench_2/solvers/test_registry.py    | 28 +++++++++++++++++++
 .../lab_bench_2/{ => solvers}/test_solvers.py |  2 +-
 5 files changed, 80 insertions(+), 24 deletions(-)
 create mode 100644 src/lab_bench_2/solvers/__init__.py
 create mode 100644 src/lab_bench_2/solvers/registry.py
 rename src/lab_bench_2/{ => solvers}/solvers.py (69%)
 create mode 100644 tests/lab_bench_2/solvers/test_registry.py
 rename tests/lab_bench_2/{ => solvers}/test_solvers.py (93%)

diff --git a/src/lab_bench_2/solvers/__init__.py b/src/lab_bench_2/solvers/__init__.py
new file mode 100644
index 0000000..0d3d7c6
--- /dev/null
+++ b/src/lab_bench_2/solvers/__init__.py
@@ -0,0 +1,22 @@
+"""Solvers for the LAB-Bench 2 evaluation.
+
+The reference benchmark runs each model in two configurations: `bare` (no tools,
+single-turn) and an agentic configuration with provider-native tools. This package
+groups the solver factories and the registry that maps a ``SolverType`` to them.
+"""
+
+from lab_bench_2.solvers.registry import (
+    SOLVERS_BY_TYPE,
+    SolverType,
+    solver_for_type,
+)
+from lab_bench_2.solvers.solvers import bare, native_tools, tools
+
+__all__ = [
+    "SOLVERS_BY_TYPE",
+    "SolverType",
+    "bare",
+    "native_tools",
+    "solver_for_type",
+    "tools",
+]
diff --git a/src/lab_bench_2/solvers/registry.py b/src/lab_bench_2/solvers/registry.py
new file mode 100644
index 0000000..95c7ec0
--- /dev/null
+++ b/src/lab_bench_2/solvers/registry.py
@@ -0,0 +1,27 @@
+"""Registry mapping a ``SolverType`` to its solver factory."""
+
+from __future__ import annotations
+
+from typing import Literal
+
+from inspect_ai.solver import Solver
+
+from lab_bench_2.solvers.solvers import bare, tools
+
+SolverType = Literal["bare", "tools"]
+
+SOLVERS_BY_TYPE = {
+    "bare": bare,
+    "tools": tools,
+}
+
+
+def solver_for_type(solver_type: SolverType) -> Solver:
+    """Return the solver for a type, or raise if the type is not yet implemented."""
+    factory = SOLVERS_BY_TYPE.get(solver_type)
+    if factory is None:
+        raise NotImplementedError(
+            f"No solver implemented for type={solver_type!r}; "
+            f"supported types: {sorted(SOLVERS_BY_TYPE)}."
+        )
+    return factory()
diff --git a/src/lab_bench_2/solvers.py b/src/lab_bench_2/solvers/solvers.py
similarity index 69%
rename from src/lab_bench_2/solvers.py
rename to src/lab_bench_2/solvers/solvers.py
index 44bd356..c0f7113 100644
--- a/src/lab_bench_2/solvers.py
+++ b/src/lab_bench_2/solvers/solvers.py
@@ -1,18 +1,14 @@
-"""Solvers for the LAB-Bench 2 evaluation.
+"""Solver factories for the LAB-Bench 2 evaluation.
 
 The reference benchmark runs each model in two configurations: `bare` (no tools,
-single-turn) and an agentic configuration with provider-native tools.
+single-turn) and an agentic configuration with provider-native, server-side tools.
 """
 
 from __future__ import annotations
 
-from typing import Literal
-
 from inspect_ai.solver import Solver, chain, generate, solver, use_tools
 from inspect_ai.tool import Tool, code_execution, web_search
 
-SolverType = Literal["bare", "tools"]
-
 
 @solver
 def bare() -> Solver:
@@ -52,20 +48,3 @@ def tools() -> Solver:
         use_tools(native_tools()),
         generate(tool_calls="loop"),
     )
-
-
-SOLVERS_BY_TYPE = {
-    "bare": bare,
-    "tools": tools,
-}
-
-
-def solver_for_type(solver_type: SolverType) -> Solver:
-    """Return the solver for a type, or raise if the type is not yet implemented."""
-    factory = SOLVERS_BY_TYPE.get(solver_type)
-    if factory is None:
-        raise NotImplementedError(
-            f"No solver implemented for type={solver_type!r}; "
-            f"supported types: {sorted(SOLVERS_BY_TYPE)}."
-        )
-    return factory()
diff --git a/tests/lab_bench_2/solvers/test_registry.py b/tests/lab_bench_2/solvers/test_registry.py
new file mode 100644
index 0000000..ae128fe
--- /dev/null
+++ b/tests/lab_bench_2/solvers/test_registry.py
@@ -0,0 +1,28 @@
+from typing import cast
+
+import pytest
+from inspect_ai.solver import Solver
+
+from lab_bench_2.solvers.registry import (
+    SOLVERS_BY_TYPE,
+    SolverType,
+    solver_for_type,
+)
+
+
+class TestSolverForType:
+    def test_bare_and_tools_are_registered(self) -> None:
+        # given / when / then the supported types are exactly bare and tools
+        assert set(SOLVERS_BY_TYPE) == {"bare", "tools"}
+
+    def test_returns_a_solver_for_each_registered_type(self) -> None:
+        # given each registered solver type
+        # when / then a Solver instance is produced
+        for solver_type in SOLVERS_BY_TYPE:
+            assert isinstance(solver_for_type(cast(SolverType, solver_type)), Solver)
+
+    def test_raises_for_unimplemented_type(self) -> None:
+        # given a solver type that is not registered
+        # when / then a NotImplementedError is raised
+        with pytest.raises(NotImplementedError):
+            solver_for_type(cast(SolverType, "nonexistent"))
diff --git a/tests/lab_bench_2/test_solvers.py b/tests/lab_bench_2/solvers/test_solvers.py
similarity index 93%
rename from tests/lab_bench_2/test_solvers.py
rename to tests/lab_bench_2/solvers/test_solvers.py
index d3b21fd..8c08eb1 100644
--- a/tests/lab_bench_2/test_solvers.py
+++ b/tests/lab_bench_2/solvers/test_solvers.py
@@ -1,7 +1,7 @@
 from inspect_ai.solver import Solver
 from inspect_ai.tool import Tool, ToolDef
 
-from lab_bench_2.solvers import bare, native_tools, tools
+from lab_bench_2.solvers.solvers import bare, native_tools, tools
 
 
 def test_bare_returns_solver() -> None:

From beefea5f7f2aa263de03e222adc961bdd3e6c684 Mon Sep 17 00:00:00 2001
From: Isabelle Phan <nliphan@gmail.com>
Date: Mon, 1 Jun 2026 18:03:43 -0700
Subject: [PATCH 3/9] Add the agentic solver and sandbox environment

Client-side agent running in a sandbox with tools.
(inert not yet wired into SolverType/dispatch)

Co-Authored-By: Sunishchal Dev <Sunishchal@users.noreply.github.com>
---
 src/lab_bench_2/solvers/Dockerfile       |   3 +
 src/lab_bench_2/solvers/agent.py         | 105 +++++++++++++++++++++++
 src/lab_bench_2/solvers/compose.yaml     |   8 ++
 src/lab_bench_2/solvers/sandbox_tools.py |  38 ++++++++
 4 files changed, 154 insertions(+)
 create mode 100644 src/lab_bench_2/solvers/Dockerfile
 create mode 100644 src/lab_bench_2/solvers/agent.py
 create mode 100644 src/lab_bench_2/solvers/compose.yaml
 create mode 100644 src/lab_bench_2/solvers/sandbox_tools.py

diff --git a/src/lab_bench_2/solvers/Dockerfile b/src/lab_bench_2/solvers/Dockerfile
new file mode 100644
index 0000000..1b93a3e
--- /dev/null
+++ b/src/lab_bench_2/solvers/Dockerfile
@@ -0,0 +1,3 @@
+FROM aisiuk/inspect-tool-support
+
+RUN pip install biopython pydna primer3-py pandas numpy scipy
diff --git a/src/lab_bench_2/solvers/agent.py b/src/lab_bench_2/solvers/agent.py
new file mode 100644
index 0000000..dc9a55e
--- /dev/null
+++ b/src/lab_bench_2/solvers/agent.py
@@ -0,0 +1,105 @@
+"""Shared agentic solver with final-warning mechanism for LabBench2 tasks.
+
+Wraps basic_agent with a message-limit-aware final-warning prompt that
+forces the agent to submit before running out of time.
+"""
+
+from textwrap import dedent
+from typing import Any
+
+from inspect_ai.model import ChatMessageTool, ChatMessageUser, execute_tools, get_model
+from inspect_ai.solver import Generate, Solver, TaskState, basic_agent, solver
+from inspect_ai.util import LimitExceededError
+
+RETRY_MESSAGE = dedent("""\
+    Your last python() call was empty or malformed.
+    Think step-by-step (<=2 lines) **then** emit a python(code: str) call for the SAME question.
+    If the code would be >2000 chars, write it to sequence.json first and import from there.""")
+
+FINAL_WARNING_MESSAGE = dedent("""\
+    You are running out of time and MUST submit your answer NOW.
+    Based on everything you have learned so far, use the submit()
+    tool to submit your single best answer. Do NOT run any more
+    code — just submit your best guess immediately.""")
+
+
+@solver
+def agent_with_final_warning(
+    warning_limit: int = 45,
+    **agent_kwargs: Any,
+) -> Solver:
+    """Wrap basic_agent with a final-warning mechanism.
+
+    Runs basic_agent with a message_limit of `warning_limit`. If the agent
+    hits that limit without submitting, injects a "submit NOW" prompt and
+    gives the model one more generation with tools to submit its best guess.
+    """
+    agent_solver = basic_agent(message_limit=warning_limit, **agent_kwargs)
+
+    async def solve(state: TaskState, generate: Generate) -> TaskState:
+        try:
+            state = await agent_solver(state, generate)
+        except LimitExceededError:
+            pass  # Agent hit message limit without submitting — expected
+
+        # Bump limit generously: room for dangling-tool resolution (2),
+        # final-warning user message (1), model response (1), and tool
+        # execution results (2).
+        state.message_limit = len(state.messages) + 6
+
+        # Resolve any dangling tool calls from the interrupted agent.
+        if state.output and state.output.message.tool_calls:
+            resolved_ids = {
+                msg.tool_call_id
+                for msg in state.messages
+                if isinstance(msg, ChatMessageTool) and msg.tool_call_id
+            }
+            has_pending = any(
+                tc.id not in resolved_ids for tc in state.output.message.tool_calls
+            )
+            if has_pending:
+                tool_results, _ = await execute_tools(
+                    [state.output.message], state.tools
+                )
+                state.messages.extend(tool_results)
+
+        # Check if submit was already called (either normally or via dangling resolution)
+        submitted = any(
+            isinstance(msg, ChatMessageTool) and msg.function == "submit"
+            for msg in state.messages
+        )
+        if submitted:
+            for msg in reversed(state.messages):
+                if isinstance(msg, ChatMessageTool) and msg.function == "submit":
+                    state.output.completion = msg.text
+                    break
+            return state
+
+        # Inject warning, one final generate
+        state.messages.append(ChatMessageUser(content=FINAL_WARNING_MESSAGE))
+
+        output = await get_model().generate(input=state.messages, tools=state.tools)
+        state.messages.append(output.message)
+        state.output = output
+
+        if output.message.tool_calls:
+            # Extract submit answer directly from tool call arguments so
+            # the completion is captured even if execute_tools is blocked
+            # by the message limit.
+            for tc in output.message.tool_calls:
+                if tc.function == "submit":
+                    answer = (
+                        tc.arguments.get("answer", "")
+                        if isinstance(tc.arguments, dict)
+                        else ""
+                    )
+                    state.output.completion = answer
+                    break
+
+            # Still execute the tools for proper message-state bookkeeping
+            tool_results, _ = await execute_tools([output.message], state.tools)
+            state.messages.extend(tool_results)
+
+        return state
+
+    return solve
diff --git a/src/lab_bench_2/solvers/compose.yaml b/src/lab_bench_2/solvers/compose.yaml
new file mode 100644
index 0000000..0961458
--- /dev/null
+++ b/src/lab_bench_2/solvers/compose.yaml
@@ -0,0 +1,8 @@
+services:
+  default:
+    build:
+      context: "."
+      dockerfile: "Dockerfile"
+    command: "tail -f /dev/null"
+    init: true
+    stop_grace_period: 1s
diff --git a/src/lab_bench_2/solvers/sandbox_tools.py b/src/lab_bench_2/solvers/sandbox_tools.py
new file mode 100644
index 0000000..650eef9
--- /dev/null
+++ b/src/lab_bench_2/solvers/sandbox_tools.py
@@ -0,0 +1,38 @@
+"""Tool helpers for LabBench2 agentic evaluations."""
+
+from __future__ import annotations
+
+import os
+
+from inspect_ai.tool import Tool, bash, python, web_search
+
+
+def _build_web_search() -> Tool | None:
+    """Build web_search with an external provider.
+
+    Internal providers (openai, anthropic, gemini) use the model's built-in
+    search but cannot coexist with other tools like python() and bash()
+    (Gemini raises "does not yet support native web search concurrently
+    with other tools").  We use external-only providers so search works
+    alongside code execution for every model.
+    """
+    if os.environ.get("TAVILY_API_KEY"):
+        return web_search("tavily")
+    if os.environ.get("EXA_API_KEY"):
+        return web_search("exa")
+    if os.environ.get("GOOGLE_CSE_API_KEY"):
+        return web_search("google")
+    return None
+
+
+def get_labbench2_tools(timeout: int = 180) -> list[Tool]:
+    """Return the standard LabBench2 tool set (python, bash, optionally web_search).
+
+    web_search is included when an external provider key is configured
+    (TAVILY_API_KEY, EXA_API_KEY, or GOOGLE_CSE_API_KEY).
+    """
+    tools: list[Tool] = [python(timeout=timeout), bash(timeout=timeout)]
+    ws = _build_web_search()
+    if ws is not None:
+        tools.append(ws)
+    return tools

From 51ef4dafe066160e3690fd3bdde479b558cb3722 Mon Sep 17 00:00:00 2001
From: Isabelle Phan <nliphan@gmail.com>
Date: Mon, 1 Jun 2026 18:23:24 -0700
Subject: [PATCH 4/9] Wire the agentic solver into the task and registry

Expose `agentic` as a third SolverType
Attach a Docker sandbox only for this new solver
---
 src/lab_bench_2/__init__.py                   | 12 ++-
 src/lab_bench_2/lab_bench_2.py                |  9 +-
 src/lab_bench_2/solvers/__init__.py           |  6 ++
 src/lab_bench_2/solvers/agent.py              | 86 ++++++++++++++--
 src/lab_bench_2/solvers/registry.py           | 20 +++-
 src/lab_bench_2/solvers/sandbox_tools.py      | 39 +++++---
 tests/lab_bench_2/solvers/test_agent.py       | 97 +++++++++++++++++++
 tests/lab_bench_2/solvers/test_registry.py    | 25 ++++-
 .../lab_bench_2/solvers/test_sandbox_tools.py | 33 +++++++
 9 files changed, 296 insertions(+), 31 deletions(-)
 create mode 100644 tests/lab_bench_2/solvers/test_agent.py
 create mode 100644 tests/lab_bench_2/solvers/test_sandbox_tools.py

diff --git a/src/lab_bench_2/__init__.py b/src/lab_bench_2/__init__.py
index 7fb4124..0d74074 100644
--- a/src/lab_bench_2/__init__.py
+++ b/src/lab_bench_2/__init__.py
@@ -17,7 +17,15 @@
     semantic_judge_scorer,
     seqqa2_scorer,
 )
-from lab_bench_2.solvers import bare, native_tools, tools
+from lab_bench_2.solvers import (
+    SolverType,
+    agentic,
+    bare,
+    native_tools,
+    sandbox_tools,
+    solver_for_type,
+    tools,
+)
 
 __all__ = [
     "DEFAULT_GRADER_MODEL",
@@ -27,6 +35,7 @@
     "SUPPORTED_TAGS",
     "Mode",
     "SolverType",
+    "agentic",
     "bare",
     "cloning_scorer",
     "exact_match_judge_scorer",
@@ -36,6 +45,7 @@
     "parse_judge_verdict",
     "recall_judge_scorer",
     "record_to_sample",
+    "sandbox_tools",
     "scorer_for_tag",
     "semantic_judge_scorer",
     "seqqa2_scorer",
diff --git a/src/lab_bench_2/lab_bench_2.py b/src/lab_bench_2/lab_bench_2.py
index f1eec86..0ba3a44 100644
--- a/src/lab_bench_2/lab_bench_2.py
+++ b/src/lab_bench_2/lab_bench_2.py
@@ -12,7 +12,7 @@
 from lab_bench_2.dataset import load_lab_bench_2_dataset
 from lab_bench_2.prompt_composer import Mode
 from lab_bench_2.scorers import scorer_for_tag
-from lab_bench_2.solvers import SolverType, solver_for_type
+from lab_bench_2.solvers import SolverType, sandbox_for_solver, solver_for_type
 from utils.metadata import load_version_from_yaml
 
 SUPPORTED_TAGS = (
@@ -65,7 +65,11 @@ def lab_bench_2(
             - ``bare``: a plain single-turn `generate()`.
             - ``tools``: the server-side agentic configuration. The model
             is given provider-native, **server-side** tools — WebSearch and
-            CodeExecution — and runs Inspect's tool-use loop
+            CodeExecution — and runs Inspect's tool-use loop.
+            - ``agentic``: the client-side agentic configuration. The model is
+            given sandboxed ``python``/``bash`` (and, with an external provider
+            key, ``web_search``) tools in a Docker sandbox and must ``submit``
+            an answer. Requires Docker.
     """
     if tag not in SUPPORTED_TAGS:
         raise NotImplementedError(
@@ -75,5 +79,6 @@ def lab_bench_2(
         dataset=load_lab_bench_2_dataset(tag=tag, mode=mode),
         solver=solver_for_type(solver),
         scorer=scorer_for_tag(tag),
+        sandbox=sandbox_for_solver(solver),
         version=EVAL_VERSION,
     )
diff --git a/src/lab_bench_2/solvers/__init__.py b/src/lab_bench_2/solvers/__init__.py
index 0d3d7c6..fc8ba9c 100644
--- a/src/lab_bench_2/solvers/__init__.py
+++ b/src/lab_bench_2/solvers/__init__.py
@@ -5,18 +5,24 @@
 groups the solver factories and the registry that maps a ``SolverType`` to them.
 """
 
+from lab_bench_2.solvers.agent import agentic
 from lab_bench_2.solvers.registry import (
     SOLVERS_BY_TYPE,
     SolverType,
+    sandbox_for_solver,
     solver_for_type,
 )
+from lab_bench_2.solvers.sandbox_tools import sandbox_tools
 from lab_bench_2.solvers.solvers import bare, native_tools, tools
 
 __all__ = [
     "SOLVERS_BY_TYPE",
     "SolverType",
+    "agentic",
     "bare",
     "native_tools",
+    "sandbox_for_solver",
+    "sandbox_tools",
     "solver_for_type",
     "tools",
 ]
diff --git a/src/lab_bench_2/solvers/agent.py b/src/lab_bench_2/solvers/agent.py
index dc9a55e..f709984 100644
--- a/src/lab_bench_2/solvers/agent.py
+++ b/src/lab_bench_2/solvers/agent.py
@@ -8,13 +8,59 @@
 from typing import Any
 
 from inspect_ai.model import ChatMessageTool, ChatMessageUser, execute_tools, get_model
-from inspect_ai.solver import Generate, Solver, TaskState, basic_agent, solver
+from inspect_ai.solver import (
+    Generate,
+    Solver,
+    TaskState,
+    basic_agent,
+    solver,
+    system_message,
+)
 from inspect_ai.util import LimitExceededError
 
-RETRY_MESSAGE = dedent("""\
-    Your last python() call was empty or malformed.
-    Think step-by-step (<=2 lines) **then** emit a python(code: str) call for the SAME question.
-    If the code would be >2000 chars, write it to sequence.json first and import from there.""")
+from lab_bench_2.file_downloader import list_files
+from lab_bench_2.solvers.sandbox_tools import sandbox_tools, web_search_available
+
+# Each agent turn is roughly one model generation plus one tool execution
+# (~2 messages); the +2 covers the system and initial user messages.
+DEFAULT_AGENTIC_MAX_TURNS = 80
+
+
+def build_sandbox_prompt(web_search: bool) -> str:
+    """System prompt for the agentic sandbox.
+
+    The available-tools sentence reflects whether a ``web_search`` tool is
+    actually present (it is only added when an external provider key is set), so
+    the model is never told about a tool it does not have.
+    """
+    tools_clause = (
+        "Python, Bash, and Web Search tools" if web_search else "Python and Bash tools"
+    )
+    return dedent(f"""\
+    You are a helpful assistant completing a scientific research task. You have \
+access to {tools_clause} in a sandboxed environment. The \
+following Python libraries are pre-installed: biopython, pydna, primer3-py, \
+pandas, numpy, scipy.
+
+    Any files related to the question are in your working directory. Start by \
+running `ls` to see what is available, then use Python or Bash to read and \
+analyze them. Use Python when computational analysis would help answer the \
+question. Before taking an action, briefly describe your reasoning.
+
+    The environment is non-interactive — imports and variables do not persist \
+between calls. Each code execution is independent. If you need to work with long \
+sequences, save them to a file and load from that file in subsequent calls.
+
+    When you have your final answer, call the submit() tool. Be specific and \
+precise — provide exact numerical values, sequences, or lists as requested. For \
+numeric answers, provide the number without units unless specifically asked.
+
+    PLEASE BE CONCISE WITH YOUR OUTPUT AND CODE.""")
+
+
+CONTINUE_MESSAGE = dedent("""\
+    You did not call a tool. Think step-by-step (<=2 lines), then either call a \
+tool to make progress on the question, or call submit() with your final answer.""")
 
 FINAL_WARNING_MESSAGE = dedent("""\
     You are running out of time and MUST submit your answer NOW.
@@ -23,6 +69,28 @@
     code — just submit your best guess immediately.""")
 
 
+@solver
+def agentic() -> Solver:
+    """The benchmark's client-side agentic configuration.
+
+    Runs an agent with sandboxed ``python``/``bash`` (and, when an external
+    provider key is set, ``web_search``) tools, wrapped in the final-warning
+    mechanism that forces a ``submit`` before the turn budget is exhausted.
+    Requires a Docker sandbox, which the task attaches for ``solver="agentic"``.
+    """
+    # Each agent turn produces ~2 messages (assistant + tool result), plus
+    # system and initial user message. Translate turns to message count.
+    message_limit = DEFAULT_AGENTIC_MAX_TURNS * 2 + 2
+    return agent_with_final_warning(
+            warning_limit=message_limit,
+            init=system_message(
+                build_sandbox_prompt(web_search=web_search_available())
+            ),
+            tools=sandbox_tools(),
+            continue_message=CONTINUE_MESSAGE,
+    )
+
+
 @solver
 def agent_with_final_warning(
     warning_limit: int = 45,
@@ -42,9 +110,11 @@ async def solve(state: TaskState, generate: Generate) -> TaskState:
         except LimitExceededError:
             pass  # Agent hit message limit without submitting — expected
 
-        # Bump limit generously: room for dangling-tool resolution (2),
-        # final-warning user message (1), model response (1), and tool
-        # execution results (2).
+        # basic_agent set the sample message_limit to warning_limit, and the
+        # agent just ran up against it — so the limit must be raised here or the
+        # final-warning generate below would immediately re-trip it. Allow room
+        # for dangling-tool resolution (2), the final-warning user message (1),
+        # the model response (1), and its tool execution results (2).
         state.message_limit = len(state.messages) + 6
 
         # Resolve any dangling tool calls from the interrupted agent.
diff --git a/src/lab_bench_2/solvers/registry.py b/src/lab_bench_2/solvers/registry.py
index 95c7ec0..eec072d 100644
--- a/src/lab_bench_2/solvers/registry.py
+++ b/src/lab_bench_2/solvers/registry.py
@@ -2,19 +2,26 @@
 
 from __future__ import annotations
 
+from pathlib import Path
 from typing import Literal
 
 from inspect_ai.solver import Solver
+from inspect_ai.util import SandboxEnvironmentType
 
+from lab_bench_2.solvers.agent import agentic
 from lab_bench_2.solvers.solvers import bare, tools
 
-SolverType = Literal["bare", "tools"]
+SolverType = Literal["bare", "tools", "agentic"]
 
 SOLVERS_BY_TYPE = {
     "bare": bare,
     "tools": tools,
+    "agentic": agentic,
 }
 
+# Docker sandbox shared by the client-side `agentic` solver.
+COMPOSE = str(Path(__file__).parent / "compose.yaml")
+
 
 def solver_for_type(solver_type: SolverType) -> Solver:
     """Return the solver for a type, or raise if the type is not yet implemented."""
@@ -25,3 +32,14 @@ def solver_for_type(solver_type: SolverType) -> Solver:
             f"supported types: {sorted(SOLVERS_BY_TYPE)}."
         )
     return factory()
+
+
+def sandbox_for_solver(solver_type: SolverType) -> SandboxEnvironmentType | None:
+    """Return the sandbox a solver needs, or ``None`` if it runs without one.
+
+    Only the client-side ``agentic`` solver executes tools in a sandbox; ``bare``
+    and ``tools`` run server-side and must not spin up a container.
+    """
+    if solver_type == "agentic":
+        return ("docker", COMPOSE)
+    return None
diff --git a/src/lab_bench_2/solvers/sandbox_tools.py b/src/lab_bench_2/solvers/sandbox_tools.py
index 650eef9..5a0af69 100644
--- a/src/lab_bench_2/solvers/sandbox_tools.py
+++ b/src/lab_bench_2/solvers/sandbox_tools.py
@@ -3,30 +3,37 @@
 from __future__ import annotations
 
 import os
+from typing import Literal
 
 from inspect_ai.tool import Tool, bash, python, web_search
 
+# External web-search providers, in priority order. Internal providers (openai,
+# anthropic, gemini) use the model's built-in search but cannot coexist with
+# other tools like python() and bash() (Gemini raises "does not yet support
+# native web search concurrently with other tools"), so we use external-only
+# providers, which work alongside code execution for every model.
+_WEB_SEARCH_PROVIDERS_BY_KEY: dict[str, Literal["tavily", "exa", "google"]] = {
+    "TAVILY_API_KEY": "tavily",
+    "EXA_API_KEY": "exa",
+    "GOOGLE_CSE_API_KEY": "google",
+}
 
-def _build_web_search() -> Tool | None:
-    """Build web_search with an external provider.
 
-    Internal providers (openai, anthropic, gemini) use the model's built-in
-    search but cannot coexist with other tools like python() and bash()
-    (Gemini raises "does not yet support native web search concurrently
-    with other tools").  We use external-only providers so search works
-    alongside code execution for every model.
-    """
-    if os.environ.get("TAVILY_API_KEY"):
-        return web_search("tavily")
-    if os.environ.get("EXA_API_KEY"):
-        return web_search("exa")
-    if os.environ.get("GOOGLE_CSE_API_KEY"):
-        return web_search("google")
+def web_search_available() -> bool:
+    """True when an external web-search provider key is configured."""
+    return any(os.environ.get(key) for key in _WEB_SEARCH_PROVIDERS_BY_KEY)
+
+
+def _build_web_search() -> Tool | None:
+    """Build web_search with the first configured external provider, if any."""
+    for key, provider in _WEB_SEARCH_PROVIDERS_BY_KEY.items():
+        if os.environ.get(key):
+            return web_search(provider)
     return None
 
 
-def get_labbench2_tools(timeout: int = 180) -> list[Tool]:
-    """Return the standard LabBench2 tool set (python, bash, optionally web_search).
+def sandbox_tools(timeout: int = 180) -> list[Tool]:
+    """Return the sandboxed client-side tool set (python, bash, optionally web_search).
 
     web_search is included when an external provider key is configured
     (TAVILY_API_KEY, EXA_API_KEY, or GOOGLE_CSE_API_KEY).
diff --git a/tests/lab_bench_2/solvers/test_agent.py b/tests/lab_bench_2/solvers/test_agent.py
new file mode 100644
index 0000000..c96cdff
--- /dev/null
+++ b/tests/lab_bench_2/solvers/test_agent.py
@@ -0,0 +1,97 @@
+from typing import Any
+
+from inspect_ai import Task, eval
+from inspect_ai.dataset import Sample
+from inspect_ai.model import ChatMessage, ModelOutput, get_model
+from inspect_ai.solver import Solver
+
+from lab_bench_2.solvers.agent import (
+    FINAL_WARNING_MESSAGE,
+    agent_with_final_warning,
+    agentic,
+)
+
+
+def test_agentic_returns_solver() -> None:
+    assert isinstance(agentic(), Solver)
+
+
+def _never_submit_until_warned(
+    messages: list[ChatMessage],
+    tools: Any,
+    tool_choice: Any,
+    config: Any,
+) -> ModelOutput:
+    """Submit only once the final-warning prompt has been injected.
+
+    Before the warning, the model returns plain content (no tool call), so the
+    agent loops until it exhausts its message budget without submitting.
+    """
+    last_text = messages[-1].text if messages else ""
+    if "MUST submit your answer NOW" in last_text:
+        return ModelOutput.for_tool_call(
+            model="mockllm/model",
+            tool_name="submit",
+            tool_arguments={"answer": "RECOVERED"},
+        )
+    return ModelOutput.from_content(
+        model="mockllm/model", content="still working, no final answer yet"
+    )
+
+
+def test_final_warning_recovers_answer_when_model_never_submits() -> None:
+    # given a model that only submits once the final-warning prompt is injected
+    model = get_model("mockllm/model", custom_outputs=_never_submit_until_warned)
+    task = Task(
+        dataset=[Sample(input="What is the answer?", target="RECOVERED")],
+        solver=agent_with_final_warning(warning_limit=4),
+    )
+    # when the agent runs out of turns without submitting
+    log = eval(task, model=model)[0]
+    # then the final-warning path recovers the submitted answer
+    assert log.status == "success"
+    assert log.samples is not None
+    assert log.samples[0].output.completion == "RECOVERED"
+    # and the warning was actually injected into the conversation
+    assert any(FINAL_WARNING_MESSAGE in m.text for m in log.samples[0].messages)
+
+
+def _dangling_submit(
+    messages: list[ChatMessage],
+    tools: Any,
+    tool_choice: Any,
+    config: Any,
+) -> ModelOutput:
+    """Emit a submit call that is cut off by the context window.
+
+    With ``stop_reason="model_length"``, basic_agent breaks its loop without
+    executing the tool, so the submit call is left dangling for the wrapper to
+    resolve.
+    """
+    output = ModelOutput.for_tool_call(
+        model="mockllm/model",
+        tool_name="submit",
+        tool_arguments={"answer": "DANGLED"},
+    )
+    output.choices[0].stop_reason = "model_length"
+    return output
+
+
+def test_recovers_dangling_submit_when_interrupted_mid_tool_call() -> None:
+    # given a model whose submit call is cut off by the context window, so
+    # basic_agent exits with the submit tool call still unresolved
+    model = get_model("mockllm/model", custom_outputs=_dangling_submit)
+    task = Task(
+        dataset=[Sample(input="What is the answer?", target="DANGLED")],
+        solver=agent_with_final_warning(warning_limit=50),
+    )
+    # when the wrapper resolves the dangling call
+    log = eval(task, model=model)[0]
+    # then the submit tool runs and its answer is recovered
+    assert log.status == "success"
+    assert log.samples is not None
+    sample = log.samples[0]
+    assert sample.output.completion == "DANGLED"
+    # and recovery came from resolving the dangling call, not from the
+    # final-warning fallback (which is never injected in this path)
+    assert all(FINAL_WARNING_MESSAGE not in m.text for m in sample.messages)
diff --git a/tests/lab_bench_2/solvers/test_registry.py b/tests/lab_bench_2/solvers/test_registry.py
index ae128fe..e4c39a5 100644
--- a/tests/lab_bench_2/solvers/test_registry.py
+++ b/tests/lab_bench_2/solvers/test_registry.py
@@ -6,14 +6,15 @@
 from lab_bench_2.solvers.registry import (
     SOLVERS_BY_TYPE,
     SolverType,
+    sandbox_for_solver,
     solver_for_type,
 )
 
 
 class TestSolverForType:
-    def test_bare_and_tools_are_registered(self) -> None:
-        # given / when / then the supported types are exactly bare and tools
-        assert set(SOLVERS_BY_TYPE) == {"bare", "tools"}
+    def test_registered_solver_types(self) -> None:
+        # given / when / then the supported types are bare, tools, and agentic
+        assert set(SOLVERS_BY_TYPE) == {"bare", "tools", "agentic"}
 
     def test_returns_a_solver_for_each_registered_type(self) -> None:
         # given each registered solver type
@@ -26,3 +27,21 @@ def test_raises_for_unimplemented_type(self) -> None:
         # when / then a NotImplementedError is raised
         with pytest.raises(NotImplementedError):
             solver_for_type(cast(SolverType, "nonexistent"))
+
+
+class TestSandboxForSolver:
+    def test_agentic_requires_a_docker_sandbox(self) -> None:
+        # given the client-side agentic solver
+        # when
+        spec = sandbox_for_solver("agentic")
+        # then a docker sandbox built from the package compose file is returned
+        assert isinstance(spec, tuple)
+        sandbox_type, compose = spec
+        assert sandbox_type == "docker"
+        assert compose.endswith("compose.yaml")
+
+    def test_server_side_solvers_need_no_sandbox(self) -> None:
+        # given the server-side solvers
+        # when / then no sandbox is attached
+        assert sandbox_for_solver("bare") is None
+        assert sandbox_for_solver("tools") is None
diff --git a/tests/lab_bench_2/solvers/test_sandbox_tools.py b/tests/lab_bench_2/solvers/test_sandbox_tools.py
new file mode 100644
index 0000000..aa7c546
--- /dev/null
+++ b/tests/lab_bench_2/solvers/test_sandbox_tools.py
@@ -0,0 +1,33 @@
+import pytest
+from inspect_ai.tool import Tool, ToolDef
+
+from lab_bench_2.solvers.sandbox_tools import sandbox_tools
+
+_WEB_SEARCH_KEYS = ("TAVILY_API_KEY", "EXA_API_KEY", "GOOGLE_CSE_API_KEY")
+
+
+class TestSandboxTools:
+    def test_code_tools_only_without_web_search_keys(
+        self, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        # given no external web-search provider keys
+        for key in _WEB_SEARCH_KEYS:
+            monkeypatch.delenv(key, raising=False)
+        # when
+        result = sandbox_tools()
+        # then only the sandboxed code-execution tools are present
+        names = {ToolDef(t).name for t in result}
+        assert all(isinstance(t, Tool) for t in result)
+        assert names == {"python", "bash"}
+
+    def test_adds_web_search_when_external_key_present(
+        self, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        # given a single external web-search provider key
+        for key in _WEB_SEARCH_KEYS:
+            monkeypatch.delenv(key, raising=False)
+        monkeypatch.setenv("TAVILY_API_KEY", "test-key")
+        # when
+        names = {ToolDef(t).name for t in sandbox_tools()}
+        # then web_search joins the code-execution tools
+        assert names == {"python", "bash", "web_search"}

From 29287c64c101b9c7195c7e4c0cfcb3c3410cd76e Mon Sep 17 00:00:00 2001
From: Isabelle Phan <nliphan@gmail.com>
Date: Mon, 1 Jun 2026 18:37:32 -0700
Subject: [PATCH 5/9] Copy question files into the agentic sandbox

---
 src/lab_bench_2/solvers/agent.py        | 45 +++++++++++++++++---
 tests/lab_bench_2/solvers/test_agent.py | 56 +++++++++++++++++++++++--
 2 files changed, 92 insertions(+), 9 deletions(-)

diff --git a/src/lab_bench_2/solvers/agent.py b/src/lab_bench_2/solvers/agent.py
index f709984..df43554 100644
--- a/src/lab_bench_2/solvers/agent.py
+++ b/src/lab_bench_2/solvers/agent.py
@@ -4,6 +4,8 @@
 forces the agent to submit before running out of time.
 """
 
+import logging
+from pathlib import Path
 from textwrap import dedent
 from typing import Any
 
@@ -13,10 +15,11 @@
     Solver,
     TaskState,
     basic_agent,
+    chain,
     solver,
     system_message,
 )
-from inspect_ai.util import LimitExceededError
+from inspect_ai.util import LimitExceededError, sandbox
 
 from lab_bench_2.file_downloader import list_files
 from lab_bench_2.solvers.sandbox_tools import sandbox_tools, web_search_available
@@ -68,26 +71,56 @@ def build_sandbox_prompt(web_search: bool) -> str:
     tool to submit your single best answer. Do NOT run any more
     code — just submit your best guess immediately.""")
 
+logger = logging.getLogger(__name__)
+
+
+@solver
+def copy_files_to_sandbox() -> Solver:
+    """Copy a question's downloaded files into the sandbox working directory.
+
+    Reads the files cached at ``metadata["files_path"]`` (set by the dataset
+    loader for file-bearing tags) and writes each into the sandbox cwd so the
+    agent can inspect them with ``python``/``bash``. A no-op for file-less tags
+    (e.g. litqa3), where no ``files_path`` is set.
+    """
+
+    async def solve(state: TaskState, generate: Generate) -> TaskState:
+        files_path = state.metadata.get("files_path")
+        if not files_path:
+            return state
+        env = sandbox()
+        for file in list_files(Path(files_path)):
+            logger.debug(f"Copying file into the sandbox: {file.name}")
+            await env.write_file(file.name, file.read_bytes())
+        return state
+
+    return solve
+
 
 @solver
 def agentic() -> Solver:
     """The benchmark's client-side agentic configuration.
 
-    Runs an agent with sandboxed ``python``/``bash`` (and, when an external
-    provider key is set, ``web_search``) tools, wrapped in the final-warning
-    mechanism that forces a ``submit`` before the turn budget is exhausted.
-    Requires a Docker sandbox, which the task attaches for ``solver="agentic"``.
+    Copies any question files into the sandbox, then runs an agent with
+    sandboxed ``python``/``bash`` (and, when an external provider key is set,
+    ``web_search``) tools, wrapped in the final-warning mechanism that forces a
+    ``submit`` before the turn budget is exhausted. Requires a Docker sandbox,
+    which the task attaches for ``solver="agentic"``.
     """
     # Each agent turn produces ~2 messages (assistant + tool result), plus
     # system and initial user message. Translate turns to message count.
     message_limit = DEFAULT_AGENTIC_MAX_TURNS * 2 + 2
-    return agent_with_final_warning(
+
+    return chain(
+        copy_files_to_sandbox(),
+        agent_with_final_warning(
             warning_limit=message_limit,
             init=system_message(
                 build_sandbox_prompt(web_search=web_search_available())
             ),
             tools=sandbox_tools(),
             continue_message=CONTINUE_MESSAGE,
+        ),
     )
 
 
diff --git a/tests/lab_bench_2/solvers/test_agent.py b/tests/lab_bench_2/solvers/test_agent.py
index c96cdff..8443b8e 100644
--- a/tests/lab_bench_2/solvers/test_agent.py
+++ b/tests/lab_bench_2/solvers/test_agent.py
@@ -1,14 +1,19 @@
-from typing import Any
+from pathlib import Path
+from typing import Any, cast
+from unittest.mock import AsyncMock, MagicMock
 
+import pytest
 from inspect_ai import Task, eval
 from inspect_ai.dataset import Sample
-from inspect_ai.model import ChatMessage, ModelOutput, get_model
-from inspect_ai.solver import Solver
+from inspect_ai.model import ChatMessage, ModelName, ModelOutput, get_model
+from inspect_ai.solver import Generate, Solver, TaskState
 
+import lab_bench_2.solvers.agent as agent_module
 from lab_bench_2.solvers.agent import (
     FINAL_WARNING_MESSAGE,
     agent_with_final_warning,
     agentic,
+    copy_files_to_sandbox,
 )
 
 
@@ -95,3 +100,48 @@ def test_recovers_dangling_submit_when_interrupted_mid_tool_call() -> None:
     # and recovery came from resolving the dangling call, not from the
     # final-warning fallback (which is never injected in this path)
     assert all(FINAL_WARNING_MESSAGE not in m.text for m in sample.messages)
+
+
+class TestCopyFilesToSandbox:
+    async def test_writes_each_question_file_into_the_sandbox(
+        self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        # given a question whose files are cached locally
+        (tmp_path / "seq.fasta").write_text(">a\nACGT\n")
+        (tmp_path / "data.csv").write_text("x,y\n1,2\n")
+        fake_env = MagicMock()
+        fake_env.write_file = AsyncMock()
+        monkeypatch.setattr(agent_module, "sandbox", lambda *a, **k: fake_env)
+        state = TaskState(
+            model=ModelName("mockllm/model"),
+            sample_id="files",
+            epoch=0,
+            input="q",
+            messages=[],
+            metadata={"files_path": str(tmp_path)},
+        )
+        # when
+        await copy_files_to_sandbox()(state, cast(Generate, AsyncMock()))
+        # then each file is written into the sandbox cwd by basename
+        written = {call.args[0] for call in fake_env.write_file.await_args_list}
+        assert written == {"seq.fasta", "data.csv"}
+
+    async def test_is_noop_without_files_path(
+        self, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        # given a question with no files (no files_path in metadata)
+        fake_env = MagicMock()
+        fake_env.write_file = AsyncMock()
+        monkeypatch.setattr(agent_module, "sandbox", lambda *a, **k: fake_env)
+        state = TaskState(
+            model=ModelName("mockllm/model"),
+            sample_id="no-files",
+            epoch=0,
+            input="q",
+            messages=[],
+            metadata={},
+        )
+        # when
+        await copy_files_to_sandbox()(state, cast(Generate, AsyncMock()))
+        # then nothing is written to the sandbox
+        fake_env.write_file.assert_not_awaited()

From 8538d42fa54666c664a6d82fbcb18063699954ef Mon Sep 17 00:00:00 2001
From: Isabelle Phan <nliphan@gmail.com>
Date: Mon, 1 Jun 2026 18:47:23 -0700
Subject: [PATCH 6/9] update readme and add e2e test

---
 src/lab_bench_2/README.md      | 15 +++++++++++----
 src/lab_bench_2/lab_bench_2.py |  5 ++---
 tests/lab_bench_2/test_e2e.py  | 29 +++++++++++++++++++++++++++++
 3 files changed, 42 insertions(+), 7 deletions(-)

diff --git a/src/lab_bench_2/README.md b/src/lab_bench_2/README.md
index 171742f..7200137 100644
--- a/src/lab_bench_2/README.md
+++ b/src/lab_bench_2/README.md
@@ -73,12 +73,12 @@ See `uv run inspect eval --help` for all available options.
 
 - `tag` (str): Which LAB-Bench 2 subset to run. Supported tags: ``cloning``, ``dbqa2``, ``figqa2`` (and ``figqa2-img`` / ``figqa2-pdf``), ``litqa3``, ``patentqa``, ``protocolqa2``, ``seqqa2``, ``sourcequality``, ``suppqa2``, ``tableqa2`` (and ``tableqa2-img`` / ``tableqa2-pdf``), ``trialqa``. (default: `'litqa3'`)
 - `mode` (Mode): How a question's data files are delivered to the model. A no-op for tags without files (such as litqa3). Options: ``file``: Files uploaded via API. PDFs/images attached as context; other files as document attachments., ``inject``: Text file contents concatenated into the prompt as text., ``retrieve``: Only file names/stems are given; prompt instructs the agent to retrieve the necessary sequences or data from a source of its choosing. File contents are withheld. (default: `'file'`)
-- `solver` (SolverType): The solver to run. Options: ``bare``: a plain single-turn `generate()`., ``tools``: the server-side agentic configuration. The model is given provider-native, **server-side** tools — WebSearch and CodeExecution — and runs Inspect's tool-use loop (default: `'bare'`)
+- `solver` (SolverType): The solver to run. Options: ``bare``: a plain single-turn `generate()`., ``tools``: the server-side agentic configuration. The model is given provider-native, **server-side** tools — WebSearch and CodeExecution — and runs Inspect's tool-use loop., ``agentic``: the client-side agentic configuration. The model is given ``python``/``bash`` (and, with an external provider key, ``web_search``) tools in a Docker sandbox. (default: `'bare'`)
 <!-- /Parameters: Automatically Generated -->
 
 ## Solvers
 
-The benchmark runs each model in two configurations, selected via the `solver`
+The benchmark can run each model in three configurations, selected via the `solver`
 parameter:
 
 - **`bare`** (default): a plain single-turn `generate()` — no tools.
@@ -87,6 +87,11 @@ parameter:
   tool-use loop, which drives each provider's server-side tool round-trips. The
   internal provider is auto-selected for the active model, so no external search
   keys or local sandbox are required.
+- **`agentic`**: the client-side agentic configuration. The model is given **sandboxed**
+  `python` / `bash` tools (plus `web_search` when an external provider key is set —
+  `TAVILY_API_KEY`, `EXA_API_KEY`, or `GOOGLE_CSE_API_KEY`) inside a **Docker sandbox**,
+  and must call `submit()` to answer. A question's data files are copied into the sandbox
+  working directory.
 
 Reasoning effort is set with Inspect's built-in `--reasoning-effort` flag (it
 applies to the model under test only, not the grader). The paper's "tools,high"
@@ -107,8 +112,10 @@ Revisit if Inspect adds a web-fetch tool.
 
 One consequence: under `solver=tools`, `mode="retrieve"` (where the model is given
 only file names and must obtain the data itself) is degraded, since the model
-cannot reliably fetch a specific record's full content. Prefer `inject` or `file`
-with the agentic configuration.
+cannot reliably fetch a specific record's full content — prefer `inject` or `file`.
+This limitation does not apply to `solver=agentic`: the question's files are copied
+into the sandbox, so `mode="retrieve"` (filenames in the prompt, contents on disk) is
+the recommended pairing there.
 
 ## Dataset
 
diff --git a/src/lab_bench_2/lab_bench_2.py b/src/lab_bench_2/lab_bench_2.py
index 0ba3a44..e2d4b59 100644
--- a/src/lab_bench_2/lab_bench_2.py
+++ b/src/lab_bench_2/lab_bench_2.py
@@ -67,9 +67,8 @@ def lab_bench_2(
             is given provider-native, **server-side** tools — WebSearch and
             CodeExecution — and runs Inspect's tool-use loop.
             - ``agentic``: the client-side agentic configuration. The model is
-            given sandboxed ``python``/``bash`` (and, with an external provider
-            key, ``web_search``) tools in a Docker sandbox and must ``submit``
-            an answer. Requires Docker.
+            given ``python``/``bash`` (and, with an external provider
+            key, ``web_search``) tools in a Docker sandbox.
     """
     if tag not in SUPPORTED_TAGS:
         raise NotImplementedError(
diff --git a/tests/lab_bench_2/test_e2e.py b/tests/lab_bench_2/test_e2e.py
index 2c1b796..6b392e8 100644
--- a/tests/lab_bench_2/test_e2e.py
+++ b/tests/lab_bench_2/test_e2e.py
@@ -1,5 +1,6 @@
 import pytest
 from inspect_ai import eval
+from inspect_ai.model import ModelOutput, get_model
 
 from lab_bench_2.lab_bench_2 import lab_bench_2
 from lab_bench_2.prompt_composer import Mode
@@ -25,6 +26,34 @@ def test_litqa3_tools_e2e() -> None:
     assert log.status == "success"
 
 
+@pytest.mark.huggingface
+@pytest.mark.dataset_download
+@pytest.mark.docker
+@pytest.mark.slow
+def test_litqa3_agentic_e2e() -> None:
+    # given the litqa3 task under the client-side agentic solver (Docker sandbox),
+    # with a model that submits immediately and a mock grader
+    model = get_model(
+        "mockllm/model",
+        custom_outputs=[
+            ModelOutput.for_tool_call(
+                model="mockllm/model",
+                tool_name="submit",
+                tool_arguments={"answer": "A"},
+            ),
+        ],
+    )
+    # when
+    [log] = eval(
+        tasks=lab_bench_2(tag="litqa3", solver="agentic"),
+        model=model,
+        model_roles={"grader": "mockllm/model"},
+        limit=1,
+    )
+    # then the agentic configuration runs end to end
+    assert log.status == "success"
+
+
 @pytest.mark.huggingface
 @pytest.mark.dataset_download
 def test_litqa3_bare_e2e() -> None:

From d9aa76fbb695eb22a4f865800d43d3f53ba65631 Mon Sep 17 00:00:00 2001
From: Isabelle Phan <nliphan@gmail.com>
Date: Tue, 2 Jun 2026 17:12:54 -0700
Subject: [PATCH 7/9] Pin python deps in sandbox for reproducibility

---
 src/lab_bench_2/solvers/Dockerfile | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/src/lab_bench_2/solvers/Dockerfile b/src/lab_bench_2/solvers/Dockerfile
index 1b93a3e..782849e 100644
--- a/src/lab_bench_2/solvers/Dockerfile
+++ b/src/lab_bench_2/solvers/Dockerfile
@@ -1,3 +1,14 @@
+# Scientific Python stack for the agentic solver's sandboxed python/bash tools.
+# Versions are pinned for reproducible builds. pydna constrains biopython,
+# numpy, pandas, and scipy, so bump them as a set (re-resolve, don't pin
+# individually). The base image is intentionally left unpinned: it must track
+# the inspect tool-support protocol of the installed inspect_ai.
 FROM aisiuk/inspect-tool-support
 
-RUN pip install biopython pydna primer3-py pandas numpy scipy
+RUN pip install --no-cache-dir \
+    biopython==1.87 \
+    pydna==5.5.12 \
+    primer3-py==2.3.0 \
+    pandas==3.0.3 \
+    numpy==2.4.6 \
+    scipy==1.17.1

From 15fb74c2aa3beffebc94109ae3817186c9b83ff9 Mon Sep 17 00:00:00 2001
From: Isabelle Phan <iphan@users.noreply.github.com>
Date: Thu, 4 Jun 2026 16:59:51 -0700
Subject: [PATCH 8/9] Readme tweaks from code review

Co-authored-by: Tania <120768997+ItsTania@users.noreply.github.com>
---
 src/lab_bench_2/README.md | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/lab_bench_2/README.md b/src/lab_bench_2/README.md
index 7200137..963bd6d 100644
--- a/src/lab_bench_2/README.md
+++ b/src/lab_bench_2/README.md
@@ -91,7 +91,7 @@ parameter:
   `python` / `bash` tools (plus `web_search` when an external provider key is set —
   `TAVILY_API_KEY`, `EXA_API_KEY`, or `GOOGLE_CSE_API_KEY`) inside a **Docker sandbox**,
   and must call `submit()` to answer. A question's data files are copied into the sandbox
-  working directory.
+  working directory. The initial docker image build should take ~ 2 minutes on a standard personal computer.
 
 Reasoning effort is set with Inspect's built-in `--reasoning-effort` flag (it
 applies to the model under test only, not the grader). The paper's "tools,high"
@@ -152,16 +152,16 @@ Not every sample is compatible with each mode of data uploading; if incompatible
 Each sample in the dataset contains flags for compatible modes - this may change and sample counts
 can be verified by running with the configuration you intend before drawing conclusions from sample counts.
 
-For most tags, those that uses files requires the `file` mode. For example;
+For most tags, those that use files requires the `file` mode. For example;
 
 `uv run inspect eval lab_bench_2 -T tag=sourcequality -T mode=retrieve`
 
- Will result in no samples being loaded in. This is also true for tags protocolqa2`,`sourcequality`,`figqa2-img`,`figqa2-pdf`,`tableqa2-img`,`tableqa2-pdf`.
+ Will result in no samples being loaded in. This is also true for tags `protocolqa2`,`sourcequality`,`figqa2-img`,`figqa2-pdf`,`tableqa2-img`,`tableqa2-pdf`.
 
 Note that the base `figqa2`, `tableqa2`, and `suppqa2` tags have no files (mode is a no-op). Their image/PDF variants do have files and are impacted by the above.
 
 `seqqa2` is the exception: all of its samples are compatible with `file` and `inject`, while only a
-subset of this tag can be used with`retrieve` (so `mode="retrieve"` loads fewer samples).
+subset of this tag can be used with `retrieve` (so `mode="retrieve"` loads fewer samples).
 
 ## Scoring
 

From ac775191d6070150c73d29fa9278001372f82973 Mon Sep 17 00:00:00 2001
From: Isabelle Phan <nliphan@gmail.com>
Date: Thu, 4 Jun 2026 18:19:21 -0700
Subject: [PATCH 9/9] Log warning if no search api key is found for the agent

---
 src/lab_bench_2/solvers/sandbox_tools.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/src/lab_bench_2/solvers/sandbox_tools.py b/src/lab_bench_2/solvers/sandbox_tools.py
index 5a0af69..1c24c0c 100644
--- a/src/lab_bench_2/solvers/sandbox_tools.py
+++ b/src/lab_bench_2/solvers/sandbox_tools.py
@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+import logging
 import os
 from typing import Literal
 
@@ -18,6 +19,8 @@
     "GOOGLE_CSE_API_KEY": "google",
 }
 
+logger = logging.getLogger(__name__)
+
 
 def web_search_available() -> bool:
     """True when an external web-search provider key is configured."""
@@ -42,4 +45,8 @@ def sandbox_tools(timeout: int = 180) -> list[Tool]:
     ws = _build_web_search()
     if ws is not None:
         tools.append(ws)
+    else:
+        logger.warn(
+            "No search provider api key found, so no web search tool is given to the agent"
+        )
     return tools