feat(v1): multi-agent api (#1939)

Co-authored-by: eligotts <78387377+eligotts@users.noreply.github.com>
Co-authored-by: Mika Senghaas <mail@mikasenghaas.de>

Author: 3 people

.gitignore

@@ -1,4 +1,4 @@
-# env 
+# env
 .venv
 .venv/
 venv/

assets/lab/environments/AGENTS.md

@@ -3,7 +3,7 @@
 > Managed by Prime Lab. Do not edit this file directly.
 > `prime lab setup` and `prime lab sync` refresh it. Put project-specific agent guidance in workspace-root `AGENTS.local.md`; agents should read that file after this one if it exists.
 
-This file mirrors the "Tasksets" documentation page.
+This file mirrors the "Tasksets" and "Multi-agent environments" documentation pages.
 
 ---
 
@@ -30,7 +30,7 @@ The command also supports:
   - Use this to create custom tools which are installed into supported harnesses via MCP.
 - `-U`, `--add-user` — also scaffold a `vf.User` simulator at `servers/user.py`
   - Use this to simulate a user interacting with the model. Not all harnesses support user simulation.
-- `-H`, `--add-harness` — also scaffold a custom `vf.Harness` at `harness.py`, selectable via `--harness.id <name>`
+- `-H`, `--add-harness` — also scaffold a custom `vf.Harness` at `harness.py`, selectable via `--env.agent.harness.id <name>`
   - Prefer a built-in harness unless the model needs to run inside a custom program.
 
 Most tasksets do not need specific tools, user simulations or custom harnesses.
@@ -116,7 +116,7 @@ class AdditionConfig(vf.TasksetConfig):
     task: AdditionTaskConfig = AdditionTaskConfig()
 ```
 
-These values can be overridden with `--taskset.num-tasks` and `--taskset.task.tolerance`, or with the equivalent TOML fields.
+These values can be overridden with `--env.taskset.num-tasks` and `--env.taskset.task.tolerance`, or with the equivalent TOML fields (`[env.taskset]`).
 
 ## Lazy and infinite tasksets
 
@@ -241,4 +241,107 @@ class JudgeTraceTaskset(vf.Taskset[JudgedTask, SetConfig]):
         ]
 ```
 
-To override the judge model, set `taskset.task.judge.model` in your config (it is a string).
+To override the judge model, set `env.taskset.task.judge.model` in your config (it is a string).
+
+## Beyond one agent
+
+One eval rollout doesn't have to be one agent run: agents, the control flow between
+agents, and cross-agent rewards are the environment's job — see
+[Multi-agent environments](https://github.com/PrimeIntellect-ai/verifiers/blob/main/docs/v1/environments.md).
+
+## Multi-agent environments
+
+One eval rollout doesn't have to be one agent run. `Environment` is abstract, and
+every run gets a concrete subclass: plain tasksets resolve to the bundled
+`SingleAgentEnv` (one `agent` playing the taskset), and a package can export
+its own (via `__all__`, alongside its [`Taskset`](https://github.com/PrimeIntellect-ai/verifiers/blob/main/docs/v1/tasksets.md) — the same plugin
+idiom as a bundled harness). An env declares its config as an `EnvConfig` subclass —
+each agent an `AgentConfig` field, plus its own knobs — writes `run()`, and
+optionally overrides `setup()` and `finalize()`:
+
+```python
+class DebateConfig(vf.EnvConfig):
+    pro: vf.AgentConfig = vf.AgentConfig()
+    con: vf.AgentConfig = vf.AgentConfig()
+    judge: vf.AgentConfig = vf.AgentConfig(model="openai/gpt-5-mini")
+
+
+class VerdictTask(vf.Task):
+    @classmethod
+    def from_traces(cls, task: vf.Task, pro: vf.Trace, con: vf.Trace) -> "VerdictTask":
+        prompt = (
+            f"Question: {task.data.prompt_text}\n\n"
+            f"PRO argued:\n{pro.last_reply}\n\nCON argued:\n{con.last_reply}\n\n"
+            "Who won? Reply with exactly 'pro' or 'con'."
+        )
+        return cls(vf.TaskData(idx=task.data.idx, prompt=prompt))
+
+
+class DebateEnv(vf.Environment[DebateConfig]):
+    async def setup(self, agents: vf.Agents) -> None:
+        """Per-agent standing the env hardcodes: the judge grades the debate,
+        so its tokens are never training data."""
+        agents.judge.trainable = False
+
+    async def run(self, task: vf.Task, agents: vf.Agents) -> None:
+        """How the agents interact on one task: imperative Python over Agent
+        values. A loop is rounds, a TaskGroup is fan-out, a Task classmethod is
+        chaining. Returns nothing — every finished run joins the episode
+        automatically, stamped with its standing."""
+        pro, con = await asyncio.gather(agents.pro.run(task), agents.con.run(task))
+        await agents.judge.run(VerdictTask.from_traces(task, pro, con))
+
+    async def finalize(self, task: vf.Task, episode: vf.Episode) -> None:
+        """Sibling-dependent judgement over the finished episode (per-trace
+        judgement already ran on each trace's own task); `episode.traces` is
+        the flat episode, each trace's `agent_name` stamp naming its agent.
+        Attach via record_reward/record_metric, in program order."""
+        by_agent = {t.agent_name: t for t in episode.traces}
+        winner = (by_agent["judge"].last_reply or "").strip().lower()
+        by_agent["pro"].record_reward("won", float(winner == "pro"))
+        by_agent["con"].record_reward("won", float(winner == "con"))
+```
+
+For the single-agent case none of this is machinery the user sees: `SingleAgentEnv`
+declares one `agent` (`--env.agent.harness.id codex`, `--env.agent.max_turns 20`),
+`run()` is `await agents.agent.run(task)`, and the episode carries exactly one
+trace.
+
+The run's `[env]` block is the whole run — the env is the encompassing entity,
+composing three separately-chosen concerns:
+
+- **`env.taskset`** — *what to solve*: the seed rows every rollout starts from,
+  their data, their per-trace judgement (`--env.taskset.id`, or the positional
+  `eval <taskset-id>`).
+- **each agent's `harness`** — *how that LLM interfaces with the world*: the
+  program driving model calls, tools, a runtime — pinned per agent, never a
+  run-wide flag.
+- **the env itself** — *the control flow between agents*: who runs, in what order,
+  judged how across the finished set (`--env.id`).
+
+### Reusable envs: `--env.id`
+
+An interaction pattern that isn't specific to one dataset — n attempts, a judge, a
+modeled user — is its own plugin, paired with any taskset from the CLI:
+
+```bash
+uv run eval gsm8k-v1 --env.id best-of-n --env.n 8
+uv run eval my-task-v1 --env.id agentic-judge --env.judge.harness.runtime.type docker
+```
+
+The same pairing as TOML — `env.id` plus one `[env.<agent>]` block per agent — is
+checked in as `configs/agentic_judge.toml` (`uv run eval @ configs/agentic_judge.toml`).
+
+`--env.id` resolves like every plugin id — a bundled env (below), a local package
+exporting an `Environment` subclass via `__all__`, or a Hub `org/name[@version]` —
+and its `EnvConfig` surface typed on the CLI (`--env.<agent>.*`, `-h` renders
+them). Empty (the default) keeps the taskset's own story: the env its package
+ships (a *recipe* env like `code_golf_v1`, where the interaction is intrinsic to
+the data), else `SingleAgentEnv`. An explicit id wins over a bundled recipe env.
+
+Bundled envs (`verifiers/v1/envs/`):
+
+| id | agents | what it does |
+| --- | --- | --- |
+| `best-of-n` | `agent` | `--env.n` independent attempts per rollout; its metrics mark the argmax-reward sibling (`best`) and whether any reached `--env.threshold` (`pass_at_n`) — rejection sampling and pass@k. A single-agent env keeps the single-agent name, so `--env.agent.*` flags compose unchanged. |
+| `agentic-judge` | `solver`, `judge` | agent-as-judge: the solver plays the task; a code-executing judge agent verifies the finished attempt with real execution, always in its own sandbox, never on the host. The judge's task mirrors the solver task's world (same image, a fresh box in its original state) with the graded transcript uploaded (`/tmp/transcript.md`/`.json`). The verdict channel is a file: the judge writes `{"score": 0-10, "reasoning": ...}` to `/tmp/verdict.json` in its box, scraped onto its trace while the box is alive and validated STRICTLY onto the solver's trace as the `judge` reward — a missing, malformed, or off-scale verdict fails the rollout instead of clamping. The judge must land in a container: pin `--env.judge.harness.runtime.type docker\|prime`, or construction refuses. A judgement that needs no execution belongs on the plugged tier (`env.taskset.task.judges`), not on an agent. |

configs/agentic_judge.toml

@@ -0,0 +1,27 @@
+# Any taskset graded by the bundled agentic judge — the multi-agent TOML shape:
+# `env.id` pairs a reusable env with the taskset, and each agent is its own
+# [env.<agent>] block (a partial override deep-merges into the agent's declared
+# default; there is no run-level harness). The judge pins what makes it a
+# different actor: a container runtime — it verifies the solver's attempt with
+# real execution, never on the host — and, here, its own model off the run's.
+#
+#   uv run eval @ configs/agentic_judge.toml --model <model>
+#
+# The judge writes {"score": 0-10, ...} to /tmp/verdict.json in its box; the env
+# validates it strictly and records it on the solver's trace as the `judge` reward.
+num_tasks = 2
+
+[env]
+id = "agentic-judge"
+
+[env.taskset]
+id = "gsm8k-v1"
+
+[env.solver.harness]
+id = "null"
+
+[env.judge]
+model = "z-ai/glm-5.2"
+
+[env.judge.harness.runtime]
+type = "docker"

configs/alphabet_sort.toml

@@ -3,12 +3,14 @@
 #   uv run eval @ configs/alphabet_sort.toml --model <small-instruct-model>
 num_tasks = 5
 num_rollouts = 2
-max_turns = 4
 
-[taskset]
+[env.taskset]
 id = "alphabet-sort-v1"
 min_turns = 2
 max_turns = 2
 
-[harness]
+[env.agent]
+max_turns = 4
+
+[env.agent.harness]
 id = "null"

configs/code_golf.toml

@@ -1,9 +1,10 @@
-# code-golf-v1 — group rewards (@vf.group_reward) score N rollouts of a task together
-# (shortest / fastest of the group), so it needs >= 2 rollouts.
+# code-golf-v1 — the taskset ships a recipe env: one env-rollout runs
+# `--env.attempts` independent attempts (default 2) and scores them against
+# each other (shortest wins), so one rollout per task is a full comparison.
 #
 #   uv run eval code-golf-v1 @ configs/code_golf.toml
 num_tasks = 1
-num_rollouts = 2
+num_rollouts = 1
 
-[taskset]
+[env.taskset]
 id = "code-golf-v1"

configs/deepwiki.toml

@@ -3,5 +3,5 @@
 #   uv run eval @ configs/deepwiki.toml
 num_tasks = 1
 
-[taskset]
+[env.taskset]
 id = "deepwiki-v1"

configs/glossary.toml

@@ -3,5 +3,5 @@
 #   uv run eval glossary-v1 @ configs/glossary.toml
 num_tasks = 1
 
-[taskset]
+[env.taskset]
 id = "glossary-v1"

configs/gsm8k.toml

@@ -4,5 +4,5 @@
 num_tasks = 5
 num_rollouts = 3
 
-[taskset]
+[env.taskset]
 id = "gsm8k-v1"

configs/gsm8k_rlm.toml

@@ -2,15 +2,16 @@
 #
 #   uv run eval @ configs/gsm8k_rlm.toml
 #
-# The discriminator ids (which taskset, which harness) come from this file's [taskset] /
-# [harness] `id` — they select the schema, and the rest of the file fills the typed config,
-# each side validated against its specific type (GSM8KConfig / RLMHarnessConfig). prime-rl
-# consumes the same shape, reading the ids straight from its config (TOML-driven there too).
+# The discriminator ids (which taskset, which harness) come from this file's
+# [env.taskset] / [env.agent.harness] `id` — they select the schema, and the rest of the
+# file fills the typed config, each side validated against its specific type (GSM8KConfig /
+# RLMHarnessConfig). prime-rl consumes the same [env] shape, reading the ids straight from
+# its config (TOML-driven there too).
 num_tasks = 1
 
-[taskset]
+[env.taskset]
 id = "gsm8k-v1"
 
-[harness]
+[env.agent.harness]
 id = "rlm"
 version = "main"

configs/harbor.toml

@@ -4,10 +4,10 @@
 #   uv run eval @ configs/harbor.toml
 num_tasks = 10
 
-[taskset]
+[env.taskset]
 id = "harbor"
 dataset = "terminal-bench/terminal-bench-2"
 
-[harness]
+[env.agent.harness]
 id = "bash"
 runtime = { type = "docker" }