From 85dae8df41ea1d3f4fc6c84b3ab88d4e3be86762 Mon Sep 17 00:00:00 2001 From: Stella Test Date: Thu, 6 Aug 2026 04:09:57 -0700 Subject: [PATCH 1/3] fix(arenabench): stop transcript text rendering twice MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stella's event stream carries both text_delta fragments (field: text) and one consolidated text event per message (field: delta) — the names are crossed on the wire. TranscriptReader appended both into the same run, so every response rendered twice: once assembled from fragments, once whole, usually split around the step_usage line that closed the run in between. The consolidated event now REPLACES its fragment run under the same seq (clients keyed by seq replace in place), found via a pending_text pointer that outlives the run-closing step_usage. A stream with no fragments still renders one entry per consolidated event. --- arenabench/arenabench/telemetry.py | 31 ++++++++++++++++- arenabench/tests/test_telemetry.py | 54 ++++++++++++++++++++++++++++++ 2 files changed, 84 insertions(+), 1 deletion(-) diff --git a/arenabench/arenabench/telemetry.py b/arenabench/arenabench/telemetry.py index 92d30d93..bc17f93b 100644 --- a/arenabench/arenabench/telemetry.py +++ b/arenabench/arenabench/telemetry.py @@ -822,6 +822,11 @@ class TranscriptState: open_entries: dict[str, int] = field(default_factory=dict) #: ``call_id`` -> entry sequence, so a tool result can find its call. tool_index: dict[str, int] = field(default_factory=dict) + #: The last fragment-built text entry not yet superseded by its + #: consolidated ``text`` event. Outlives ``open_entries`` on purpose: + #: ``step_usage`` routinely closes the run *before* the consolidated + #: event arrives, and the consolidation must still find its run. + pending_text: int | None = None class TranscriptReader: @@ -925,7 +930,10 @@ def entry( } # ---- streaming text / reasoning: coalesce into one growing entry ---- - if kind in ("text", "text_delta", "reasoning"): + # The field names are crossed on the wire and that is the trap: + # `text_delta` events carry a *fragment* (in `text`), while the one + # `text` event per message carries the *complete* body (in `delta`). + if kind in ("text_delta", "reasoning"): bucket = "reasoning" if kind == "reasoning" else "text" fragment = str(event.get("delta") or event.get("text") or "") if not fragment: @@ -934,6 +942,8 @@ def entry( if seq is None: seq = self._next_seq(state) state.open_entries[bucket] = seq + if bucket == "text": + state.pending_text = seq key = (path_key, seq) self._bodies[key] = self._bodies.get(key, "") + fragment return [ @@ -946,6 +956,25 @@ def entry( ) ] + if kind == "text": + # The consolidated message. It REPLACES the fragment run rather + # than appending to it: appending rendered every response twice + # (once assembled from fragments, once whole), and replacing also + # self-heals any fragment this reader never saw. `pending_text` + # rather than `open_entries` finds the run, because a `step_usage` + # usually closed the run before this event arrived. The message is + # complete, so both trackers reset here. + full = str(event.get("delta") or event.get("text") or "") + if not full: + return [] + seq = state.open_entries.get("text") or state.pending_text + if seq is None: + seq = self._next_seq(state) + state.open_entries.clear() + state.pending_text = None + self._bodies[(path_key, seq)] = full + return [entry(seq, "text", "response", full)] + # Any non-delta event closes the open text/reasoning runs, so the next # fragment starts a fresh entry rather than reopening a finished one. state.open_entries.clear() diff --git a/arenabench/tests/test_telemetry.py b/arenabench/tests/test_telemetry.py index 637ee01e..ab2c76be 100644 --- a/arenabench/tests/test_telemetry.py +++ b/arenabench/tests/test_telemetry.py @@ -599,6 +599,60 @@ def test_two_trials_do_not_share_streaming_buffers(self, tmp_path: Path): assert reader.read(one)[-1]["body"] == "AAA" assert reader.read(two)[-1]["body"] == "BBB" + def test_the_consolidated_text_event_replaces_the_fragment_run( + self, tmp_path: Path + ): + """Stella emits both `text_delta` fragments and one consolidated + `text` event per message (the field names are crossed on the wire: + the fragment rides in `text`, the complete body in `delta`). + Appending both rendered every response twice in the transcript — + once assembled, once whole. The consolidated event must replace the + run under the same seq, not extend it. + """ + path = tmp_path / "e.jsonl" + write_events(path, [ + {"type": "text_delta", "text": "The work"}, + {"type": "text_delta", "text": "space root is `/app`."}, + {"type": "text", "delta": "The workspace root is `/app`."}, + ]) + entries = TranscriptReader().read(path) + text = [e for e in entries if e["kind"] == "text"] + assert len({e["seq"] for e in text}) == 1, "one message, one entry" + assert text[-1]["body"] == "The workspace root is `/app`." + + def test_a_consolidated_text_after_a_closed_run_does_not_duplicate( + self, tmp_path: Path + ): + """The shape from the field report: fragments stream, `step_usage` + closes the run, and only then the consolidated event arrives. The + old reader opened a *second* entry with the full body — the exact + doubled paragraph seen around every usage line.""" + path = tmp_path / "e.jsonl" + write_events(path, [ + {"type": "text_delta", "text": "Config "}, + {"type": "text_delta", "text": "validates."}, + {"type": "step_usage", "step": 6, "role": "worker"}, + {"type": "text", "delta": "Config validates."}, + ]) + entries = TranscriptReader().read(path) + text = [e for e in entries if e["kind"] == "text"] + # One message, one entry — a client keyed by seq replaces in place. + assert len({e["seq"] for e in text}) == 1 + assert text[-1]["body"] == "Config validates." + + def test_a_lone_consolidated_text_still_renders(self, tmp_path: Path): + """A stream with no fragments (another emitter, a replay) is one + complete message per `text` event — each its own entry.""" + path = tmp_path / "e.jsonl" + write_events(path, [ + {"type": "text", "delta": "first message"}, + {"type": "text", "delta": "second message"}, + ]) + entries = TranscriptReader().read(path) + text = [e for e in entries if e["kind"] == "text"] + assert [e["body"] for e in text] == ["first message", "second message"] + assert len({e["seq"] for e in text}) == 2 + class TestInfrastructureFailuresAreNotLosses: """A trial the agent never got to attempt is not a trial it lost.""" From 8d889588a64c35a66ba94f5272a746e2dde6d960 Mon Sep 17 00:00:00 2001 From: Stella Test Date: Thu, 6 Aug 2026 04:16:13 -0700 Subject: [PATCH 2/3] feat(arenabench): transcript as its own searchable page MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The transcript drawer becomes /transcript?match=&task=&seat= — a URL you can bookmark, reload, and send. New reading tools the drawer had no room for: kind filters (responses / thinking / tools / results / usage / stages) and case-insensitive full-text search with highlighting across every entry; an active search bypasses replay pacing, because a search answers over the whole transcript. Rendering follows the Command Deck's transcript grammar: stages are section rules (the label is the stage), reasoning is dim, italic and collapsed to a preview with its line count, tool names take the accent while bodies stay plain, and long results fold at twelve lines. The task table's transcript button is now a link; the drawer is deleted. server._static learns the exporter's naming (.html, index.html in a directory) so the new route resolves without touching the URL shape. --- arenabench/arenabench/server.py | 10 + arenabench/ui/app/transcript/page.tsx | 5 + arenabench/ui/components/arena/arena-view.tsx | 23 +- arenabench/ui/components/arena/task-table.tsx | 27 +- .../ui/components/arena/transcript-drawer.tsx | 343 ---------- .../ui/components/arena/transcript-page.tsx | 646 ++++++++++++++++++ 6 files changed, 672 insertions(+), 382 deletions(-) create mode 100644 arenabench/ui/app/transcript/page.tsx delete mode 100644 arenabench/ui/components/arena/transcript-drawer.tsx create mode 100644 arenabench/ui/components/arena/transcript-page.tsx diff --git a/arenabench/arenabench/server.py b/arenabench/arenabench/server.py index a5d52ebc..1ef44df3 100644 --- a/arenabench/arenabench/server.py +++ b/arenabench/arenabench/server.py @@ -531,6 +531,16 @@ def _static(self, rel: str) -> None: except ValueError: self._error(HTTPStatus.FORBIDDEN, "path escapes the web root") return + # The exported client writes each route as `.html` (and a + # directory route as `/index.html`), but a browser asks for + # the clean path. Resolve the way the exporter wrote, never past + # the web root the check above already pinned. + if target.is_dir(): + target = target / "index.html" + if not target.is_file() and not target.suffix: + sibling = target.with_suffix(".html") + if sibling.is_file(): + target = sibling if not target.is_file(): self._error(HTTPStatus.NOT_FOUND, f"no such asset: {rel}") return diff --git a/arenabench/ui/app/transcript/page.tsx b/arenabench/ui/app/transcript/page.tsx new file mode 100644 index 00000000..ba61071e --- /dev/null +++ b/arenabench/ui/app/transcript/page.tsx @@ -0,0 +1,5 @@ +import { TranscriptPage } from "@/components/arena/transcript-page"; + +export default function Page() { + return ; +} diff --git a/arenabench/ui/components/arena/arena-view.tsx b/arenabench/ui/components/arena/arena-view.tsx index abf7f777..2a54ca8a 100644 --- a/arenabench/ui/components/arena/arena-view.tsx +++ b/arenabench/ui/components/arena/arena-view.tsx @@ -12,7 +12,6 @@ import { Race } from "@/components/arena/race"; import { StatStrip } from "@/components/arena/stat-strip"; import { SeatNotices } from "@/components/arena/seat-notices"; import { TaskTable } from "@/components/arena/task-table"; -import { TranscriptDrawer } from "@/components/arena/transcript-drawer"; export function ArenaView({ matchId, @@ -26,7 +25,6 @@ export function ArenaView({ onMatchEnded: () => void; }) { const [snapshot, setSnapshot] = React.useState(seedSnapshot); - const [activeTask, setActiveTask] = React.useState(null); const [confirmStop, setConfirmStop] = React.useState(false); const [cancelError, setCancelError] = React.useState(null); @@ -65,11 +63,6 @@ export function ArenaView({ return
connecting to the match…
; } - const activeRow = activeTask ? snapshot.rows.find((r) => r.task === activeTask) : null; - // "Live" is a property of the match, not of one trial: it decides whether - // the drawer offers speed control or only pause/resume. - const live = snapshot.status === "running"; - return ( /* The page gets real margins and a max width. Before this the arena ran edge to edge at any window size, which is what made a dense table of @@ -130,11 +123,7 @@ export function ArenaView({ glanceable summary and gets one. They stack on a narrow window rather than squeezing the table into an unreadable column. */}
- +
- {activeTask !== null && ( - setActiveTask(null)} - /> - )} ); } diff --git a/arenabench/ui/components/arena/task-table.tsx b/arenabench/ui/components/arena/task-table.tsx index e01f7f1a..a3dbf30c 100644 --- a/arenabench/ui/components/arena/task-table.tsx +++ b/arenabench/ui/components/arena/task-table.tsx @@ -4,7 +4,6 @@ import * as React from "react"; import type { Cell, ContestantSnap, Snapshot } from "@/lib/types"; import { fmtClock, fmtMoney, fmtTokens } from "@/lib/format"; import { cn, seatStyle } from "@/lib/utils"; -import { Button } from "@/components/ui/button"; import { Tip } from "@/components/ui/tooltip"; /** @@ -111,15 +110,7 @@ function Verdict({ cell }: { cell: Cell | null | undefined }) { ); } -export function TaskTable({ - snapshot, - onOpenTranscript, - activeTask, -}: { - snapshot: Snapshot; - onOpenTranscript: (task: string) => void; - activeTask: string | null; -}) { +export function TaskTable({ snapshot }: { snapshot: Snapshot }) { const seats = snapshot.contestants; return ( @@ -157,7 +148,6 @@ export function TaskTable({ {snapshot.rows.map((row) => { const won = winnersFor(row.cells, seats); - const isActive = row.task === activeTask; return seats.map((seat, seatIndex) => { const cell = row.cells[seat.id]; return ( @@ -166,7 +156,6 @@ export function TaskTable({ style={seatStyle(seat.color)} className={cn( "border-b border-line-soft/60 align-middle", - isActive && "bg-(--seat)/6", seatIndex === seats.length - 1 && "border-b-line", )} > @@ -224,14 +213,18 @@ export function TaskTable({ })} {seatIndex === 0 && ( - + )} diff --git a/arenabench/ui/components/arena/transcript-drawer.tsx b/arenabench/ui/components/arena/transcript-drawer.tsx deleted file mode 100644 index 0b499624..00000000 --- a/arenabench/ui/components/arena/transcript-drawer.tsx +++ /dev/null @@ -1,343 +0,0 @@ -"use client"; - -import * as React from "react"; -import type { Cell, ContestantSnap, TranscriptEntry } from "@/lib/types"; -import { fmtClock, fmtMoney, fmtTokens } from "@/lib/format"; -import { cn, seatStyle } from "@/lib/utils"; -import { Button } from "@/components/ui/button"; - -/** - * One task's transcript, in a drawer that flies out over the table. - * - * **One transcript at a time, by design.** The old side-by-side lanes split a - * narrow column two ways and truncated both; a transcript is prose and wants - * width. So the drawer shows a single seat and lets you switch — which is - * also how you actually read these, one agent at a time, comparing from - * memory of the table you just left. - * - * # Live and historical are the same reader, with one difference - * - * A finished trial's transcript arrives all at once (the server replays the - * backlog on connect), so "watching" it means **pacing the reveal ourselves** - * against each entry's own `t` — the elapsed seconds it was emitted at. That - * makes historical playback a client concern and needs nothing from the - * server, which is why speed control exists only here. - * - * - **Historical**: play/pause, replay once it has run out, and 1×/2×/3×/6×. - * Seeking is the scrollbar. - * - **Live**: pause and resume only. There is no speed for a stream that has - * not happened yet, and resuming jumps to the tail — a live reader who - * pauses wants to stop the scroll, and on resume wants *now*, not the - * middle of a backlog they already scrolled past. - */ - -const SPEEDS = [1, 2, 3, 6] as const; - -/** Wall-clock ms one playback step waits, given a gap in transcript seconds. */ -function pacedDelay(gapSeconds: number, speed: number): number { - // Clamped so a trial with a four-minute think does not stall playback, and - // a burst of same-second entries still animates rather than dumping. - const ms = (gapSeconds * 1000) / speed; - return Math.max(16, Math.min(1200, ms)); -} - -function useTranscript(matchId: string, contestantId: string, task: string) { - const [entries, setEntries] = React.useState([]); - const [waiting, setWaiting] = React.useState(true); - const [ended, setEnded] = React.useState(false); - - React.useEffect(() => { - setEntries([]); - setWaiting(true); - setEnded(false); - const bySeq = new Map(); - const url = - `/api/matches/${encodeURIComponent(matchId)}/transcript/` + - `${encodeURIComponent(contestantId)}/${encodeURIComponent(task)}`; - const source = new EventSource(url); - source.addEventListener("entries", (event) => { - const payload = JSON.parse((event as MessageEvent).data) as { entries: TranscriptEntry[] }; - for (const entry of payload.entries) bySeq.set(entry.seq, entry); - if (bySeq.size) { - setWaiting(false); - setEntries([...bySeq.values()]); - } - }); - source.addEventListener("end", () => { - setWaiting(false); - setEnded(true); - source.close(); - }); - return () => source.close(); - }, [matchId, contestantId, task]); - - return { entries, waiting, ended }; -} - -const KIND_TONE: Record = { - reasoning: "text-acc-violet opacity-85", - text: "text-foreground", - tool: "text-acc-cyan", - tool_result: "text-muted", - stage: "text-accent", - error: "text-bad", - verdict: "font-semibold text-ok", - complete: "text-ok", - usage: "text-dim", -}; - -function entryBody(entry: TranscriptEntry): string { - const meta = (entry.meta || {}) as Record; - const body = entry.body || ""; - if (entry.kind === "usage") { - return ( - `${meta.model || ""} · in ${fmtTokens(meta.tokens_in)} out ${fmtTokens(meta.tokens_out)}` + - ` · cache ${fmtTokens(meta.cache_read)}/${fmtTokens(meta.cache_write)}` + - ` · self-rep ${fmtMoney(meta.cost_usd)}` - ); - } - if (entry.kind === "tool") return `${entry.title ?? ""} ${body}`; - if (entry.kind === "tool_result") { - const lines = body.split("\n"); - return lines.slice(0, 12).join("\n") + (lines.length > 12 ? `\n… +${lines.length - 12} lines` : ""); - } - if (entry.kind === "stage" || entry.kind === "complete" || entry.kind === "verdict") { - return (entry.title ?? "") + (body ? ` — ${body}` : ""); - } - return body; -} - -export function TranscriptDrawer({ - matchId, - task, - seats, - cells, - live, - onClose, -}: { - matchId: string; - task: string; - seats: ContestantSnap[]; - cells: Record; - /** Whether the match is still running — decides pause-vs-speed controls. */ - live: boolean; - onClose: () => void; -}) { - // Contestant 1 by default, as specified; the picker switches seats without - // tearing the drawer down. - const [seatId, setSeatId] = React.useState(seats[0]?.id ?? ""); - React.useEffect(() => { - if (!seats.some((s) => s.id === seatId)) setSeatId(seats[0]?.id ?? ""); - }, [seats, seatId]); - - const [showReasoning, setShowReasoning] = React.useState(true); - const [playing, setPlaying] = React.useState(true); - const [speed, setSpeed] = React.useState(1); - /** How many entries of the historical transcript have been revealed. */ - const [revealed, setRevealed] = React.useState(0); - - const { entries, waiting, ended } = useTranscript(matchId, seatId, task); - const feedRef = React.useRef(null); - - const filtered = React.useMemo( - () => (showReasoning ? entries : entries.filter((e) => e.kind !== "reasoning")), - [entries, showReasoning], - ); - - // Historical playback: a finished trial reveals on a timer paced by each - // entry's own elapsed stamp. A live trial reveals everything immediately — - // the stream itself is the pacing. - const historical = ended && !live; - - React.useEffect(() => { - // Restart the reveal whenever the source changes. - setRevealed(historical ? 0 : filtered.length); - setPlaying(true); - }, [seatId, task, historical, filtered.length]); - - React.useEffect(() => { - if (!historical || !playing) return; - if (revealed >= filtered.length) return; - const previous = filtered[revealed - 1]?.t ?? 0; - const current = filtered[revealed]?.t ?? previous; - const timer = window.setTimeout( - () => setRevealed((n) => Math.min(n + 1, filtered.length)), - pacedDelay(Math.max(0, current - previous), speed), - ); - return () => window.clearTimeout(timer); - }, [historical, playing, revealed, filtered, speed]); - - const shown = historical ? filtered.slice(0, revealed) : filtered; - - // Follow the tail while playing. On resume this snaps to the newest entry, - // which is the documented live behaviour and the sane historical one too. - React.useEffect(() => { - if (playing && feedRef.current) { - feedRef.current.scrollTop = feedRef.current.scrollHeight; - } - }, [shown.length, playing]); - - // Escape closes, as every drawer should. - React.useEffect(() => { - const onKey = (event: KeyboardEvent) => { - if (event.key === "Escape") onClose(); - }; - window.addEventListener("keydown", onKey); - return () => window.removeEventListener("keydown", onKey); - }, [onClose]); - - const seat = seats.find((s) => s.id === seatId); - const cell = cells[seatId]; - const done = historical && revealed >= filtered.length && filtered.length > 0; - - // Three states share one button, and `done` has to be read *before* - // `playing`: playback stops by running out of entries, not by clearing the - // flag, so a finished replay is still `playing` and would otherwise offer - // "pause" — and toggling the flag on a fully-revealed transcript is a no-op, - // which is what made the old "replay" do nothing. Replay rewinds instead. - const replay = React.useCallback(() => { - setRevealed(0); - setPlaying(true); - }, []); - - return ( -
- {/* Scrim: clicking outside closes, which is the gesture people try - first and the one a modal drawer owes them. */} - - - - {/* Whose transcript. Default is contestant 1; switching is one click - and keeps the drawer open. */} -
- {seats.map((option) => ( - - ))} -
- - {/* Playback. Historical gets speed; live gets pause/resume only. */} -
- - {historical ? ( -
- {SPEEDS.map((option) => ( - - ))} -
- ) : ( - - live · resuming jumps to the newest entry - - )} - {historical && ( - - {Math.min(revealed, filtered.length)} / {filtered.length} - - )} - -
- - {cell && ( -
- steps {cell.steps} - tools {cell.tools} - in {fmtTokens(cell.tokens_in)} - out {fmtTokens(cell.tokens_out)} - cost {fmtMoney(cell.priced_cost)} - {fmtClock(cell.clock_time)} -
- )} - -
- {waiting && shown.length === 0 ? ( -
waiting for the trial to start…
- ) : shown.length === 0 ? ( -
no transcript entries for this trial.
- ) : ( - shown.map((entry) => ( -
- - {fmtClock(entry.t)} - - - {entryBody(entry)} - -
- )) - )} -
- -
- ); -} diff --git a/arenabench/ui/components/arena/transcript-page.tsx b/arenabench/ui/components/arena/transcript-page.tsx new file mode 100644 index 00000000..92856972 --- /dev/null +++ b/arenabench/ui/components/arena/transcript-page.tsx @@ -0,0 +1,646 @@ +"use client"; + +import * as React from "react"; +import { api } from "@/lib/api"; +import type { Cell, Snapshot, TranscriptEntry } from "@/lib/types"; +import { fmtClock, fmtMoney, fmtTokens } from "@/lib/format"; +import { cn, seatStyle } from "@/lib/utils"; +import { Button } from "@/components/ui/button"; +import { Input } from "@/components/ui/input"; + +/** + * A trial's transcript as its own page: `/transcript?match=…&task=…&seat=…`. + * + * This replaced the fly-out drawer. A transcript is the artifact people + * actually study after a match — it deserves a URL you can bookmark, send, + * and reload, plus the reading tools a drawer never had room for: kind + * filters and full-text search across every entry. + * + * The rendering follows the Command Deck's transcript grammar, because a + * reader who lives in Stella should not have to relearn the page: + * + * - **Stages are section rules**, not rows — the label *is* the stage. + * - **Reasoning is quiet.** Dim, italic, collapsed to a short preview with + * its line count in the header; the least load-bearing text on screen + * never outshouts the response. + * - **The label is coloured, the value is read.** Tool names take the + * accent; bodies stay plain. A colour earns its place by being rare. + * + * Playback matches the drawer it replaced: a finished trial replays paced by + * each entry's own elapsed stamp, a live one streams. Search and filters are + * reading tools, so an active search shows every match immediately — pacing + * a search result would just hide hits behind a timer. + */ + +const SPEEDS = [1, 2, 3, 6] as const; + +/** Wall-clock ms one playback step waits, given a gap in transcript seconds. */ +function pacedDelay(gapSeconds: number, speed: number): number { + const ms = (gapSeconds * 1000) / speed; + return Math.max(16, Math.min(1200, ms)); +} + +/** Filterable groups, in reading order. Unknown kinds always render. */ +const GROUPS: Array<{ key: string; label: string; kinds: string[] }> = [ + { key: "text", label: "responses", kinds: ["text"] }, + { key: "reasoning", label: "thinking", kinds: ["reasoning"] }, + { key: "tool", label: "tools", kinds: ["tool"] }, + { key: "tool_result", label: "results", kinds: ["tool_result"] }, + { key: "usage", label: "usage", kinds: ["usage"] }, + { key: "flow", label: "stages", kinds: ["stage", "proof", "verdict", "complete"] }, +]; + +function groupOf(kind: string): string | null { + for (const group of GROUPS) if (group.kinds.includes(kind)) return group.key; + return null; +} + +function useTranscript(matchId: string, contestantId: string, task: string) { + const [entries, setEntries] = React.useState([]); + const [waiting, setWaiting] = React.useState(true); + const [ended, setEnded] = React.useState(false); + + React.useEffect(() => { + if (!matchId || !contestantId || !task) return; + setEntries([]); + setWaiting(true); + setEnded(false); + const bySeq = new Map(); + const url = + `/api/matches/${encodeURIComponent(matchId)}/transcript/` + + `${encodeURIComponent(contestantId)}/${encodeURIComponent(task)}`; + const source = new EventSource(url); + source.addEventListener("entries", (event) => { + const payload = JSON.parse((event as MessageEvent).data) as { + entries: TranscriptEntry[]; + }; + for (const entry of payload.entries) bySeq.set(entry.seq, entry); + if (bySeq.size) { + setWaiting(false); + setEntries([...bySeq.values()].sort((a, b) => a.seq - b.seq)); + } + }); + source.addEventListener("end", () => { + setWaiting(false); + setEnded(true); + source.close(); + }); + return () => source.close(); + }, [matchId, contestantId, task]); + + return { entries, waiting, ended }; +} + +/** The match snapshot, refreshed while the match is live so ✓/✗ stay true. */ +function useMatchSnapshot(matchId: string) { + const [snapshot, setSnapshot] = React.useState(null); + const [error, setError] = React.useState(null); + + React.useEffect(() => { + if (!matchId) return; + let cancelled = false; + let timer: number | undefined; + const pull = async () => { + try { + const snap = await api(`/api/matches/${encodeURIComponent(matchId)}`); + if (cancelled) return; + setSnapshot(snap); + if (snap.status === "running") { + timer = window.setTimeout(pull, 10_000); + } + } catch (err) { + if (!cancelled) setError(String(err)); + } + }; + pull(); + return () => { + cancelled = true; + if (timer !== undefined) window.clearTimeout(timer); + }; + }, [matchId]); + + return { snapshot, error }; +} + +/** Case-insensitive `` highlighting, keeping the original casing. */ +function Highlight({ text, query }: { text: string; query: string }) { + if (!query) return <>{text}; + const needle = query.toLowerCase(); + const parts: React.ReactNode[] = []; + let rest = text; + let key = 0; + for (;;) { + const at = rest.toLowerCase().indexOf(needle); + if (at === -1) break; + if (at > 0) parts.push(rest.slice(0, at)); + parts.push( + + {rest.slice(at, at + query.length)} + , + ); + rest = rest.slice(at + query.length); + } + parts.push(rest); + return <>{parts}; +} + +function usageLine(entry: TranscriptEntry): string { + const meta = (entry.meta || {}) as Record; + return ( + `${entry.title ?? "usage"} — ${meta.model || ""}` + + ` · in ${fmtTokens(meta.tokens_in)} out ${fmtTokens(meta.tokens_out)}` + + ` · cache ${fmtTokens(meta.cache_read)}/${fmtTokens(meta.cache_write)}` + + ` · self-rep ${fmtMoney(meta.cost_usd)}` + ); +} + +const RESULT_PREVIEW_LINES = 12; +const THINKING_PREVIEW_LINES = 5; + +/** One transcript entry, rendered in the deck's grammar. */ +function Entry({ + entry, + query, + thinkingOpen, + toggleThinking, + resultOpen, + toggleResult, +}: { + entry: TranscriptEntry; + query: string; + thinkingOpen: boolean; + toggleThinking: () => void; + resultOpen: boolean; + toggleResult: () => void; +}) { + const body = entry.body || ""; + + if (entry.kind === "stage") { + // A section rule, not a row: the label is the stage. + return ( +
+ {entry.title || body} + +
+ ); + } + + if (entry.kind === "reasoning") { + const lines = body.split("\n"); + const folded = !thinkingOpen && lines.length > THINKING_PREVIEW_LINES; + const shown = thinkingOpen ? lines : lines.slice(0, THINKING_PREVIEW_LINES); + return ( +
+ +
+ +
+ {folded && ( + + )} +
+ ); + } + + if (entry.kind === "tool") { + return ( +
+ + ⚙ + + {body && ( +
+ +
+ )} +
+ ); + } + + if (entry.kind === "tool_result") { + const isError = Boolean((entry.meta as Record | undefined)?.error); + const lines = body.split("\n"); + const folded = !resultOpen && lines.length > RESULT_PREVIEW_LINES; + const shown = resultOpen ? lines : lines.slice(0, RESULT_PREVIEW_LINES); + return ( +
+ + ↳ {isError ? "error" : "result"} + +
+ +
+ {folded && ( + + )} + {resultOpen && lines.length > RESULT_PREVIEW_LINES && ( + + )} +
+ ); + } + + if (entry.kind === "usage") { + return
{usageLine(entry)}
; + } + + if (entry.kind === "verdict" || entry.kind === "complete") { + return ( +
+ {(entry.title ?? entry.kind) + (body ? ` — ${body}` : "")} +
+ ); + } + + if (entry.kind === "error") { + return ( +
+ +
+ ); + } + + // The agent's response — plain foreground, full width. + return ( +
+ +
+ ); +} + +function readParams(): { match: string; task: string; seat: string } { + const params = new URLSearchParams(window.location.search); + return { + match: params.get("match") ?? "", + task: params.get("task") ?? "", + seat: params.get("seat") ?? "", + }; +} + +function writeParams(update: Partial<{ task: string; seat: string }>): void { + const url = new URL(window.location.href); + for (const [key, value] of Object.entries(update)) { + if (value) url.searchParams.set(key, value); + } + window.history.replaceState(null, "", url); +} + +export function TranscriptPage() { + // Read once on mount: this is a static export, so the URL is the only + // input. Same pattern as the app shell's match restore. + const [params, setParams] = React.useState<{ + match: string; + task: string; + seat: string; + } | null>(null); + React.useEffect(() => setParams(readParams()), []); + + if (params === null) return null; + if (!params.match) { + return ( +
+

no match in the URL.

+ + ← back to the arena + +
+ ); + } + return ; +} + +function TranscriptView({ + matchId, + initial, +}: { + matchId: string; + initial: { task: string; seat: string }; +}) { + const { snapshot, error } = useMatchSnapshot(matchId); + + const tasks = React.useMemo( + () => (snapshot ? snapshot.rows.map((row) => row.task) : []), + [snapshot], + ); + const seats = snapshot?.contestants ?? []; + + const [task, setTask] = React.useState(initial.task); + const [seatId, setSeatId] = React.useState(initial.seat); + + // Fill unset selections once the snapshot names the options. + React.useEffect(() => { + if (!snapshot) return; + if (!task && tasks.length) setTask(tasks[0]); + if (!seatId && seats.length) setSeatId(seats[0].id); + }, [snapshot, task, seatId, tasks, seats]); + + React.useEffect(() => { + if (task || seatId) writeParams({ task, seat: seatId }); + if (task) document.title = `transcript · ${task}`; + }, [task, seatId]); + + const live = snapshot?.status === "running"; + const { entries, waiting, ended } = useTranscript(matchId, seatId, task); + + // -- reading tools ------------------------------------------------------ + const [enabled, setEnabled] = React.useState>(() => + Object.fromEntries(GROUPS.map((group) => [group.key, true])), + ); + const [query, setQuery] = React.useState(""); + const searching = query.trim().length > 0; + + // Per-entry fold overrides ride on top of the global thinking default. + const [thinkingDefault, setThinkingDefault] = React.useState(false); + const [thinkingOverrides, setThinkingOverrides] = React.useState< + Record + >({}); + const [openResults, setOpenResults] = React.useState>({}); + React.useEffect(() => { + setThinkingOverrides({}); + setOpenResults({}); + }, [seatId, task]); + + const visible = React.useMemo(() => { + const needle = query.trim().toLowerCase(); + return entries.filter((entry) => { + const group = groupOf(entry.kind); + if (group !== null && !enabled[group]) return false; + if (!needle) return true; + return ( + (entry.body ?? "").toLowerCase().includes(needle) || + (entry.title ?? "").toLowerCase().includes(needle) || + (entry.kind === "usage" && usageLine(entry).toLowerCase().includes(needle)) + ); + }); + }, [entries, enabled, query]); + + // -- playback (paced replay for a finished trial) ----------------------- + const [playing, setPlaying] = React.useState(true); + const [speed, setSpeed] = React.useState(1); + const [revealed, setRevealed] = React.useState(0); + const historical = ended && !live; + // Search is a reading tool: it answers over the whole transcript, never + // behind the replay timer. + const paced = historical && !searching; + + React.useEffect(() => { + setRevealed(paced ? 0 : visible.length); + setPlaying(true); + }, [seatId, task, paced, visible.length]); + + React.useEffect(() => { + if (!paced || !playing) return; + if (revealed >= visible.length) return; + const previous = visible[revealed - 1]?.t ?? 0; + const current = visible[revealed]?.t ?? previous; + const timer = window.setTimeout( + () => setRevealed((n) => Math.min(n + 1, visible.length)), + pacedDelay(Math.max(0, current - previous), speed), + ); + return () => window.clearTimeout(timer); + }, [paced, playing, revealed, visible, speed]); + + const shown = paced ? visible.slice(0, revealed) : visible; + const done = paced && revealed >= visible.length && visible.length > 0; + + const feedRef = React.useRef(null); + React.useEffect(() => { + if (playing && !searching && feedRef.current) { + feedRef.current.scrollTop = feedRef.current.scrollHeight; + } + }, [shown.length, playing, searching]); + + const seat = seats.find((option) => option.id === seatId); + const row = snapshot?.rows.find((r) => r.task === task); + const cell: Cell | null | undefined = row?.cells?.[seatId]; + + const replay = React.useCallback(() => { + setRevealed(0); + setPlaying(true); + }, []); + + return ( +
+
+ + ← arena + + + {snapshot?.match.name ?? matchId} + + + {snapshot?.dataset.title ?? ""} · {live ? "live" : (snapshot?.status ?? "…")} + + {error && {error}} +
+ + {/* Which trial: task × seat. Both write back to the URL, so the page + you are looking at is always the page you can share. */} +
+ + {seats.map((option) => ( + + ))} + {cell && ( + + steps {cell.steps} + tools {cell.tools} + in {fmtTokens(cell.tokens_in)} + out {fmtTokens(cell.tokens_out)} + cost {fmtMoney(cell.priced_cost)} + {fmtClock(cell.clock_time)} + + )} +
+ + {/* Reading tools: search, kind filters, playback. */} +
+ setQuery(event.target.value)} + onKeyDown={(event) => { + if (event.key === "Escape") setQuery(""); + }} + placeholder="search the transcript…" + aria-label="search the transcript" + className="h-8 w-[240px] font-mono text-[12px]" + /> + {searching && ( + + {visible.length} of {entries.length} entries + + )} +
+ {GROUPS.map((group) => ( + + ))} +
+
+ + {paced || done ? ( + <> + +
+ {SPEEDS.map((option) => ( + + ))} +
+ + {Math.min(revealed, visible.length)} / {visible.length} + + + ) : live ? ( + + ) : null} +
+
+ +
+ {waiting && shown.length === 0 ? ( +
waiting for the trial to start…
+ ) : shown.length === 0 ? ( +
+ {searching ? "nothing matches this search." : "no transcript entries for this trial."} +
+ ) : ( + shown.map((entry) => ( +
+ + {fmtClock(entry.t)} + +
+ + setThinkingOverrides((state) => ({ + ...state, + [entry.seq]: !(state[entry.seq] ?? thinkingDefault), + })) + } + resultOpen={openResults[entry.seq] ?? false} + toggleResult={() => + setOpenResults((state) => ({ + ...state, + [entry.seq]: !state[entry.seq], + })) + } + /> +
+
+ )) + )} +
+
+ ); +} From 9daa4a8033865874383ab43c70b44532a9755ecc Mon Sep 17 00:00:00 2001 From: Stella Test Date: Thu, 6 Aug 2026 11:35:55 -0700 Subject: [PATCH 3/3] fix(arenabench): resolve a clean route past the exporter's RSC directory Next writes transcript.html AND a transcript/ directory of RSC payload text files. The directory branch looked only for index.html inside it and 404ed the clean /transcript URL; it now falls back to the .html sibling when the directory holds no page. --- arenabench/arenabench/server.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/arenabench/arenabench/server.py b/arenabench/arenabench/server.py index 1ef44df3..ddeb3dfd 100644 --- a/arenabench/arenabench/server.py +++ b/arenabench/arenabench/server.py @@ -531,13 +531,16 @@ def _static(self, rel: str) -> None: except ValueError: self._error(HTTPStatus.FORBIDDEN, "path escapes the web root") return - # The exported client writes each route as `.html` (and a - # directory route as `/index.html`), but a browser asks for - # the clean path. Resolve the way the exporter wrote, never past - # the web root the check above already pinned. + # The exported client writes a route as `.html` — and ALSO + # leaves a `/` directory of RSC payload files beside it, so + # a clean path can resolve to a directory that holds no page. Try + # the directory's index first, then the `.html` sibling, never + # past the web root the check above already pinned. if target.is_dir(): - target = target / "index.html" - if not target.is_file() and not target.suffix: + index = target / "index.html" + sibling = target.with_suffix(".html") + target = index if index.is_file() else sibling + elif not target.is_file() and not target.suffix: sibling = target.with_suffix(".html") if sibling.is_file(): target = sibling