Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions arenabench/arenabench/server.py
Original file line number Diff line number Diff line change
Expand Up @@ -531,6 +531,16 @@ def _static(self, rel: str) -> None:
except ValueError:
self._error(HTTPStatus.FORBIDDEN, "path escapes the web root")
return
# The exported client writes each route as `<route>.html` (and a
# directory route as `<route>/index.html`), but a browser asks for
# the clean path. Resolve the way the exporter wrote, never past
# the web root the check above already pinned.
if target.is_dir():
target = target / "index.html"
if not target.is_file() and not target.suffix:
sibling = target.with_suffix(".html")
if sibling.is_file():
target = sibling
if not target.is_file():
self._error(HTTPStatus.NOT_FOUND, f"no such asset: {rel}")
return
Expand Down
31 changes: 30 additions & 1 deletion arenabench/arenabench/telemetry.py
Original file line number Diff line number Diff line change
Expand Up @@ -822,6 +822,11 @@ class TranscriptState:
open_entries: dict[str, int] = field(default_factory=dict)
#: ``call_id`` -> entry sequence, so a tool result can find its call.
tool_index: dict[str, int] = field(default_factory=dict)
#: The last fragment-built text entry not yet superseded by its
#: consolidated ``text`` event. Outlives ``open_entries`` on purpose:
#: ``step_usage`` routinely closes the run *before* the consolidated
#: event arrives, and the consolidation must still find its run.
pending_text: int | None = None


class TranscriptReader:
Expand Down Expand Up @@ -925,7 +930,10 @@ def entry(
}

# ---- streaming text / reasoning: coalesce into one growing entry ----
if kind in ("text", "text_delta", "reasoning"):
# The field names are crossed on the wire and that is the trap:
# `text_delta` events carry a *fragment* (in `text`), while the one
# `text` event per message carries the *complete* body (in `delta`).
if kind in ("text_delta", "reasoning"):
bucket = "reasoning" if kind == "reasoning" else "text"
fragment = str(event.get("delta") or event.get("text") or "")
if not fragment:
Expand All @@ -934,6 +942,8 @@ def entry(
if seq is None:
seq = self._next_seq(state)
state.open_entries[bucket] = seq
if bucket == "text":
state.pending_text = seq
key = (path_key, seq)
self._bodies[key] = self._bodies.get(key, "") + fragment
return [
Expand All @@ -946,6 +956,25 @@ def entry(
)
]

if kind == "text":
# The consolidated message. It REPLACES the fragment run rather
# than appending to it: appending rendered every response twice
# (once assembled from fragments, once whole), and replacing also
# self-heals any fragment this reader never saw. `pending_text`
# rather than `open_entries` finds the run, because a `step_usage`
# usually closed the run before this event arrived. The message is
# complete, so both trackers reset here.
full = str(event.get("delta") or event.get("text") or "")
if not full:
return []
seq = state.open_entries.get("text") or state.pending_text
if seq is None:
seq = self._next_seq(state)
state.open_entries.clear()
state.pending_text = None
self._bodies[(path_key, seq)] = full
return [entry(seq, "text", "response", full)]

# Any non-delta event closes the open text/reasoning runs, so the next
# fragment starts a fresh entry rather than reopening a finished one.
state.open_entries.clear()
Expand Down
54 changes: 54 additions & 0 deletions arenabench/tests/test_telemetry.py
Original file line number Diff line number Diff line change
Expand Up @@ -599,6 +599,60 @@ def test_two_trials_do_not_share_streaming_buffers(self, tmp_path: Path):
assert reader.read(one)[-1]["body"] == "AAA"
assert reader.read(two)[-1]["body"] == "BBB"

def test_the_consolidated_text_event_replaces_the_fragment_run(
self, tmp_path: Path
):
"""Stella emits both `text_delta` fragments and one consolidated
`text` event per message (the field names are crossed on the wire:
the fragment rides in `text`, the complete body in `delta`).
Appending both rendered every response twice in the transcript —
once assembled, once whole. The consolidated event must replace the
run under the same seq, not extend it.
"""
path = tmp_path / "e.jsonl"
write_events(path, [
{"type": "text_delta", "text": "The work"},
{"type": "text_delta", "text": "space root is `/app`."},
{"type": "text", "delta": "The workspace root is `/app`."},
])
entries = TranscriptReader().read(path)
text = [e for e in entries if e["kind"] == "text"]
assert len({e["seq"] for e in text}) == 1, "one message, one entry"
assert text[-1]["body"] == "The workspace root is `/app`."

def test_a_consolidated_text_after_a_closed_run_does_not_duplicate(
self, tmp_path: Path
):
"""The shape from the field report: fragments stream, `step_usage`
closes the run, and only then the consolidated event arrives. The
old reader opened a *second* entry with the full body — the exact
doubled paragraph seen around every usage line."""
path = tmp_path / "e.jsonl"
write_events(path, [
{"type": "text_delta", "text": "Config "},
{"type": "text_delta", "text": "validates."},
{"type": "step_usage", "step": 6, "role": "worker"},
{"type": "text", "delta": "Config validates."},
])
entries = TranscriptReader().read(path)
text = [e for e in entries if e["kind"] == "text"]
# One message, one entry — a client keyed by seq replaces in place.
assert len({e["seq"] for e in text}) == 1
assert text[-1]["body"] == "Config validates."

def test_a_lone_consolidated_text_still_renders(self, tmp_path: Path):
"""A stream with no fragments (another emitter, a replay) is one
complete message per `text` event — each its own entry."""
path = tmp_path / "e.jsonl"
write_events(path, [
{"type": "text", "delta": "first message"},
{"type": "text", "delta": "second message"},
])
entries = TranscriptReader().read(path)
text = [e for e in entries if e["kind"] == "text"]
assert [e["body"] for e in text] == ["first message", "second message"]
assert len({e["seq"] for e in text}) == 2


class TestInfrastructureFailuresAreNotLosses:
"""A trial the agent never got to attempt is not a trial it lost."""
Expand Down
5 changes: 5 additions & 0 deletions arenabench/ui/app/transcript/page.tsx
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
import { TranscriptPage } from "@/components/arena/transcript-page";

export default function Page() {
return <TranscriptPage />;
}
23 changes: 1 addition & 22 deletions arenabench/ui/components/arena/arena-view.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@ import { Race } from "@/components/arena/race";
import { StatStrip } from "@/components/arena/stat-strip";
import { SeatNotices } from "@/components/arena/seat-notices";
import { TaskTable } from "@/components/arena/task-table";
import { TranscriptDrawer } from "@/components/arena/transcript-drawer";

export function ArenaView({
matchId,
Expand All @@ -26,7 +25,6 @@ export function ArenaView({
onMatchEnded: () => void;
}) {
const [snapshot, setSnapshot] = React.useState<Snapshot | null>(seedSnapshot);
const [activeTask, setActiveTask] = React.useState<string | null>(null);
const [confirmStop, setConfirmStop] = React.useState(false);
const [cancelError, setCancelError] = React.useState<string | null>(null);

Expand Down Expand Up @@ -65,11 +63,6 @@ export function ArenaView({
return <div className="p-8 text-[13px] text-dim">connecting to the match…</div>;
}

const activeRow = activeTask ? snapshot.rows.find((r) => r.task === activeTask) : null;
// "Live" is a property of the match, not of one trial: it decides whether
// the drawer offers speed control or only pause/resume.
const live = snapshot.status === "running";

return (
/* The page gets real margins and a max width. Before this the arena ran
edge to edge at any window size, which is what made a dense table of
Expand Down Expand Up @@ -130,11 +123,7 @@ export function ArenaView({
glanceable summary and gets one. They stack on a narrow window
rather than squeezing the table into an unreadable column. */}
<section className="grid items-start gap-4 lg:[grid-template-columns:3fr_1fr]">
<TaskTable
snapshot={snapshot}
activeTask={activeTask}
onOpenTranscript={setActiveTask}
/>
<TaskTable snapshot={snapshot} />
<aside className="min-w-0">
<div className="mb-1.5 px-1 text-[10px] lowercase tracking-[0.1em] text-dim">
head to head
Expand All @@ -143,16 +132,6 @@ export function ArenaView({
</aside>
</section>

{activeTask !== null && (
<TranscriptDrawer
matchId={matchId}
task={activeTask}
seats={snapshot.contestants}
cells={activeRow?.cells ?? {}}
live={live}
onClose={() => setActiveTask(null)}
/>
)}
</div>
);
}
27 changes: 10 additions & 17 deletions arenabench/ui/components/arena/task-table.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@ import * as React from "react";
import type { Cell, ContestantSnap, Snapshot } from "@/lib/types";
import { fmtClock, fmtMoney, fmtTokens } from "@/lib/format";
import { cn, seatStyle } from "@/lib/utils";
import { Button } from "@/components/ui/button";
import { Tip } from "@/components/ui/tooltip";

/**
Expand Down Expand Up @@ -111,15 +110,7 @@ function Verdict({ cell }: { cell: Cell | null | undefined }) {
);
}

export function TaskTable({
snapshot,
onOpenTranscript,
activeTask,
}: {
snapshot: Snapshot;
onOpenTranscript: (task: string) => void;
activeTask: string | null;
}) {
export function TaskTable({ snapshot }: { snapshot: Snapshot }) {
const seats = snapshot.contestants;

return (
Expand Down Expand Up @@ -157,7 +148,6 @@ export function TaskTable({
<tbody>
{snapshot.rows.map((row) => {
const won = winnersFor(row.cells, seats);
const isActive = row.task === activeTask;
return seats.map((seat, seatIndex) => {
const cell = row.cells[seat.id];
return (
Expand All @@ -166,7 +156,6 @@ export function TaskTable({
style={seatStyle(seat.color)}
className={cn(
"border-b border-line-soft/60 align-middle",
isActive && "bg-(--seat)/6",
seatIndex === seats.length - 1 && "border-b-line",
)}
>
Expand Down Expand Up @@ -224,14 +213,18 @@ export function TaskTable({
})}
{seatIndex === 0 && (
<td rowSpan={seats.length} className="px-3 py-2 text-right align-middle">
<Button
variant="ghost"
size="sm"
onClick={() => onOpenTranscript(row.task)}
{/* A link, not a drawer: the transcript is its own
page, so it can be bookmarked and shared. */}
<a
href={
`/transcript?match=${encodeURIComponent(snapshot.match.id)}` +
`&task=${encodeURIComponent(row.task)}`
}
title={`Read the transcripts for ${row.task}`}
className="inline-flex cursor-pointer items-center rounded-[7px] border border-line bg-transparent px-2.5 py-[5px] font-mono text-xs lowercase text-foreground transition-colors hover:border-dim"
>
transcript
</Button>
</a>
</td>
)}
</tr>
Expand Down
Loading
Loading