From e251b744856b117b05f60fb6b51852cb2ce8a14d Mon Sep 17 00:00:00 2001 From: Liav Edry Date: Sun, 7 Jun 2026 18:17:53 +0300 Subject: [PATCH 1/4] webterm: drop the activity-based client reaper that disconnected live panes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit a3a0ef2 added reap_stale_clients to free --max-clients slots from abandoned tabs, keyed on tmux client_activity past CLIENT_IDLE_MAX (1h). But client_activity only advances on terminal I/O, NOT on the WebSocket keepalive pings (app.py ping_interval=25) that keep a quiet pane's socket warm. A wall tile watching an agent that produces no output for an hour — the normal case — looked idle and got force-detached, dropping a live pane to ttyd's reconnect screen (observed on the ccbot wall, disconnecting ~1h in). tmux can't tell an abandoned tab from a live-but-quiet one, so any activity-based reaper kills real sessions. Remove the reaper, its call, and CLIENT_IDLE_MAX; keep --max-clients=10. Dead sockets are already reclaimed by the keepalive path (a failed ping raises in the proxy pumps, ttyd drops the client, tmux frees the slot); the cap absorbs the rest. Documented in a WHY-NO-REAPER note above the poll loop. Co-Authored-By: Claude Opus 4.8 (1M context) --- scripts/agent-terminals.sh | 41 ++++++++++++++------------------------ 1 file changed, 15 insertions(+), 26 deletions(-) diff --git a/scripts/agent-terminals.sh b/scripts/agent-terminals.sh index 34031c2..ef36886 100755 --- a/scripts/agent-terminals.sh +++ b/scripts/agent-terminals.sh @@ -34,14 +34,11 @@ TMUX_SESSION="${CHELA_TMUX_SESSION:-chela}" BASE_PORT="${CHELA_TERM_BASE:-5101}" POLL="${CHELA_TERM_POLL:-12}" # Each ttyd allows this many concurrent browser connections (= tmux clients on -# its grouped session). Generous headroom: a dropped/backgrounded tab leaves a -# client attached until reaped (see CLIENT_IDLE_MAX), and a tight cap would lock -# the tile out of new connections once a few stale ones pile up. +# its grouped session). Generous headroom: a dropped/backgrounded tab can leave a +# client attached until its socket is torn down, and a tight cap would lock the +# tile out of new connections once a few stale ones pile up. We deliberately do +# NOT time-reap quiet clients — see the WHY-NO-REAPER note above the poll loop. MAX_CLIENTS="${CHELA_TERM_MAX_CLIENTS:-10}" -# Detach a wall ttyd client after this many seconds idle, so an abandoned tab -# can't hold a --max-clients slot forever. Scoped to webterm_* clients only — the -# user's own interactive tmux session is never reaped. -CLIENT_IDLE_MAX="${CHELA_TERM_CLIENT_IDLE_MAX:-3600}" MAP_FILE="${CHELA_DIR:-${HOME}/.chela}/agent_terminals.json" # Ensure the state dir exists — a fresh install has no ~/.chela yet, and the # atomic map write (write_map: printf > "$MAP_FILE.tmp" && mv) silently fails @@ -140,24 +137,6 @@ spawn() { PORT_OF["$wid"]=$port } -# Detach wall ttyd clients idle longer than CLIENT_IDLE_MAX. ttyd keeps one tmux -# client per WebSocket; a dropped or backgrounded browser tab can leave its client -# attached indefinitely, consuming a --max-clients slot until the grouped session -# refuses new connections. Filtered to WEBTERM_PREFIX sessions so the user's own -# interactive `${TMUX_SESSION}` client is never touched. -reap_stale_clients() { - local now act tty sess - now="$(date +%s)" - while IFS='|' read -r act tty sess; do - [[ -z "$tty" ]] && continue - [[ "$sess" == "${WEBTERM_PREFIX}"* ]] || continue - if (( now - act > CLIENT_IDLE_MAX )); then - tmux detach-client -t "$tty" 2>/dev/null \ - && echo "agent-terminals: reaped idle client ${tty} (${sess}, idle $((now-act))s)" - fi - done < <(tmux list-clients -F '#{client_activity}|#{client_name}|#{client_session}' 2>/dev/null) -} - despawn() { local wid="$1" [[ -n "${PID_OF[$wid]:-}" ]] && kill "${PID_OF[$wid]}" 2>/dev/null @@ -224,6 +203,17 @@ echo "agent-terminals: dynamic supervisor (poll ${POLL}s, base port ${BASE_PORT} write_map # write an (empty) map even before any agent is seen +# WHY-NO-REAPER: we used to detach wall ttyd clients idle past a timeout to free +# --max-clients slots held by abandoned tabs. That was wrong: the only idle signal +# tmux exposes is `client_activity`, which advances on terminal I/O — NOT on the +# WebSocket keepalive pings (app.py ping_interval=25) that keep a quiet pane's +# socket warm. An agent terminal that simply produces no output (the normal case +# for a wall you're watching) thus looked "idle" and got force-detached after the +# timeout, dropping a perfectly live tile to ttyd's reconnect screen. tmux cannot +# tell an abandoned tab from a live-but-quiet one, so any activity-based reaper +# kills real sessions. Genuinely dead sockets are already torn down by the +# keepalive: a failed ping raises in the proxy pumps, ttyd drops the client, and +# tmux detaches it — which frees the slot. --max-clients headroom absorbs the rest. declare -A CUR NAME_OF while :; do # snapshot the live window set, keyed by stable window id @@ -268,6 +258,5 @@ while :; do fi (( changed )) && write_map - reap_stale_clients # free --max-clients slots held by abandoned tabs sleep "${POLL}" done From 5ffb43d24b1ab9b71b75b3b1a676fb9b5aec11ce Mon Sep 17 00:00:00 2001 From: Liav Edry Date: Mon, 8 Jun 2026 10:18:46 +0300 Subject: [PATCH 2/4] webterm: release ttyd clients when the wall tab is backgrounded MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-up to dropping the activity-based reaper. Grouped tmux sessions share one real window with a single size; under `window-size largest`, a wall left open in a backgrounded tab keeps its ttyd WebSocket alive forever (keepalive pings auto-pong while hidden) and pins every shared window to that ghost's dimensions — so the wall you're actively viewing gets its rows/cols cropped to the stale tab's size (and a horizontal scrollbar appears). Terminal activity can't distinguish a quiet-but-watched pane from an abandoned tab, which is exactly why the server-side reaper was wrong. Page Visibility can: it reports when THIS tab is hidden. On visibilitychange to hidden, after a 45s grace, blank the ttyd iframes (src=about:blank) so their sockets close and the backing tmux clients drop, freeing window-size; on return, restore each iframe's src in place so it reconnects sized to the now-visible tile. termTick and _absorbFreshTerminals are guarded so nothing reconnects an iframe while suspended; _restoreTermFrames re-ticks to reconcile any adds/drops missed while hidden. A quick tab flip stays under the grace, so there's no needless reload. Co-Authored-By: Claude Opus 4.8 (1M context) --- chela/dashboard/static/js/terminals.js | 52 ++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/chela/dashboard/static/js/terminals.js b/chela/dashboard/static/js/terminals.js index acac6d4..eb19646 100644 --- a/chela/dashboard/static/js/terminals.js +++ b/chela/dashboard/static/js/terminals.js @@ -844,8 +844,59 @@ function stopTermTimer() { if (_termTimer) { clearInterval(_termTimer); _termTimer = null; } } +// ---- Background-tab teardown ---------------------------------------------- +// +// Grouped tmux sessions SHARE one real window, and a tmux window has exactly one +// size. With `window-size largest` (scripts/agent-terminals.sh), a wall left open +// in a backgrounded tab keeps its ttyd WebSocket alive indefinitely — the +// keepalive pings auto-pong even while hidden — and pins every shared window to +// that ghost's dimensions, so the wall you're actively viewing gets its rows/cols +// CROPPED to the stale tab's size. Terminal activity can't tell a quiet-but- +// watched pane from an abandoned one (that's what the old server-side reaper got +// wrong, and why it disconnected live panes). The browser CAN: Page Visibility +// reports when THIS tab is hidden. So when the tab goes hidden past a short grace, +// blank its ttyd iframes — closing their sockets, which drops the backing tmux +// clients so they stop distorting window-size — and reconnect when it's shown +// again. A quick tab flip stays under the grace, so there's no needless reload. +const TERM_HIDE_GRACE_MS = 45000; // hidden this long → release ttyd clients +let _termSuspended = false; +let _termHideTimer = null; + +function _teardownTermFrames() { + document.querySelectorAll('#term-stage iframe.term-frame').forEach(ifr => { + const src = ifr.getAttribute('src') || ''; + if (!src || src === 'about:blank') return; // nothing live to release + ifr.dataset.suspendedSrc = src; // stash for restore + ifr.src = 'about:blank'; // unload → WS close → tmux client drops + }); + _termSuspended = true; +} + +function _restoreTermFrames() { + _termSuspended = false; + document.querySelectorAll('#term-stage iframe.term-frame[data-suspended-src]').forEach(ifr => { + ifr.src = ifr.dataset.suspendedSrc; // reconnect, freshly sized to the now-visible tile + delete ifr.dataset.suspendedSrc; + }); + // Reconcile any agent adds/drops we skipped while suspended (guarded below). + if (TERMINALS_ON && currentTab === 'terminals') termTick(); +} + +// Single registration at module load. TERMINALS_ON is checked at fire time (it +// may not be resolved yet here), mirroring the window 'resize' handler above. +document.addEventListener('visibilitychange', () => { + if (!TERMINALS_ON) return; + clearTimeout(_termHideTimer); + if (document.hidden) { + _termHideTimer = setTimeout(_teardownTermFrames, TERM_HIDE_GRACE_MS); + } else if (_termSuspended) { + _restoreTermFrames(); + } +}); + async function termTick() { if (!TERMINALS_ON || currentTab !== 'terminals') return; + if (_termSuspended) return; // tab hidden: don't add/reconnect iframes we just released let agents; try { agents = await api('/api/agents'); } catch (e) { return; } // transient — try again next tick _agentsCache = agents; @@ -1109,6 +1160,7 @@ async function addWallTiles(wids) { // only refreshes the dropdown so a new shell is selectable (the displayed // iframe is never touched). Shared by the 4s termTick and the SSE windows event. async function _absorbFreshTerminals(live) { + if (_termSuspended) return; // tab hidden: defer new tiles until _restoreTermFrames re-ticks const fresh = live.filter(w => !_renderedWids.includes(w)); if (!fresh.length) return; if (_termMode === 'wall') { From a2f2ecb502c15e281287248965c65b1540a80cdf Mon Sep 17 00:00:00 2001 From: Liav Edry Date: Mon, 8 Jun 2026 11:18:13 +0300 Subject: [PATCH 3/4] dashboard: sync sidebar status dots to the wall's 4s tick (kill the 30s lag) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The sidebar agent dots only re-rendered on the global 30s refresh() loop, while the wall pane dots recolour every 4s from termTick — both off the same /api/agents session_status. So an agent that went idle (e.g. carmel) stayed "busy" green in the sidebar for up to 30s after the wall had already gone grey. Add syncSidebarDots(agents): recolour the sidebar .health-dot elements in place (no list rebuild — rows are name-sorted, so status never reorders them) and call it from termTick off the same poll that refreshes the wall dots. The sidebar now tracks the wall in lockstep; row add/remove still rides the 30s refreshSidebar. Co-Authored-By: Claude Opus 4.8 (1M context) --- chela/dashboard/static/js/nav.js | 18 ++++++++++++++++++ chela/dashboard/static/js/terminals.js | 5 ++++- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/chela/dashboard/static/js/nav.js b/chela/dashboard/static/js/nav.js index 2338e18..f409a89 100644 --- a/chela/dashboard/static/js/nav.js +++ b/chela/dashboard/static/js/nav.js @@ -119,6 +119,24 @@ function renderSidebarAgents(agents) { }).join(''); } +// Recolour the sidebar dots in place from fresh /api/agents data — no list +// rebuild (rows are name-sorted, so status never reorders them; only the dot +// colour changes). This lets a fast caller (the 4s wall tick) keep the sidebar +// in lockstep with the wall's pane dots instead of lagging up to REFRESH_MS (30s) +// behind it. Both read the same a.session_status, so once they refresh off the +// same poll they agree exactly. Row add/remove still rides the 30s refreshSidebar. +function syncSidebarDots(agents) { + if (!agents) return; + const by = {}; + agents.forEach(a => { if (a && a.name) by[a.name] = a; }); + document.querySelectorAll('#sidebar-agents .agent-row').forEach(row => { + const a = by[row.dataset.agent]; + if (!a) return; + const dot = row.querySelector('.health-dot'); + if (dot) dot.className = 'health-dot ' + agentDotColor(a); + }); +} + // Single source of the always-visible sidebar agent list. Owns the /api/agents // fetch that also primes _agentsCache (schedule dropdown, detail view, etc.). async function refreshSidebar() { diff --git a/chela/dashboard/static/js/terminals.js b/chela/dashboard/static/js/terminals.js index eb19646..b53aee8 100644 --- a/chela/dashboard/static/js/terminals.js +++ b/chela/dashboard/static/js/terminals.js @@ -915,8 +915,11 @@ async function termTick() { await _absorbFreshTerminals(live); // Refresh the busy/idle/waiting dots from this poll's fresh status. Also - // refresh labels in case a rename changed a pane's friendly name. + // recolour the sidebar dots off the SAME poll so they stay in lockstep with + // the wall instead of lagging until the next 30s refresh, and refresh labels + // in case a rename changed a pane's friendly name. _applyTermStatus(agents); + syncSidebarDots(agents); _refreshPaneLabels(); try { _applyTermContext(await api('/api/agents/context')); } catch (e) { /* keep prior fills */ } } From c81e48189682d4248e9779a5d2161923d2a35d80 Mon Sep 17 00:00:00 2001 From: Liav Edry Date: Mon, 8 Jun 2026 11:31:29 +0300 Subject: [PATCH 4/4] webterm: only release backgrounded panes that have >1 viewer (contention gate) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The visibility teardown blanked every ttyd iframe when the tab went hidden, which needlessly churned the connection for the common case: a single wall watching its agents, no second viewer. Blanking a sole viewer has no upside — a tmux window has one size shared by all its clients, so a lone client always fits; cropping only happens when 2+ differently-sized clients share a window. Gate teardown on actual contention. New GET /api/term/clients returns per-wid ttyd client counts (one `tmux list-clients` call, counted by grouped-session name, mirroring scripts/agent-terminals.sh). On hide, the wall fetches it and blanks only panes whose window has >1 viewer; sole-viewer panes stay connected, so a backgrounded single wall keeps its terminals and returns with no reconnect. It enters the suspended state only if it actually released something, and aborts if the tab became visible mid-fetch. Co-Authored-By: Claude Opus 4.8 (1M context) --- chela/dashboard/app.py | 41 ++++++++++++++++++++++++++ chela/dashboard/static/js/terminals.js | 34 +++++++++++++++++---- 2 files changed, 69 insertions(+), 6 deletions(-) diff --git a/chela/dashboard/app.py b/chela/dashboard/app.py index 8a34d8e..b3f7397 100644 --- a/chela/dashboard/app.py +++ b/chela/dashboard/app.py @@ -576,6 +576,47 @@ def api_term_ready(): return jsonify({"ready": bool(port), "port": port if port else None}) +def _webterm_session(wid: str) -> str: + """The grouped tmux session name a wid's ttyd attaches to. Mirrors + scripts/agent-terminals.sh: WEBTERM_PREFIX = webterm__, + grp = WEBTERM_PREFIX, where sanitize = tr -c 'A-Za-z0-9_' '_'. + Kept in lockstep with that script — both must agree for the count to be + meaningful (a mismatch just yields 0, which the wall treats as "no contention, + don't tear down").""" + san = lambda s: re.sub(r"[^A-Za-z0-9_]", "_", s) + return f"webterm_{san(TMUX_SESSION)}_{san(wid)}" + + +@app.route("/api/term/clients") +@require_auth +def api_term_clients(): + """Per-wid ttyd client count: how many browser connections currently share + each pane's grouped tmux session. The wall reads this so a backgrounded tab + only releases panes that actually have >1 viewer — with a single viewer there + is no window-size contention to resolve (a tmux window has one size shared by + all its clients), so tearing it down would churn the connection for nothing. + One `tmux list-clients` call, counted by session; never touches ttyd.""" + _require_terminals() + wids = list(_terminals_port_map().keys()) + counts = {w: 0 for w in wids} + try: + out = subprocess.run( + ["tmux", "list-clients", "-F", "#{client_session}"], + capture_output=True, text=True, timeout=5, + ) + if out.returncode == 0: + per_session: dict = {} + for line in out.stdout.splitlines(): + s = line.strip() + if s: + per_session[s] = per_session.get(s, 0) + 1 + for w in wids: + counts[w] = per_session.get(_webterm_session(w), 0) + except Exception: + pass # tmux hiccup → all-zero counts → wall skips teardown (safe) + return jsonify(counts) + + _TERM_PASTE_MAX = 64 * 1024 # reject pastes larger than 64 KB diff --git a/chela/dashboard/static/js/terminals.js b/chela/dashboard/static/js/terminals.js index b53aee8..4eaab7d 100644 --- a/chela/dashboard/static/js/terminals.js +++ b/chela/dashboard/static/js/terminals.js @@ -858,18 +858,40 @@ function stopTermTimer() { // blank its ttyd iframes — closing their sockets, which drops the backing tmux // clients so they stop distorting window-size — and reconnect when it's shown // again. A quick tab flip stays under the grace, so there's no needless reload. -const TERM_HIDE_GRACE_MS = 45000; // hidden this long → release ttyd clients +// +// CONTENTION GATE: a pane is only torn down when its window has >1 viewer +// (/api/term/clients). With a single viewer there's no size contention to resolve +// — a tmux window has one size shared by all its clients, so a lone client always +// fits — and tearing it down would just churn the connection for nothing. So a +// backgrounded wall that's the only thing watching its agents keeps its terminals; +// teardown only fires for panes a second tab/device is also viewing. +const TERM_HIDE_GRACE_MS = 45000; // hidden this long → release contended ttyd clients let _termSuspended = false; let _termHideTimer = null; -function _teardownTermFrames() { +// /term// → wid. Reads the live src (or the stashed one once blanked). +function _widOfFrame(ifr) { + const src = ifr.dataset.suspendedSrc || ifr.getAttribute('src') || ''; + const m = src.match(/\/term\/([^/]+)\//); + return m ? decodeURIComponent(m[1]) : null; +} + +async function _teardownTermFrames() { + let counts; + try { counts = await api('/api/term/clients'); } + catch (e) { return; } // can't tell contention → disturb nothing + if (document.visibilityState !== 'hidden') return; // became visible mid-fetch → abort + let released = 0; document.querySelectorAll('#term-stage iframe.term-frame').forEach(ifr => { const src = ifr.getAttribute('src') || ''; - if (!src || src === 'about:blank') return; // nothing live to release - ifr.dataset.suspendedSrc = src; // stash for restore - ifr.src = 'about:blank'; // unload → WS close → tmux client drops + if (!src || src === 'about:blank') return; // nothing live to release + const wid = _widOfFrame(ifr); + if (!wid || (counts[wid] || 0) <= 1) return; // sole viewer → leave it connected + ifr.dataset.suspendedSrc = src; // stash for restore + ifr.src = 'about:blank'; // unload → WS close → tmux client drops + released++; }); - _termSuspended = true; + if (released) _termSuspended = true; // only suspend reactive ticks if we actually released } function _restoreTermFrames() {