webterm: fix wall session disconnects, stale-tab cropping, and sidebar status lag

chela/dashboard/app.py

@@ -576,6 +576,47 @@ def api_term_ready():
     return jsonify({"ready": bool(port), "port": port if port else None})
 
 
+def _webterm_session(wid: str) -> str:
+    """The grouped tmux session name a wid's ttyd attaches to. Mirrors
+    scripts/agent-terminals.sh: WEBTERM_PREFIX = webterm_<sanitized session>_,
+    grp = WEBTERM_PREFIX<sanitized wid>, where sanitize = tr -c 'A-Za-z0-9_' '_'.
+    Kept in lockstep with that script — both must agree for the count to be
+    meaningful (a mismatch just yields 0, which the wall treats as "no contention,
+    don't tear down")."""
+    san = lambda s: re.sub(r"[^A-Za-z0-9_]", "_", s)
+    return f"webterm_{san(TMUX_SESSION)}_{san(wid)}"
+
+
+@app.route("/api/term/clients")
+@require_auth
+def api_term_clients():
+    """Per-wid ttyd client count: how many browser connections currently share
+    each pane's grouped tmux session. The wall reads this so a backgrounded tab
+    only releases panes that actually have >1 viewer — with a single viewer there
+    is no window-size contention to resolve (a tmux window has one size shared by
+    all its clients), so tearing it down would churn the connection for nothing.
+    One `tmux list-clients` call, counted by session; never touches ttyd."""
+    _require_terminals()
+    wids = list(_terminals_port_map().keys())
+    counts = {w: 0 for w in wids}
+    try:
+        out = subprocess.run(
+            ["tmux", "list-clients", "-F", "#{client_session}"],
+            capture_output=True, text=True, timeout=5,
+        )
+        if out.returncode == 0:
+            per_session: dict = {}
+            for line in out.stdout.splitlines():
+                s = line.strip()
+                if s:
+                    per_session[s] = per_session.get(s, 0) + 1
+            for w in wids:
+                counts[w] = per_session.get(_webterm_session(w), 0)
+    except Exception:
+        pass  # tmux hiccup → all-zero counts → wall skips teardown (safe)
+    return jsonify(counts)
+
+
 _TERM_PASTE_MAX = 64 * 1024  # reject pastes larger than 64 KB
 
 

chela/dashboard/static/js/nav.js

@@ -119,6 +119,24 @@ function renderSidebarAgents(agents) {
     }).join('');
 }
 
+// Recolour the sidebar dots in place from fresh /api/agents data — no list
+// rebuild (rows are name-sorted, so status never reorders them; only the dot
+// colour changes). This lets a fast caller (the 4s wall tick) keep the sidebar
+// in lockstep with the wall's pane dots instead of lagging up to REFRESH_MS (30s)
+// behind it. Both read the same a.session_status, so once they refresh off the
+// same poll they agree exactly. Row add/remove still rides the 30s refreshSidebar.
+function syncSidebarDots(agents) {
+    if (!agents) return;
+    const by = {};
+    agents.forEach(a => { if (a && a.name) by[a.name] = a; });
+    document.querySelectorAll('#sidebar-agents .agent-row').forEach(row => {
+        const a = by[row.dataset.agent];
+        if (!a) return;
+        const dot = row.querySelector('.health-dot');
+        if (dot) dot.className = 'health-dot ' + agentDotColor(a);
+    });
+}
+
 // Single source of the always-visible sidebar agent list. Owns the /api/agents
 // fetch that also primes _agentsCache (schedule dropdown, detail view, etc.).
 async function refreshSidebar() {

chela/dashboard/static/js/terminals.js

@@ -844,8 +844,81 @@ function stopTermTimer() {
     if (_termTimer) { clearInterval(_termTimer); _termTimer = null; }
 }
 
+// ---- Background-tab teardown ----------------------------------------------
+//
+// Grouped tmux sessions SHARE one real window, and a tmux window has exactly one
+// size. With `window-size largest` (scripts/agent-terminals.sh), a wall left open
+// in a backgrounded tab keeps its ttyd WebSocket alive indefinitely — the
+// keepalive pings auto-pong even while hidden — and pins every shared window to
+// that ghost's dimensions, so the wall you're actively viewing gets its rows/cols
+// CROPPED to the stale tab's size. Terminal activity can't tell a quiet-but-
+// watched pane from an abandoned one (that's what the old server-side reaper got
+// wrong, and why it disconnected live panes). The browser CAN: Page Visibility
+// reports when THIS tab is hidden. So when the tab goes hidden past a short grace,
+// blank its ttyd iframes — closing their sockets, which drops the backing tmux
+// clients so they stop distorting window-size — and reconnect when it's shown
+// again. A quick tab flip stays under the grace, so there's no needless reload.
+//
+// CONTENTION GATE: a pane is only torn down when its window has >1 viewer
+// (/api/term/clients). With a single viewer there's no size contention to resolve
+// — a tmux window has one size shared by all its clients, so a lone client always
+// fits — and tearing it down would just churn the connection for nothing. So a
+// backgrounded wall that's the only thing watching its agents keeps its terminals;
+// teardown only fires for panes a second tab/device is also viewing.
+const TERM_HIDE_GRACE_MS = 45000;   // hidden this long → release contended ttyd clients
+let _termSuspended = false;
+let _termHideTimer = null;
+
+// /term/<wid>/ → wid. Reads the live src (or the stashed one once blanked).
+function _widOfFrame(ifr) {
+    const src = ifr.dataset.suspendedSrc || ifr.getAttribute('src') || '';
+    const m = src.match(/\/term\/([^/]+)\//);
+    return m ? decodeURIComponent(m[1]) : null;
+}
+
+async function _teardownTermFrames() {
+    let counts;
+    try { counts = await api('/api/term/clients'); }
+    catch (e) { return; }                              // can't tell contention → disturb nothing
+    if (document.visibilityState !== 'hidden') return; // became visible mid-fetch → abort
+    let released = 0;
+    document.querySelectorAll('#term-stage iframe.term-frame').forEach(ifr => {
+        const src = ifr.getAttribute('src') || '';
+        if (!src || src === 'about:blank') return;      // nothing live to release
+        const wid = _widOfFrame(ifr);
+        if (!wid || (counts[wid] || 0) <= 1) return;    // sole viewer → leave it connected
+        ifr.dataset.suspendedSrc = src;                 // stash for restore
+        ifr.src = 'about:blank';                         // unload → WS close → tmux client drops
+        released++;
+    });
+    if (released) _termSuspended = true;   // only suspend reactive ticks if we actually released
+}
+
+function _restoreTermFrames() {
+    _termSuspended = false;
+    document.querySelectorAll('#term-stage iframe.term-frame[data-suspended-src]').forEach(ifr => {
+        ifr.src = ifr.dataset.suspendedSrc;             // reconnect, freshly sized to the now-visible tile
+        delete ifr.dataset.suspendedSrc;
+    });
+    // Reconcile any agent adds/drops we skipped while suspended (guarded below).
+    if (TERMINALS_ON && currentTab === 'terminals') termTick();
+}
+
+// Single registration at module load. TERMINALS_ON is checked at fire time (it
+// may not be resolved yet here), mirroring the window 'resize' handler above.
+document.addEventListener('visibilitychange', () => {
+    if (!TERMINALS_ON) return;
+    clearTimeout(_termHideTimer);
+    if (document.hidden) {
+        _termHideTimer = setTimeout(_teardownTermFrames, TERM_HIDE_GRACE_MS);
+    } else if (_termSuspended) {
+        _restoreTermFrames();
+    }
+});
+
 async function termTick() {
     if (!TERMINALS_ON || currentTab !== 'terminals') return;
+    if (_termSuspended) return;   // tab hidden: don't add/reconnect iframes we just released
     let agents;
     try { agents = await api('/api/agents'); } catch (e) { return; }   // transient — try again next tick
     _agentsCache = agents;
@@ -864,8 +937,11 @@ async function termTick() {
     await _absorbFreshTerminals(live);
 
     // Refresh the busy/idle/waiting dots from this poll's fresh status. Also
-    // refresh labels in case a rename changed a pane's friendly name.
+    // recolour the sidebar dots off the SAME poll so they stay in lockstep with
+    // the wall instead of lagging until the next 30s refresh, and refresh labels
+    // in case a rename changed a pane's friendly name.
     _applyTermStatus(agents);
+    syncSidebarDots(agents);
     _refreshPaneLabels();
     try { _applyTermContext(await api('/api/agents/context')); } catch (e) { /* keep prior fills */ }
 }
@@ -1109,6 +1185,7 @@ async function addWallTiles(wids) {
 // only refreshes the dropdown so a new shell is selectable (the displayed
 // iframe is never touched). Shared by the 4s termTick and the SSE windows event.
 async function _absorbFreshTerminals(live) {
+    if (_termSuspended) return;   // tab hidden: defer new tiles until _restoreTermFrames re-ticks
     const fresh = live.filter(w => !_renderedWids.includes(w));
     if (!fresh.length) return;
     if (_termMode === 'wall') {

scripts/agent-terminals.sh

@@ -34,14 +34,11 @@ TMUX_SESSION="${CHELA_TMUX_SESSION:-chela}"
 BASE_PORT="${CHELA_TERM_BASE:-5101}"
 POLL="${CHELA_TERM_POLL:-12}"
 # Each ttyd allows this many concurrent browser connections (= tmux clients on
-# its grouped session). Generous headroom: a dropped/backgrounded tab leaves a
-# client attached until reaped (see CLIENT_IDLE_MAX), and a tight cap would lock
-# the tile out of new connections once a few stale ones pile up.
+# its grouped session). Generous headroom: a dropped/backgrounded tab can leave a
+# client attached until its socket is torn down, and a tight cap would lock the
+# tile out of new connections once a few stale ones pile up. We deliberately do
+# NOT time-reap quiet clients — see the WHY-NO-REAPER note above the poll loop.
 MAX_CLIENTS="${CHELA_TERM_MAX_CLIENTS:-10}"
-# Detach a wall ttyd client after this many seconds idle, so an abandoned tab
-# can't hold a --max-clients slot forever. Scoped to webterm_* clients only — the
-# user's own interactive tmux session is never reaped.
-CLIENT_IDLE_MAX="${CHELA_TERM_CLIENT_IDLE_MAX:-3600}"
 MAP_FILE="${CHELA_DIR:-${HOME}/.chela}/agent_terminals.json"
 # Ensure the state dir exists — a fresh install has no ~/.chela yet, and the
 # atomic map write (write_map: printf > "$MAP_FILE.tmp" && mv) silently fails
@@ -140,24 +137,6 @@ spawn() {
     PORT_OF["$wid"]=$port
 }
 
-# Detach wall ttyd clients idle longer than CLIENT_IDLE_MAX. ttyd keeps one tmux
-# client per WebSocket; a dropped or backgrounded browser tab can leave its client
-# attached indefinitely, consuming a --max-clients slot until the grouped session
-# refuses new connections. Filtered to WEBTERM_PREFIX sessions so the user's own
-# interactive `${TMUX_SESSION}` client is never touched.
-reap_stale_clients() {
-    local now act tty sess
-    now="$(date +%s)"
-    while IFS='|' read -r act tty sess; do
-        [[ -z "$tty" ]] && continue
-        [[ "$sess" == "${WEBTERM_PREFIX}"* ]] || continue
-        if (( now - act > CLIENT_IDLE_MAX )); then
-            tmux detach-client -t "$tty" 2>/dev/null \
-                && echo "agent-terminals: reaped idle client ${tty} (${sess}, idle $((now-act))s)"
-        fi
-    done < <(tmux list-clients -F '#{client_activity}|#{client_name}|#{client_session}' 2>/dev/null)
-}
-
 despawn() {
     local wid="$1"
     [[ -n "${PID_OF[$wid]:-}" ]] && kill "${PID_OF[$wid]}" 2>/dev/null
@@ -224,6 +203,17 @@ echo "agent-terminals: dynamic supervisor (poll ${POLL}s, base port ${BASE_PORT}
 
 write_map               # write an (empty) map even before any agent is seen
 
+# WHY-NO-REAPER: we used to detach wall ttyd clients idle past a timeout to free
+# --max-clients slots held by abandoned tabs. That was wrong: the only idle signal
+# tmux exposes is `client_activity`, which advances on terminal I/O — NOT on the
+# WebSocket keepalive pings (app.py ping_interval=25) that keep a quiet pane's
+# socket warm. An agent terminal that simply produces no output (the normal case
+# for a wall you're watching) thus looked "idle" and got force-detached after the
+# timeout, dropping a perfectly live tile to ttyd's reconnect screen. tmux cannot
+# tell an abandoned tab from a live-but-quiet one, so any activity-based reaper
+# kills real sessions. Genuinely dead sockets are already torn down by the
+# keepalive: a failed ping raises in the proxy pumps, ttyd drops the client, and
+# tmux detaches it — which frees the slot. --max-clients headroom absorbs the rest.
 declare -A CUR NAME_OF
 while :; do
     # snapshot the live window set, keyed by stable window id
@@ -268,6 +258,5 @@ while :; do
     fi
 
     (( changed )) && write_map
-    reap_stale_clients      # free --max-clients slots held by abandoned tabs
     sleep "${POLL}"
 done