From 62d8163f10855af9e647a71cd0807814fbe9c3ef Mon Sep 17 00:00:00 2001
From: Guy Cochran <guycochran@gmail.com>
Date: Sat, 10 Oct 2026 21:21:08 +0000
Subject: [PATCH 1/3] =?UTF-8?q?fix(healers):=20runtime=20flow-UUID=20disco?=
 =?UTF-8?q?very=20=E2=80=94=20commit=20pgm-heal=20+=20de-hardcode=20thumbs?=
 =?UTF-8?q?=20(R2)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The healers carried facility-specific flow UUIDs, so pgm-heal could never be
committed (it only lived on the box) and the thumbs probe baked in one lab's
SLOTS — stale UUIDs made the Health panel show dead slots for sources that were
never present and miss the ones that were. This is the healer-side of reviewer
issue #2 (the backend/quickstart side was fixed via facility_from_discovery.py).

Every flow is already self-describing: `mxl-info -l` groups flows under a stable
ROLE name (Pattern, Playout, Guest1..N, Guest5Stable, Keyer, Selector) and
flow_def.json carries the same role in its grouphint tag. So the healers name
ROLES and resolve UUIDs at runtime — portable to any facility.

- tools/mxl-flows.sh: NEW sourceable POSIX discovery helper (flow_by_role /
  flow_present / all_video_roles) over a cached `mxl-info -l` snapshot.
- tools/mxl-pgm-heal.sh: NEW (was box-only). Boot/recovery heal — wire selector
  from DISCOVERED present roles, cut to the first advancing slot, point the
  encoder at the selector output (keyer bypassed by default), verify advancing.
  Zero hardcoded UUIDs; correspondent slots prefer GuestNStable (never-interrupt).
- tools/mxl_thumbs.py: discover SLOTS from the domain instead of a static dict;
  a supervisor re-scans so sources appearing AFTER startup get a worker and a
  role whose flow is recreated with a new UUID re-attaches (fixes the ~2min slow
  re-attach wedge); exclude selector/keyer (program outputs), collapse
  GuestNStable -> guestN. Run loop guarded under __main__ so it's importable.
- tests/test_thumb_slots.py: pin the discovery contract (7 cases).

Suite: 156 pytest + 68 node green. No live-system changes (code+tests only).

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_017YNw5igfxCp4mUoMMMM9pv
---
 tests/test_thumb_slots.py |  88 +++++++++++++++++++
 tools/mxl-flows.sh        |  67 +++++++++++++++
 tools/mxl-pgm-heal.sh     | 174 ++++++++++++++++++++++++++++++++++++++
 tools/mxl_thumbs.py       | 143 ++++++++++++++++++++++++++-----
 4 files changed, 453 insertions(+), 19 deletions(-)
 create mode 100644 tests/test_thumb_slots.py
 create mode 100755 tools/mxl-flows.sh
 create mode 100755 tools/mxl-pgm-heal.sh

diff --git a/tests/test_thumb_slots.py b/tests/test_thumb_slots.py
new file mode 100644
index 0000000..6707ce9
--- /dev/null
+++ b/tests/test_thumb_slots.py
@@ -0,0 +1,88 @@
+"""mxl_thumbs.py discovers its multiview slots from the live domain (reviewer R2:
+no hardcoded, facility-specific UUIDs). These tests pin the discovery contract
+against a synthetic domain of flow_def.json files — no GStreamer, no threads.
+
+discover_slots() is imported from the probe module; the run loop is guarded under
+__main__, and Gst.init happens there, so importing for the test starts nothing.
+"""
+import importlib
+import json
+import sys
+from pathlib import Path
+
+import pytest
+
+REPO = Path(__file__).resolve().parent.parent
+TOOLS = REPO / "tools"
+
+pytest.importorskip("gi")  # the probe imports GStreamer bindings at module top
+
+
+@pytest.fixture(scope="module")
+def mod():
+    sys.path.insert(0, str(TOOLS))
+    try:
+        m = importlib.import_module("mxl_thumbs")
+    finally:
+        sys.path.pop(0)
+    return m
+
+
+def _flow(domain, uuid, fmt, grouphint=None, label=""):
+    d = domain / f"{uuid}.mxl-flow"
+    d.mkdir()
+    body = {"id": uuid, "format": f"urn:x-nmos:format:{fmt}", "label": label}
+    if grouphint:
+        body["tags"] = {"urn:x-nmos:tag:grouphint/v1.0": [grouphint]}
+    (d / "flow_def.json").write_text(json.dumps(body))
+
+
+def test_discovers_video_sources_by_role(mod, tmp_path):
+    _flow(tmp_path, "d3e15194-6d1f-5955-8d52-52e7076b7a99", "video", "Pattern:Video", "Pattern Video")
+    _flow(tmp_path, "9998da48-9041-5ae1-b730-d0a918563107", "video", "Playout:Video", "Clip Video")
+    _flow(tmp_path, "9e111e00-aaaa-4bbb-8ccc-000000000001", "video", "Guest1:Video", "Guest 1")
+    slots = mod.discover_slots(str(tmp_path))
+    assert slots == {
+        "pattern": "d3e15194-6d1f-5955-8d52-52e7076b7a99",
+        "playout": "9998da48-9041-5ae1-b730-d0a918563107",
+        "guest1": "9e111e00-aaaa-4bbb-8ccc-000000000001",
+    }
+
+
+def test_audio_flows_are_skipped(mod, tmp_path):
+    _flow(tmp_path, "d3e15194-6d1f-5955-8d52-52e7076b7a99", "video", "Pattern:Video")
+    _flow(tmp_path, "7c1dd9ba-ca2c-5737-889b-abdd23c06020", "audio", "Pattern:Audio")
+    slots = mod.discover_slots(str(tmp_path))
+    assert list(slots) == ["pattern"]  # the audio flow produced no tile
+
+
+def test_program_outputs_excluded(mod, tmp_path):
+    """selector/keyer are the program bus, not selectable source tiles."""
+    _flow(tmp_path, "5a617f19-7c48-598c-9e94-4611c6316273", "video", "Selector:Video", "Selector PGM")
+    _flow(tmp_path, "86efffa4-9b4e-558c-8816-7e200e20cf17", "video", "Keyer:Video", "Keyer PGM")
+    _flow(tmp_path, "9e333e00-aaaa-4bbb-8ccc-000000000001", "video", "Guest3:Video", "Guest 3")
+    slots = mod.discover_slots(str(tmp_path))
+    assert list(slots) == ["guest3"]
+
+
+def test_stable_collapses_to_friendly_guest_name(mod, tmp_path):
+    """A never-interrupt slot's GuestNStable flow shows under the friendly 'guestN'."""
+    _flow(tmp_path, "57ab5e00-aaaa-4bbb-8ccc-000000000001", "video", "Guest5Stable:Video", "Guest 5 Stable")
+    slots = mod.discover_slots(str(tmp_path))
+    assert slots == {"guest5": "57ab5e00-aaaa-4bbb-8ccc-000000000001"}
+
+
+def test_env_force_overrides_discovery(mod, tmp_path):
+    forced = "cam=aaa,guest1=bbb"
+    slots = mod.discover_slots(str(tmp_path), forced=forced)
+    assert slots == {"cam": "aaa", "guest1": "bbb"}
+
+
+def test_missing_domain_is_empty_not_error(mod, tmp_path):
+    assert mod.discover_slots(str(tmp_path / "does-not-exist")) == {}
+
+
+def test_untagged_flow_falls_back_to_uuid_prefix(mod, tmp_path):
+    _flow(tmp_path, "abcd1234-6d1f-5955-8d52-52e7076b7a99", "video")  # no grouphint
+    slots = mod.discover_slots(str(tmp_path))
+    assert slots == {"abcd1234": "abcd1234-6d1f-5955-8d52-52e7076b7a99"}
diff --git a/tools/mxl-flows.sh b/tools/mxl-flows.sh
new file mode 100755
index 0000000..bb6a3ad
--- /dev/null
+++ b/tools/mxl-flows.sh
@@ -0,0 +1,67 @@
+#!/usr/bin/env sh
+# mxl-flows.sh — runtime flow-UUID discovery for the MXL switcher healers.
+#
+# WHY: pgm-heal and selfheal used to carry HARDCODED flow UUIDs ("G1=9e111e00…",
+# "PATTERN=d3e15194…"). Those UUIDs are box-specific — they are whatever the
+# quickstart happened to mint on THIS facility — so a script full of them cannot
+# be committed to the shared repo (reviewer issue #2): on any other facility the
+# UUIDs are wrong, the selector gets wired to flows that do not exist, and the
+# down-selector rebuild wires only pattern/playout while 400ing on the phantom
+# guest UUIDs (HW Oct 10).
+#
+# The facility is already self-describing. `mxl-info -d DOMAIN -l` prints, per
+# device group, a stable ROLE name and the flows under it:
+#
+#     Guest1: mxl:///mxl-domain?id=9e111e00-…
+#         Video : 9e111e00-… - Guest 1
+#     Pattern: mxl:///mxl-domain?id=d3e15194-…&id=7c1dd9ba-…
+#         Video : d3e15194-… - Pattern Video
+#         Audio : 7c1dd9ba-… - Pattern Audio
+#
+# The group names (Pattern, Playout, Guest1..GuestN, Guest5Stable, Keyer,
+# Selector) are the role identity; the UUID under each is discovered at runtime.
+# So the healers name ROLES, never UUIDs, and work on any facility.
+#
+# Sourced, not executed. POSIX sh. Requires: DOMAIN set, DOCKER set (the caller's
+# `docker` or `sudo docker`), SEL_CTR set (the selector container name).
+
+# mxl_flows_dump — cache a single `mxl-info -l` snapshot in MXL_FLOWS_CACHE so
+# repeated lookups in one heal pass cost one docker exec, not one per role.
+# Call mxl_flows_refresh at the top of each pass to invalidate it.
+MXL_FLOWS_CACHE=""
+mxl_flows_refresh(){ MXL_FLOWS_CACHE=""; }
+mxl_flows_dump(){
+  [ -n "$MXL_FLOWS_CACHE" ] && { printf '%s' "$MXL_FLOWS_CACHE"; return 0; }
+  MXL_FLOWS_CACHE=$($DOCKER exec "$SEL_CTR" sh -c "/opt/mxl/tools/mxl-info/mxl-info -d $DOMAIN -l" 2>/dev/null)
+  [ -n "$MXL_FLOWS_CACHE" ] || return 1
+  printf '%s' "$MXL_FLOWS_CACHE"
+}
+
+# flow_by_role ROLE [MEDIA]  — UUID of the flow for a device group ($1, e.g.
+# "Guest1", "Pattern", "Guest5Stable") and media kind ($2, Video|Audio, default
+# Video). Empty string + rc1 if that role/media is not present in the domain.
+#
+#   The group header line is "ROLE: mxl://…"; the flow lines under it are
+#   "<ws>Video : <uuid> - <label>". We locate the group block, then the first
+#   matching media line within it. Portable awk — no bashisms.
+flow_by_role(){
+  _role="$1"; _media="${2:-Video}"
+  mxl_flows_dump | awk -v role="$_role" -v media="$_media" '
+    $0 ~ "^"role":" { inblk=1; next }
+    /^[A-Za-z0-9]+:/  { inblk=0 }           # next group header ends the block
+    inblk && $1==media && $2==":" { print $3; exit }
+  '
+}
+
+# flow_present ROLE [MEDIA]  — true if the role/media flow exists right now.
+flow_present(){ [ -n "$(flow_by_role "$1" "${2:-Video}")" ]; }
+
+# all_video_roles — every device group that currently offers a Video flow, one
+# role per line, in mxl-info order. Lets a healer enumerate "whatever sources
+# exist" instead of hardcoding a slot list.
+all_video_roles(){
+  mxl_flows_dump | awk '
+    /^[A-Za-z0-9]+:/ { role=$1; sub(":","",role); next }
+    $1=="Video" && $2==":" && role!="" { print role; role="" }
+  '
+}
diff --git a/tools/mxl-pgm-heal.sh b/tools/mxl-pgm-heal.sh
new file mode 100755
index 0000000..2f34a37
--- /dev/null
+++ b/tools/mxl-pgm-heal.sh
@@ -0,0 +1,174 @@
+#!/usr/bin/env sh
+# mxl-pgm-heal — bring the whole PGM/output chain to a known-good state after a (re)boot.
+#
+# WHY: a facility bring-up (quickstart) starts the containers, but the SELECTOR wiring + the
+# mxl2webrtc publish need to settle in the right ORDER, each AFTER its upstream is live, or a remote
+# viewer gets grey/frozen (HW lessons, Oct 8). With the CEF keyer BYPASSED (MXL_KEYER_ENABLE=0,
+# encoder reads the selector directly), the heal is: wait for live source flows -> wire them into the
+# selector (so all are cuttable) -> make the encoder read the selector output -> republish mxl2webrtc
+# -> verify the selector output is readable + advancing (real moving frames to viewers).
+#
+# Idempotent + verification-driven. Ordered LAST (After=everything). POSIX sh.
+#
+# RUNTIME UUID DISCOVERY (no hardcoded flow ids): every source is named by its ROLE
+# (Pattern, Playout, Guest1..GuestN, Guest5Stable/Guest6Stable, Keyer, Selector) and the
+# UUID is resolved at runtime from `mxl-info -l` via tools/mxl-flows.sh. This is what makes
+# the script facility-portable: on any box it wires whatever sources are actually present,
+# instead of 400ing on another facility's phantom UUIDs (reviewer issue #2).
+set -u
+
+DOMAIN="${MXL_DOMAIN:-/mxl-domain}"
+SEL_PORT="${MXL_SELECTOR_PORT:-9604}"; ENC_PORT="${MXL_ENCODER_PORT:-9601}"
+SEL_CTR="${SELECTOR_CONTAINER:-input-selector}"
+
+# docker directly if we can (root), else sudo docker.
+if docker ps >/dev/null 2>&1; then DOCKER="docker"; else DOCKER="sudo docker"; fi
+
+# discovery helpers (flow_by_role / flow_present / all_video_roles / mxl_flows_refresh)
+_here=$(CDPATH= cd "$(dirname "$0")" && pwd)
+. "$_here/mxl-flows.sh"
+
+# Fixed studio sources stay on their VOLATILE flow (they don't reconnect mid-show). The
+# roving-correspondent slots prefer their STABLE flow (never-interrupt, docs/NEVER-INTERRUPT-PGM.md):
+# a stabilizer (mxl-stabilizer@guestN) owns GuestNStable, created once at boot and kept alive across
+# the correspondent connecting/dropping, so a join NEVER forces a selector re-wire -> PGM never drops.
+# Discovered-by-role, so a facility with different guest counts just works.
+FIXED_ROLES="${MXL_FIXED_ROLES:-Pattern Playout Guest1 Guest2 Guest3 Guest4}"
+STABLE_ROLES="${MXL_STABLE_ROLES:-Guest5 Guest6}"   # prefer GuestNStable, else fall back to GuestN
+
+log(){ echo "mxl-pgm-heal: $*"; }
+headidx(){ $DOCKER exec "$SEL_CTR" sh -c "/opt/mxl/tools/mxl-info/mxl-info -d $DOMAIN -f $1 2>/dev/null" 2>/dev/null | grep -i 'Head index' | grep -oE '[0-9]+' | head -1; }
+# advancing: is a flow actually MOVING right now? (costs a 1s sample). A stable flow that
+# exists but freewheeled dark reads "not advancing" — so we must never CUT to it, only PRE-WIRE
+# it. The default program cut uses this to avoid the dead-source blank (HW Oct 10 bug).
+advancing(){ a=$(headidx "$1"); sleep 1; b=$(headidx "$1"); [ -n "$a" ] && [ -n "$b" ] && [ "$b" -gt "$a" ] 2>/dev/null; }
+
+# 1. wait for the selector to answer + the always-on Pattern source to exist
+i=0
+while [ "$i" -lt 30 ]; do
+  mxl_flows_refresh
+  curl -s -m3 "http://127.0.0.1:${SEL_PORT}/pipeline/status" >/dev/null 2>&1 && flow_present Pattern && break
+  i=$((i+1)); sleep 2
+done
+mxl_flows_refresh
+PATTERN=$(flow_by_role Pattern Video)
+[ -n "$PATTERN" ] || { log "FATAL: Pattern flow never appeared (facility not up?)"; exit 1; }
+
+# 2. build the input list from PRESENT roles. Pattern is slot 0 (always-on fallback). Fixed
+# studio sources on their volatile flow; correspondent slots prefer GuestNStable, else GuestN.
+# SLOTS[] tracks the uuid each wired slot carries, so the default-cut below tests the RIGHT flow.
+INPUTS="\"$PATTERN\""
+SLOTS="$PATTERN"
+add_uuid(){ INPUTS="$INPUTS,\"$1\""; SLOTS="$SLOTS $1"; }
+for role in $FIXED_ROLES; do
+  [ "$role" = "Pattern" ] && continue        # already slot 0
+  u=$(flow_by_role "$role" Video); [ -n "$u" ] && add_uuid "$u"
+done
+for role in $STABLE_ROLES; do
+  u=$(flow_by_role "${role}Stable" Video)     # prefer the stable (never-interrupt) flow
+  [ -n "$u" ] || u=$(flow_by_role "$role" Video)   # fall back to the volatile flow
+  [ -n "$u" ] && add_uuid "$u"
+done
+log "wiring selector: $INPUTS"
+curl -s -m8 -X POST "http://127.0.0.1:${SEL_PORT}/pipeline/start" -H 'Content-Type: application/json' \
+  -d "{\"domain_path\":\"$DOMAIN\",\"input_flow_uuids\":[$INPUTS],\"grouphint\":\"Selector\",\"description\":\"program out\",\"label\":\"Selector PGM\"}" \
+  -o /dev/null -w 'selector wire -> %{http_code}\n' | sed 's/^/mxl-pgm-heal: /'
+sleep 2
+# default program on the first slot that is ACTUALLY ADVANCING (not merely present) — a
+# dead/idle source (incl. a freewheeled-dark stable flow) would blank PGM (HW Oct 10 bug).
+# Walk slots in order; pattern(0) is always-on so it's the guaranteed fallback.
+CUT=0; idx=0
+for u in $SLOTS; do
+  if [ "$idx" -gt 0 ] && advancing "$u"; then CUT=$idx; break; fi
+  idx=$((idx+1))
+done
+curl -s -m5 -X POST "http://127.0.0.1:${SEL_PORT}/pipeline/active-input" -H 'Content-Type: application/json' -d "{\"slot\":$CUT}" >/dev/null 2>&1
+log "program cut to slot $CUT (verified advancing, else pattern)"
+
+# 3. the encoder must read the SELECTOR output (keyer bypass). Get the live selector out flow.
+SEL_OUT=$(curl -s -m4 "http://127.0.0.1:${SEL_PORT}/pipeline/status" 2>/dev/null | python3 -c 'import sys,json;print(json.load(sys.stdin).get("output_flow_uuid",""))' 2>/dev/null)
+[ -n "$SEL_OUT" ] || { log "FATAL: no selector output flow"; exit 1; }
+log "selector output = $SEL_OUT"
+
+# 3b. KEYER source selection. MXL_KEYER_ENABLE=1 means the operator WANTS the gst-keyer
+# lower-third on program; wait (with retries) for Keyer-out to be advancing before wiring
+# the encoder to it, so a brief gap during the program cut doesn't drop graphics to bypass.
+# ⚠️ Re-enabling the keyer needs its throughput fixed first (HW Oct 10: it throttled the whole
+# chain to ~300KB/s and its flow keeps a stale advancing head even when stopped). Default OFF.
+ENC_SRC="$SEL_OUT"
+if [ "${MXL_KEYER_ENABLE:-0}" = "1" ]; then
+  KEYER_OUT=$(flow_by_role Keyer Video)
+  if [ -n "$KEYER_OUT" ]; then
+    t=0
+    while [ "$t" -lt 8 ]; do
+      if advancing "$KEYER_OUT"; then
+        ENC_SRC="$KEYER_OUT"; log "keyer-out advancing -> encoder reads KEYED program (lower-third ON)"; break
+      fi
+      t=$((t+1))
+    done
+    [ "$ENC_SRC" = "$KEYER_OUT" ] || log "MXL_KEYER_ENABLE=1 but keyer-out never advanced after 8s -> bypass (check mxl-gst-keyer.service)"
+  else
+    log "MXL_KEYER_ENABLE=1 but no Keyer flow present -> bypass"
+  fi
+else
+  log "keyer disabled (MXL_KEYER_ENABLE!=1) -> encoder reads selector (bypass)"
+fi
+# 4. restart mxl2webrtc fresh, then re-point it at ENC_SRC. Two HW-learned hazards (Oct 10):
+#   (a) a fixed `sleep 10` after `docker restart` races the container's API coming up -> the
+#       start POST returns 000 and the encoder is left reading its stale/default flow (the
+#       keyer, which may be dead) -> PGM grey. So WAIT for the API to actually answer.
+#   (b) a `start` while the pipeline is already `running` does NOT switch the flow -> the
+#       encoder sticks on the old flow (reported running:true, video_flow_uuid unchanged). So
+#       STOP until running:false BEFORE the start, and VERIFY the start actually took.
+# Only HARD-restart the container if its API is unreachable. A `docker restart` makes the
+# encoder auto-start on its DEFAULT flow (the keyer), which then races our stop/start below
+# and can leave PGM reading a dead keyer -> grey (HW Oct 10). If the API already answers,
+# a stop-until-stopped + start is cleaner and switches the flow reliably.
+if curl -s -m3 "http://127.0.0.1:${ENC_PORT}/pipeline/status" >/dev/null 2>&1; then
+  log "mxl2webrtc API up — re-pointing without a container restart"
+else
+  log "mxl2webrtc API down — restarting container"
+  docker restart mxl2webrtc >/dev/null 2>&1 || sudo docker restart mxl2webrtc >/dev/null 2>&1
+  j=0; while [ "$j" -lt 20 ]; do
+    curl -s -m3 "http://127.0.0.1:${ENC_PORT}/pipeline/status" >/dev/null 2>&1 && break
+    j=$((j+1)); sleep 2
+  done
+fi
+# stop until confirmed stopped (so the flow switch below actually applies)
+s=0; while [ "$s" -lt 6 ]; do
+  curl -s -m8 -X POST "http://127.0.0.1:${ENC_PORT}/pipeline/stop" -d '{}' -o /dev/null 2>/dev/null
+  sleep 2
+  r=$(curl -s -m4 "http://127.0.0.1:${ENC_PORT}/pipeline/status" 2>/dev/null | python3 -c 'import sys,json;print(json.load(sys.stdin).get("running"))' 2>/dev/null)
+  [ "$r" = "False" ] && break
+  s=$((s+1))
+done
+curl -s -m12 -X POST "http://127.0.0.1:${ENC_PORT}/pipeline/start" -H 'Content-Type: application/json' \
+  -d "{\"domain_path\":\"$DOMAIN\",\"video_flow_uuid\":\"$ENC_SRC\",\"use_mediamtx\":true,\"encoder\":{\"tune\":4,\"speed_preset\":2,\"bitrate\":6000,\"key_int_max\":30}}" \
+  -o /dev/null -w 'encoder start -> %{http_code}\n' | sed 's/^/mxl-pgm-heal: /'
+sleep 3
+# verify the start TOOK (encoder is reading what we asked); one retry if it stuck
+_enc_now=$(curl -s -m4 "http://127.0.0.1:${ENC_PORT}/pipeline/status" 2>/dev/null | python3 -c 'import sys,json;print(json.load(sys.stdin).get("video_flow_uuid",""))' 2>/dev/null)
+if [ "$_enc_now" != "$ENC_SRC" ]; then
+  log "encoder stuck on ${_enc_now:-none} (wanted $ENC_SRC) — forcing stop+start once more"
+  t=0; while [ "$t" -lt 6 ]; do curl -s -m8 -X POST "http://127.0.0.1:${ENC_PORT}/pipeline/stop" -d '{}' -o /dev/null 2>/dev/null; sleep 2
+    [ "$(curl -s -m4 "http://127.0.0.1:${ENC_PORT}/pipeline/status" 2>/dev/null | python3 -c 'import sys,json;print(json.load(sys.stdin).get("running"))' 2>/dev/null)" = "False" ] && break; t=$((t+1)); done
+  curl -s -m12 -X POST "http://127.0.0.1:${ENC_PORT}/pipeline/start" -H 'Content-Type: application/json' \
+    -d "{\"domain_path\":\"$DOMAIN\",\"video_flow_uuid\":\"$ENC_SRC\",\"use_mediamtx\":true,\"encoder\":{\"tune\":4,\"speed_preset\":2,\"bitrate\":6000,\"key_int_max\":30}}" \
+    -o /dev/null -w 'encoder start (retry) -> %{http_code}\n' | sed 's/^/mxl-pgm-heal: /'
+fi
+
+# 5. verify: mxl2webrtc published + selector output is ADVANCING (real moving frames)
+i=0; PUB=0
+while [ "$i" -lt 10 ]; do
+  docker logs --since 40s mxl2webrtc 2>&1 | grep -q 'streaming to MediaMTX' && { PUB=1; break; }
+  sudo docker logs --since 40s mxl2webrtc 2>&1 | grep -q 'streaming to MediaMTX' && { PUB=1; break; }
+  i=$((i+1)); sleep 3
+done
+[ "$PUB" = 1 ] && log "mxl2webrtc PUBLISHED to mediamtx" || log "WARN: mxl2webrtc publish not confirmed in log window"
+h0=$(headidx "$SEL_OUT"); sleep 3; h1=$(headidx "$SEL_OUT")
+if [ -n "$h0" ] && [ -n "$h1" ] && [ "$h1" -gt "$h0" ] 2>/dev/null; then
+  log "OK — selector output ADVANCING ($h0 -> $h1); PGM is live + moving"
+else
+  log "WARN: selector output not clearly advancing ($h0 -> $h1)"
+fi
+log "done"
diff --git a/tools/mxl_thumbs.py b/tools/mxl_thumbs.py
index 0df801a..5936212 100644
--- a/tools/mxl_thumbs.py
+++ b/tools/mxl_thumbs.py
@@ -20,24 +20,88 @@
 from gi.repository import Gst
 
 OUT = '/mxl-domain/thumbs'
-SLOTS = {
-    'cam':     'ca111e00-aaaa-4bbb-8ccc-000000000001',
-    'playout': '2f34c189-64bf-5971-993a-332a28a7a6ee',
-    'pattern': '6b5d8d68-64ce-56f8-bea2-e79b6c282a86',
-    'cam2':    'ca222e00-aaaa-4bbb-8ccc-000000000001',
-    'guest1':  '9e111e00-aaaa-4bbb-8ccc-000000000001',
-    'guest2':  '9e222e00-aaaa-4bbb-8ccc-000000000001',
-    'layout':  '1a900700-aaaa-4bbb-8ccc-000000000001',
-}
+DOMAIN = '/mxl-domain'
+
+# Slots are DISCOVERED at runtime, never hardcoded — the probe renders whatever
+# video sources actually exist on this facility (same discipline as the selector/
+# healers in tools/mxl-flows.sh and the manifest emitter facility_from_discovery.py).
+# Stale baked-in UUIDs are exactly what made the Health panel show dead slots for
+# sources never present and miss the ones that were (reviewer R2). A background
+# supervisor re-scans so sources that appear AFTER startup (a correspondent
+# joining) get a worker, and a role whose flow is recreated with a new UUID has
+# its worker re-attach to the live flow (fixes the ~2min slow re-attach wedge,
+# HW Oct 10). Env MXL_THUMBS_SLOTS ("name=uuid,...") forces an explicit map.
+#
+# NOTE: kept INLINE (not imported) on purpose — run-mxl-thumbs.sh `docker cp`s
+# this ONE file into the container, so it must be self-contained. The discovery
+# helpers are module-level + the run loop is under __main__, so tests can import
+# this module and call discover_slots() without starting any threads.
+
+# PGM outputs, not source tiles — the multiview shows what you can CUT TO, and
+# the selector/keyer are the program bus itself. (The 2-up/PiP composite 'layout'
+# IS a wanted preview tile, so it is not excluded.)
+EXCLUDE_ROLES = {'selector', 'keyer'}
+
+
+def _grouphint_role(flow_def):
+    """The role a flow belongs to, from its grouphint tag; None if untagged.
+    "Guest1:Video" -> "guest1"; the never-interrupt stable flow collapses to its
+    friendly name: "Guest5Stable:Video" -> "guest5" (operator sees "guest5")."""
+    gh = flow_def.get('tags', {}).get('urn:x-nmos:tag:grouphint/v1.0') or []
+    if gh:
+        role = gh[0].split(':', 1)[0].strip().lower()
+        if role.startswith('guest') and role.endswith('stable'):
+            role = role[:-len('stable')]
+        return role
+    return None
+
+
+def discover_slots(domain=DOMAIN, forced=None):
+    """role-name -> video-flow UUID for every selectable VIDEO source present.
+    forced: explicit "name=uuid,name=uuid" (from MXL_THUMBS_SLOTS) bypasses
+    discovery for tests / odd facilities; None reads the env."""
+    if forced is None:
+        forced = os.environ.get('MXL_THUMBS_SLOTS', '').strip()
+    if forced:
+        out = {}
+        for pair in forced.split(','):
+            if '=' in pair:
+                n, u = pair.split('=', 1)
+                out[n.strip()] = u.strip()
+        return out
+    slots = {}
+    try:
+        entries = sorted(os.listdir(domain))
+    except OSError:
+        return slots
+    for e in entries:
+        if not e.endswith('.mxl-flow'):
+            continue
+        try:
+            with open(f'{domain}/{e}/flow_def.json') as f:
+                d = json.load(f)
+        except (OSError, ValueError):
+            continue
+        if not str(d.get('format', '')).endswith(':video'):
+            continue  # thumbnails are a video preview; skip audio/data flows
+        role = _grouphint_role(d) or d.get('id', '')[:8]
+        if role in EXCLUDE_ROLES:
+            continue  # program outputs, not source tiles
+        # For a never-interrupt slot GuestN and GuestNStable both collapse to
+        # "guestN"; the STABLE flow is the one wired to the selector (what goes to
+        # air), so it's the right tile. Stable sorts first by UUID, so first-wins
+        # (setdefault) keeps it and ignores the volatile duplicate.
+        slots.setdefault(role, d.get('id'))
+    return slots
+
+
+SLOTS = discover_slots(DOMAIN)
 
 # per-slot overrides: the layout slot doubles as the live PREVIEW for the
 # 2-up/PiP controls on mxl.html, so it renders faster and larger
 FPS = {'layout': '2/1'}        # default 1/2 (one frame per 2s)
 SIZE = {'layout': (480, 270)}  # default 320x180
 
-Gst.init(None)
-os.makedirs(OUT, exist_ok=True)
-
 # per-slot delivery state for health.json: 'last' = a frame arrived,
 # 'changed' = the frame CONTENT changed (repeat-wedged readers keep 'last'
 # fresh while 'changed' ages — the invisible wedge species, now visible)
@@ -49,6 +113,12 @@ def worker(name, uuid):
     tmp = f'{OUT}/.{name}.tmp'
     while True:
         pipe = None
+        # Re-read the role's CURRENT uuid each rebuild. The supervisor updates
+        # SLOTS when a source reconnects under a new flow id; without this, a
+        # wedged worker would keep re-attaching to the DEAD old uuid forever
+        # (the ~2min "slow re-attach" wedge — HW Oct 10). Now a rebuild lands on
+        # the live flow on the next cycle.
+        uuid = SLOTS.get(name, uuid)
         try:
             fps = FPS.get(name, '1/2')
             w, h = SIZE.get(name, (320, 180))
@@ -126,7 +196,7 @@ def health():
                     mem[k] = int(v.strip().split()[0])
             now = time.time()
             slots = {}
-            for name in SLOTS:
+            for name in list(SLOTS):  # snapshot: the supervisor may add roles concurrently
                 st = STATE.get(name)
                 if st:
                     slots[name] = {'age': round(now - st['last'], 1),
@@ -175,9 +245,44 @@ def health():
         time.sleep(15)
 
 
-for n, u in SLOTS.items():
-    threading.Thread(target=worker, args=(n, u), daemon=True).start()
-threading.Thread(target=health, daemon=True).start()
-print('mxl_thumbs running', flush=True)
-while True:
-    time.sleep(3600)
+_running = set()  # roles that already have a worker thread
+
+
+def _start_worker(name, uuid):
+    _running.add(name)
+    threading.Thread(target=worker, args=(name, uuid), daemon=True).start()
+
+
+def supervisor():
+    """Keep the worker set in sync with the live domain. Picks up sources that
+    appear AFTER startup (a correspondent joining) and refreshes a role's UUID
+    when its flow is recreated with a new id, so the worker re-attaches to the
+    live flow instead of the dead one. Env MXL_THUMBS_SLOTS disables discovery
+    (fixed map) — then there's nothing to re-scan."""
+    if os.environ.get('MXL_THUMBS_SLOTS', '').strip():
+        return
+    while True:
+        time.sleep(10)
+        try:
+            found = discover_slots(DOMAIN)
+        except Exception as e:
+            print(f'supervisor: {e}', flush=True)
+            continue
+        for name, uuid in found.items():
+            if SLOTS.get(name) != uuid:
+                SLOTS[name] = uuid  # worker re-reads this on its next rebuild
+            if name not in _running:
+                print(f'supervisor: new source {name} -> starting worker', flush=True)
+                _start_worker(name, uuid)
+
+
+if __name__ == '__main__':
+    Gst.init(None)
+    os.makedirs(OUT, exist_ok=True)
+    for n, u in SLOTS.items():
+        _start_worker(n, u)
+    threading.Thread(target=health, daemon=True).start()
+    threading.Thread(target=supervisor, daemon=True).start()
+    print(f'mxl_thumbs running ({len(SLOTS)} slots discovered)', flush=True)
+    while True:
+        time.sleep(3600)

From 84334f8ba9508c9a68bf64c26eca0d1bde75a45f Mon Sep 17 00:00:00 2001
From: Guy Cochran <guycochran@gmail.com>
Date: Mon, 12 Oct 2026 23:28:13 +0000
Subject: [PATCH 2/3] feat(selfheal): re-sync a selector wedged on a grain 'too
 late' error (SwitchFrame-inspired)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The CBC input-selector parks running:true + 'read source grain N: Out of range - too late'
when its active source delivers a grain outside the readable window (jittery/relayed/
framerate-mismatched — HW Oct: PTZ over the Azure relay wedged it on every cut, all further
cuts 415-reject, PGM frozen). CBC's selector does not self-recover.

Borrowing the strategy from thiagopnts/switchframe's Go MXL reader
(server/mxl/reader.go: 'On too-late errors, re-sync to the current write head instead of
dying'): we can't patch CBC's Rust, so we apply the same idea at our layer — on a too-late
error, re-cut the selector to a VERIFIED-ADVANCING slot, which forces the router to re-grab
from the current write head and clears the wedge. 20s cooldown; role/discovery based (uses
tools/mxl-flows.sh), no hardcoded UUIDs.

Verified: wedge-string detection matches the exact HW error; ignores non-wedge states.
NOT yet HW-tested (box unreachable at commit time — ssh tunnel down); deploy + verify next.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01GwSKEZZz4yEc2CGq5hcTmT
---
 tools/mxl-selfheal.sh | 66 +++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 64 insertions(+), 2 deletions(-)

diff --git a/tools/mxl-selfheal.sh b/tools/mxl-selfheal.sh
index 3fbb156..c1e2c1b 100755
--- a/tools/mxl-selfheal.sh
+++ b/tools/mxl-selfheal.sh
@@ -12,6 +12,11 @@
 #   2. RELAY WAITING FOR AUDIO — mxl2webrtc defaults to mode:video+audio and waits
 #      forever for an audio_pgm flow that a bare quickstart never starts, so the
 #      WebRTC program monitor shows "stream not found" even though video exists.
+#   3. SELECTOR WEDGED ON A GRAIN-TIMING ERROR — the selector reports running:true but
+#      parks "read source grain N: Out of range - too late" from a jittery/relayed/
+#      framerate-mismatched source, and every subsequent cut 415-rejects (PGM frozen).
+#      We re-cut to a live source so the selector re-grabs from the current write head —
+#      the "re-sync instead of die" strategy SwitchFrame's MXL reader uses.
 #
 # This watcher polls the LOCAL control ports (no external backend) and fixes
 # exactly those two, idempotently. Safe to run as a plain loop, a systemd unit, or
@@ -38,10 +43,17 @@ KEYER_FLOW=""   # discovered lazily
 # Use `docker` directly if we can (quickstart runs as root), else `sudo docker`.
 if docker ps >/dev/null 2>&1; then DOCKER="docker"; else DOCKER="sudo docker"; fi
 
+# Runtime flow-UUID discovery (flow_by_role / advancing by role) — portable, no hardcoded ids.
+_here=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
+. "$_here/mxl-flows.sh" 2>/dev/null || true
+
 log(){ printf '%s mxl-selfheal: %s\n' "$(date -u +%H:%M:%SZ)" "$*"; }
 _get(){ curl -s -m 5 "http://127.0.0.1:$1$2" 2>/dev/null; }
 _post(){ curl -s -m 25 -X POST -H 'Content-Type: application/json' -d "$3" "http://127.0.0.1:$1$2" 2>/dev/null; }
 _json(){ python3 -c "import sys,json;d=json.load(sys.stdin);print(d.get('$1',''))" 2>/dev/null; }
+# head index of a flow uuid (empty if unreadable), and a 1s advancing check.
+headidx(){ $DOCKER exec "$SEL_CTR" sh -c "/opt/mxl/tools/mxl-info/mxl-info -d $DOMAIN -f $1 2>/dev/null" 2>/dev/null | grep -i 'Head index' | grep -oE '[0-9]+' | head -1; }
+advancing(){ _a=$(headidx "$1"); sleep 1; _b=$(headidx "$1"); [ -n "$_a" ] && [ -n "$_b" ] && [ "$_b" -gt "$_a" ] 2>/dev/null; }
 
 # UUIDs that actually exist in the domain right now (one per line).
 existing_flows(){
@@ -124,10 +136,60 @@ heal_relay(){
   log "relay restarted video-only"
 }
 
-pass(){ heal_selector; heal_relay; }
+# ── heal 3: selector WEDGED on a grain-timing error → re-sync (don't die) ───────
+# The CBC input-selector reports `running:true` but parks an error like
+#   "read source grain 53749…: Out of range - too late"
+# when its active source delivers a grain outside the readable window (a jittery /
+# relayed / framerate-mismatched source — HW Oct: PTZ over the Azure relay wedged it
+# on every cut; all further cuts 415-reject and PGM freezes). The selector does NOT
+# self-recover — but re-cutting the active input makes it re-grab from the CURRENT
+# write head, exactly the "re-sync to head instead of dying" strategy SwitchFrame's
+# MXL reader uses (server/mxl/reader.go: "On 'too late' errors, re-sync to the
+# current write head instead of dying"). We can't patch CBC's Rust, so we apply the
+# same idea at our layer: on a too-late error, re-cut to a VERIFIED-ADVANCING slot
+# (prefer the one it's already on if that's live, else the first advancing source).
+_wedge_last_heal=0
+heal_selector_wedge(){
+  local st err active uuids
+  st=$(_get 9604 /pipeline/status); [ -n "$st" ] || return 0
+  err=$(printf '%s' "$st" | _json error)
+  # only the grain-timing wedge — leave other errors to heal_selector / operators
+  case "$err" in
+    *"Out of range"*|*"too late"*|*"too early"*) : ;;
+    *) return 0 ;;
+  esac
+  local now; now=$(date +%s)
+  [ $((now - _wedge_last_heal)) -lt 20 ] && return 0   # brief cooldown; one re-sync per ~20s
+  # the slot list + current active, to choose a live re-cut target
+  uuids=$(printf '%s' "$st" | python3 -c "import sys,json;print(' '.join(json.load(sys.stdin).get('input_flow_uuids',[])))" 2>/dev/null)
+  active=$(printf '%s' "$st" | _json active_input)
+  log "SELECTOR WEDGED (error: $err) — re-syncing (re-cut to a live source)"
+  # pick a target: first slot whose flow is ADVANCING. Walk from slot 0 so the
+  # always-on Pattern/Playout are the guaranteed fallback; prefer a non-active live
+  # slot so we actually move off the stuck source if it's the dead one.
+  local idx=0 target="" i=0
+  for u in $uuids; do
+    if advancing "$u"; then target=$i; [ "$i" != "$active" ] && break; fi
+    i=$((i+1))
+  done
+  [ -n "$target" ] || target=0
+  # re-cut: this forces the selector router to re-attach the slot from the live head.
+  _post 9604 /pipeline/active-input "{\"slot\":$target}" >/dev/null
+  _wedge_last_heal=$now
+  # verify it cleared
+  sleep 1
+  local e2; e2=$(_get 9604 /pipeline/status | _json error)
+  case "$e2" in
+    *"Out of range"*|*"too late"*|*"too early"*)
+      log "still wedged after re-cut to slot $target — leaving for next pass" ;;
+    *) log "selector re-synced (cut to slot $target, error cleared)" ;;
+  esac
+}
+
+pass(){ heal_selector; heal_selector_wedge; heal_relay; }
 
 if [ "${1:-}" = "--watch" ]; then
-  log "watching (every ${INTERVAL}s) — selector + relay"
+  log "watching (every ${INTERVAL}s) — selector down + selector wedge (too-late re-sync) + relay"
   while :; do pass; sleep "$INTERVAL"; done
 else
   pass

From 2a6267dceb696019705df6226e3d810ab802654c Mon Sep 17 00:00:00 2001
From: Guy Cochran <guycochran@gmail.com>
Date: Mon, 12 Oct 2026 23:29:07 +0000
Subject: [PATCH 3/3] =?UTF-8?q?docs:=20comparison=20vs=20thiagopnts/switch?=
 =?UTF-8?q?frame=20=E2=80=94=20same=20north=20star=20(MXL+server-switch+br?=
 =?UTF-8?q?owser),=20borrowable=20ideas=20ranked?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

SwitchFrame is a mature Go/Svelte engine on the same MXL substrate. Documents where it's ahead
(re-sync reader, SIMD v210, atomic graph swap, GPU kernels), the one lesson we already applied
(re-sync-instead-of-die for the selector too-late wedge), and a ranked list of ideas to borrow
next. Strategic read: borrow their algorithms, keep our adoptable/portable/commodity-HW edge.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01GwSKEZZz4yEc2CGq5hcTmT
---
 docs/COMPARISON-switchframe.md | 64 ++++++++++++++++++++++++++++++++++
 1 file changed, 64 insertions(+)
 create mode 100644 docs/COMPARISON-switchframe.md

diff --git a/docs/COMPARISON-switchframe.md b/docs/COMPARISON-switchframe.md
new file mode 100644
index 0000000..c8de9ee
--- /dev/null
+++ b/docs/COMPARISON-switchframe.md
@@ -0,0 +1,64 @@
+# Comparison: our MXL switcher vs. thiagopnts/switchframe
+
+A reference comparison (Oct 2026). **SwitchFrame** (github.com/thiagopnts/switchframe) is an
+independent project with the **same north star as ours**: a browser-controlled, server-side
+live production switcher built on the EBU **MXL** shared-memory transport. It is far more
+built-out than our prototype, so it is a valuable **source of ideas** — not a competitor to
+adopt (2 stars / 0 forks: it's one person's deep solo engine, not a standard).
+
+## What SwitchFrame is
+- **Go 1.25 engine + Svelte 5 control UI**, single binary, WebTransport/QUIC (HTTP-3) control.
+- Sources decode to **YUV420 server-side**; all switching/mixing/keying/compositing happens on
+  the server; browsers are pure control surfaces (identical philosophy to our Core/Skin split).
+- **Same substrate as us:** MXL (V210 video / float32 audio shared memory), SRT in/out
+  (listener+caller), MPEG-TS recording. Plus MoQ ingest, ST 2110, multi-dest SRT.
+- Scale: ~3,000 commits, 291 REST endpoints, 59 GPU/SIMD kernels (Metal + CUDA), AI captions
+  (whisper.cpp → CEA-608), AI background removal (TensorRT RVM), 3D-LUT color grade, 8-layer
+  DSK, instant replay, multi-operator roles + Opus comms.
+
+## Where it is ahead of us, and why
+| Area | SwitchFrame | Us (this repo) |
+|---|---|---|
+| Core engine | one Go binary, lock-free hot path (<1µs/frame), **atomic graph swap** (zero-drop reconfig) | orchestrated containers: CBC Rust **input-selector** + Python ingests + gst pipelines |
+| **"grain too late" wedge** | reader **re-syncs to `head-2` and continues** (transient, not fatal) | CBC selector **wedges/dies** on it → our PTZ-over-relay bug |
+| GPU | Metal/CUDA, 59 kernels, actually used | GTX 970 idle; v210 has no HW path; CEF forces SwiftShader (upstream image) |
+| v210 pack/unpack | **SIMD** (`v210_10bit_kernels_{amd64,arm64}.s`) | gst `videoconvert` in software (~1 core/active fanout before our tsparse fix) |
+| Timing | dedicated `clock` + `stmap` modules, independent monotonic PTS per stream | hand-tuned restamp in `contribution_core.py`, fragile under relay jitter |
+
+## The single most transferable lesson (applied)
+SwitchFrame's MXL reader (`server/mxl/reader.go`):
+> *"On 'too late' errors, re-sync to the current write head instead of dying."*
+> *"Timeout and too-early are transient — don't count as fatal errors."*
+It jumps to `head-2` and continues; only ~50 **consecutive** fatals stop it.
+
+This is the exact cure for our recurring **PTZ selector wedge** (`read source grain N: Out of
+range - too late`). We hit it because our stack uses CBC's Rust input-selector, which **dies**
+on that error where SwitchFrame's own reader **re-syncs**. We can't patch CBC's Rust, so we
+applied the same principle at the layer we control: **`mxl-selfheal` now detects the wedge and
+re-cuts to a verified-advancing slot**, forcing the router to re-grab from the current head
+(commit `feat(selfheal): re-sync a selector wedged on a grain 'too late' error`). That's a
+watchdog-level re-sync vs. SwitchFrame's in-reader re-sync — same idea, our architecture.
+
+## Ideas worth borrowing next (ranked)
+1. **Re-sync reader (DONE at watchdog layer).** The "transient, not fatal; re-grab head" rule
+   is the biggest reliability win and is now in selfheal. If we ever own the selector, bake it
+   into the reader itself (the proper place) instead of healing after the fact.
+2. **Role-based discovery (already aligned).** Their `discovery.go` ≈ our new `tools/mxl-flows.sh`
+   (resolve flows by stable ROLE, not baked UUIDs). We arrived at the same design independently.
+3. **SIMD v210** (`v210_10bit_kernels`). Our fanout/conform is the CPU hog (profile: ~1 core
+   each). A SIMD v210 pack/unpack (or moving conform to a Go/Rust helper) would reclaim cores
+   that gst `videoconvert` burns — cheaper than the GPU path (v210 has no GPU HW path anyway).
+4. **Atomic graph swap** for zero-drop reconfig. Our re-wires interrupt PGM (why we built
+   never-interrupt + stable-flow pre-wiring). Their atomic swap is the clean general answer; our
+   stable-flow approach is the pragmatic version on a selector we don't own.
+5. **GPU, selectively** (deferred). Their 59 kernels need a Go/CUDA engine. For us the only
+   realistic near-term GPU win is NVENC on the mxl2webrtc x264 encode — a container rebuild,
+   lower priority than #3.
+
+## Strategic read
+SwitchFrame validates our architecture (MXL + server-side switch + browser control is the right
+shape) and shows the mature endpoint. But it's a from-scratch Go engine; our value is being the
+**adoptable, MXL-native, container-orchestrated** path that runs on commodity hardware a facility
+already has. Borrow their proven algorithms (re-sync reader, SIMD v210, atomic swap); don't chase
+feature parity with a 3,000-commit solo engine. Keep our edge: quickstart-able, portable, honest
+about what runs on a $400 box.