From 26f4623e959e346232814ba06ee70464cd090e8b Mon Sep 17 00:00:00 2001 From: Guy Cochran Date: Fri, 9 Oct 2026 01:02:05 +0000 Subject: [PATCH 01/37] =?UTF-8?q?feat(health):=20read-only=20Health=20Skin?= =?UTF-8?q?=20=E2=80=94=20grain=20flow=20(+unique-fps),=20system,=20pipeli?= =?UTF-8?q?ne=20writers?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Second Skin over the same Core, deliberately separate from the control Skin: - GET /api/mxl/health + GET /health (read-only; no mutation path) - backend/health-info.js: pure aggregation — classifies flows ok/slow/frozen/dead. 'frozen' = grains tick at rate but unique-fps collapsed (repeat-wedge detection). - Reuses tools/grain_probe.py's grains.json snapshot (already load-guarded) + /proc + the 3 pipeline/status endpoints; 2s server cache so N viewers = one scrape. No docker exec, no probe spawning, live PGM path untouched. - web/health.html: air-gapped (self-hosted fonts), 3s poll, Skin-styled. - Tests: 14 node unit tests + 5 python (air-gap, read-only, separation-from-Core). Full suite green (50 node / 139 py). Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01GwSKEZZz4yEc2CGq5hcTmT --- backend/health-info.js | 160 +++++++++++++++++++++++++ backend/local-server.js | 64 ++++++++++ docs/HEALTH-SKIN.md | 60 ++++++++++ tests/js/health-info.test.js | 119 +++++++++++++++++++ tests/test_local_ui.py | 50 ++++++++ web/health.html | 218 +++++++++++++++++++++++++++++++++++ 6 files changed, 671 insertions(+) create mode 100644 backend/health-info.js create mode 100644 docs/HEALTH-SKIN.md create mode 100644 tests/js/health-info.test.js create mode 100644 web/health.html diff --git a/backend/health-info.js b/backend/health-info.js new file mode 100644 index 0000000..ef86dd7 --- /dev/null +++ b/backend/health-info.js @@ -0,0 +1,160 @@ +// backend/health-info.js — read-only health aggregation for the Health Skin. +// +// This is deliberately SEPARATE from the control plane (mxl-routes.js). The Core +// Skin (web/local.html) is what a TD touches on air — cut/arm/take. Grain +// internals, CPU/mem and pipeline-writer state belong on their own read-only +// page (web/health.html, served at /health). Nothing here mutates the facility. +// +// Cost note: the expensive part — measuring per-flow grain rate AND unique-frames +// /s (to catch repeat-wedges where the head advances but the picture is frozen) — +// is already done by tools/grain_probe.py, which writes a rolling-window snapshot +// to /grains.json every 3s and has hard load guards (pauses >26, exits +// >29). We only READ that file here, so N health viewers add ~one file read each, +// never a container spawn. The live PGM/encode path is never touched. + +'use strict'; + +const fs = require('fs'); + +// ── System: CPU / mem / load from /proc (Linux). Pure reads, microseconds. ── +// CPU % needs two samples of /proc/stat; the caller holds the previous snapshot +// so a single scrape is cheap and the delta is computed across polls. +function readProcStat(readFile = fs.readFileSync) { + // Returns {total, idle} jiffies for the aggregate "cpu " line, or null. + try { + const line = String(readFile('/proc/stat', 'utf8')).split('\n') + .find((l) => l.startsWith('cpu ')); + if (!line) return null; + const n = line.trim().split(/\s+/).slice(1).map(Number); + // user nice system idle iowait irq softirq steal ... + const idle = (n[3] || 0) + (n[4] || 0); // idle + iowait + const total = n.reduce((a, b) => a + (Number.isFinite(b) ? b : 0), 0); + return { total, idle }; + } catch { return null; } +} + +function cpuPercent(prev, cur) { + if (!prev || !cur) return null; + const dt = cur.total - prev.total; + const di = cur.idle - prev.idle; + if (dt <= 0) return null; + return Math.max(0, Math.min(100, Math.round((1 - di / dt) * 100))); +} + +function readMem(readFile = fs.readFileSync) { + try { + const info = {}; + String(readFile('/proc/meminfo', 'utf8')).split('\n').forEach((l) => { + const m = l.match(/^(\w+):\s+(\d+)/); + if (m) info[m[1]] = Number(m[2]); // kB + }); + const totalKb = info.MemTotal || 0; + const availKb = info.MemAvailable != null ? info.MemAvailable : info.MemFree || 0; + if (!totalKb) return null; + const usedKb = totalKb - availKb; + return { + total_mb: Math.round(totalKb / 1024), + used_mb: Math.round(usedKb / 1024), + used_pct: Math.round((usedKb / totalKb) * 100), + }; + } catch { return null; } +} + +function readLoad(readFile = fs.readFileSync) { + try { + const p = String(readFile('/proc/loadavg', 'utf8')).trim().split(/\s+/); + return { load1: Number(p[0]), load5: Number(p[1]), load15: Number(p[2]) }; + } catch { return null; } +} + +function cpuCount(readFile = fs.readFileSync) { + try { + return String(readFile('/proc/stat', 'utf8')).split('\n') + .filter((l) => /^cpu\d+ /.test(l)).length || null; + } catch { return null; } +} + +// ── Grain flow: read grain_probe's snapshot (already on disk, atomic-renamed). ─ +// Shape written by tools/grain_probe.py: {ts, flows: {name: {bps, ufps}|null}}. +// We classify each flow so the UI can paint green/amber/red without logic: +// dead — null (probe read nothing in the window) +// frozen — bps healthy but ufps collapsed (repeat-wedge: head moves, picture +// doesn't). This is the exact keyer-bypass symptom we hit live. +// ok — bps and ufps both near the expected rate +// expectedRate defaults to 30 (the facility is 1080p30). stale = snapshot older +// than staleAfterS (probe died / not running). +function classifyGrains(snapshot, { expectedRate = 30, nowS, staleAfterS = 12 } = {}) { + if (!snapshot || typeof snapshot !== 'object' || !snapshot.flows) { + return { stale: true, age_s: null, flows: {} }; + } + const now = nowS != null ? nowS : Date.now() / 1000; + const ageS = snapshot.ts != null ? Math.max(0, now - snapshot.ts) : null; + const stale = ageS == null ? true : ageS > staleAfterS; + const flows = {}; + for (const [name, v] of Object.entries(snapshot.flows)) { + if (!v || v.bps == null) { + flows[name] = { state: 'dead', bps: 0, ufps: 0 }; + continue; + } + const bps = v.bps; + const ufps = v.ufps != null ? v.ufps : null; + // frozen: grains tick at ~rate but unique frames are a fraction of them. + // Use 60% of expected as the unique-fps floor for "moving". + let state = 'ok'; + if (bps < expectedRate * 0.5) state = 'slow'; + if (ufps != null && ufps < expectedRate * 0.6 && bps >= expectedRate * 0.5) state = 'frozen'; + flows[name] = { state, bps, ufps }; + } + return { stale, age_s: ageS != null ? Math.round(ageS * 10) / 10 : null, flows }; +} + +function readGrainsSnapshot(grainsPath, readFile = fs.readFileSync) { + try { return JSON.parse(String(readFile(grainsPath, 'utf8'))); } + catch { return null; } +} + +// ── Pipeline writers: normalize the three pipeline/status payloads for the UI. ─ +// Pure — the caller fetches the JSON (or passes nulls when a port is unreachable) +// and we shape it. No network here, so it's unit-testable. +function shapePipelines({ selector, keyer, encoder } = {}) { + const sel = selector || {}; + const key = keyer || {}; + const enc = encoder || {}; + const fmt = (f) => (f && f.format + ? { + w: f.format.frame_width, h: f.format.frame_height, + rate: f.format.grain_rate + ? Math.round((f.format.grain_rate.numerator || 0) / (f.format.grain_rate.denominator || 1)) + : null, + } + : null); + return { + selector: selector ? { + running: !!sel.running, error: sel.error || null, + active_input: sel.active_input != null ? sel.active_input : null, + inputs: Array.isArray(sel.input_flow_uuids) ? sel.input_flow_uuids.length : null, + format: fmt(sel), + } : { running: false, unreachable: true }, + keyer: keyer ? { + running: !!key.running, error: key.error || null, + key_on: !!key.key_on, mode: key.mode || null, + format: fmt(key), + } : { running: false, unreachable: true }, + encoder: encoder ? { + running: !!enc.running, error: enc.error || null, + viewers: enc.viewers != null ? enc.viewers : null, + bitrate_kbps: enc.encoder && enc.encoder.bitrate != null ? enc.encoder.bitrate : null, + tune: enc.encoder ? enc.encoder.tune : null, + } : { running: false, unreachable: true }, + }; +} + +// ── Compose the whole payload (pure; takes already-read inputs). ────────────── +function composeHealth({ system, grains, pipelines, ts }) { + return { ts, system: system || null, grains: grains || { stale: true, flows: {} }, pipelines: pipelines || {} }; +} + +module.exports = { + readProcStat, cpuPercent, readMem, readLoad, cpuCount, + classifyGrains, readGrainsSnapshot, shapePipelines, composeHealth, +}; diff --git a/backend/local-server.js b/backend/local-server.js index 5dea2c4..1c786b4 100644 --- a/backend/local-server.js +++ b/backend/local-server.js @@ -137,6 +137,70 @@ app.get('/api/mxl/ingest/qr/:slot.png', (req, res) => { }); }); +// ── Health Skin (read-only) ────────────────────────────────────────────────── +// A SEPARATE page from the control Skin: System (CPU/mem/load), Grain flow (bps + +// unique-fps per flow, catching repeat-wedges), per-source fps and pipeline-writer +// state. Nothing here mutates the facility — the Core stays lean and on-air-focused. +// +// The grain measurement is already done by tools/grain_probe.py (writes +// /grains.json every 3s, with its own load guards). We only read that file +// + /proc + the three pipeline/status endpoints, and cache the aggregate for a +// couple seconds so N open Health tabs collapse to one scrape. +const health = require('./health-info'); +const GRAINS_PATH = process.env.MXL_GRAINS_PATH || path.join(THUMBS_DIR, 'grains.json'); +const HEALTH_RATE = parseInt(process.env.MXL_GRAIN_RATE || '30', 10); +const HEALTH_TTL_MS = parseInt(process.env.MXL_HEALTH_TTL_MS || '2000', 10); +// Resolve the MXL control-API base the same way mxl-routes does, so the pipeline +// status probes hit the same VM the cuts do. +const HEALTH_MXL_VM = process.env.MXL_VM_URL + || (process.env.MXL_VM_FROM_MANIFEST === '1' && facility && facility.network && `http://${facility.network.mxl_vm}`) + || 'http://127.0.0.1'; + +function fetchJson(port, apiPath, timeoutMs = 3000) { + return fetch(`${HEALTH_MXL_VM}:${port}${apiPath}`, { signal: AbortSignal.timeout(timeoutMs) }) + .then((r) => (r.ok ? r.json() : null)) + .catch(() => null); // unreachable port -> null; shapePipelines marks it so +} + +let _prevCpu = health.readProcStat(); +let _healthCache = { at: 0, payload: null }; + +async function buildHealth() { + const [selector, keyer, encoder] = await Promise.all([ + fetchJson(9604, '/pipeline/status'), + fetchJson(9605, '/pipeline/status'), + fetchJson(9601, '/pipeline/status'), + ]); + const curCpu = health.readProcStat(); + const system = { + cpu_pct: health.cpuPercent(_prevCpu, curCpu), + cpus: health.cpuCount(), + mem: health.readMem(), + load: health.readLoad(), + }; + _prevCpu = curCpu; + const grains = health.classifyGrains( + health.readGrainsSnapshot(GRAINS_PATH), { expectedRate: HEALTH_RATE }); + const pipelines = health.shapePipelines({ selector, keyer, encoder }); + return health.composeHealth({ ts: Date.now() / 1000, system, grains, pipelines }); +} + +app.get('/api/mxl/health', async (req, res) => { + try { + const now = Date.now(); + if (!_healthCache.payload || now - _healthCache.at > HEALTH_TTL_MS) { + _healthCache = { at: now, payload: await buildHealth() }; + } + res.set('Cache-Control', 'no-store'); + res.json(_healthCache.payload); + } catch (e) { + res.status(500).json({ error: String((e && e.message) || e) }); + } +}); + +// The Health Skin page (read-only). Separate URL from the control Skin. +app.get('/health', (req, res) => res.sendFile(path.join(WEB_DIR, 'health.html'))); + // Serve the operator UI + its static assets. app.get('/', (req, res) => res.sendFile(path.join(WEB_DIR, 'local.html'))); app.use(express.static(WEB_DIR)); diff --git a/docs/HEALTH-SKIN.md b/docs/HEALTH-SKIN.md new file mode 100644 index 0000000..3d3074e --- /dev/null +++ b/docs/HEALTH-SKIN.md @@ -0,0 +1,60 @@ +# Health Skin — read-only facility monitor + +A second **Skin** over the same Core: a read-only page that shows what the switcher +is doing under the hood — grain flow, system load, and pipeline-writer state. It is +deliberately **separate from the control Skin** (`web/local.html`). A TD cutting a +show should see sources and a cut button, not grain internals; an engineer watching +health should not be able to touch the program bus. Core vs Skin, applied twice. + +## What it shows + +- **System** — CPU %, memory, 1/5/15-min load, CPU count, and the live WebRTC + viewer count (reported for free by the encoder). +- **Grain flow** — per flow: grains/s **and unique-frames/s**. The unique-fps column + is the one that matters: a flow can tick 30 grains/s while the *picture is frozen* + (a repeat-wedge — head advances, frame doesn't). When that happens unique-fps + collapses and the row goes amber (`frozen`). `dead` = probe read nothing; `slow` = + rate below half; `ok` = both near rate. +- **Pipeline writers** — selector / keyer / encoder: running state, errors, active + input, format, key-on, viewers, bitrate. + +## Where the data comes from (and why it's cheap) + +The expensive part — measuring grain rate *and* unique frames — is already done by +`tools/grain_probe.py`, which runs long-lived readers (one per flow) and writes a +rolling-window snapshot to `/grains.json` every 3 s. It has hard load guards +(pauses above load 26, exits above 29), so the probe can never be the thing that tips +the box over. + +The Core does **not** spawn probes or `docker exec` anything. `GET /api/mxl/health`: + +1. reads `grains.json` (already on disk), +2. reads `/proc` for CPU/mem/load, +3. fetches the three `:/pipeline/status` endpoints (same VM the cuts use), + +then caches the aggregate for ~2 s (`MXL_HEALTH_TTL_MS`) so N open Health tabs +collapse to a single scrape. The live PGM/encode path is never touched. + +## Running it + +- Serve: the Core (`backend/local-server.js`) serves the page at **`/health`** and the + data at **`/api/mxl/health`** — no extra process. +- Requires the grain probe to be running for the Grain-flow section to populate + (`tools/grain_probe.py`, started by the facility bring-up / its runner). If the probe + isn't running, the page still shows System + Pipeline writers and marks grains stale. + +## Config + +| env | default | meaning | +|-----|---------|---------| +| `MXL_GRAINS_PATH` | `/grains.json` | where grain_probe writes its snapshot | +| `MXL_GRAIN_RATE` | `30` | expected grain rate (1080p30) — classifies ok/slow/frozen | +| `MXL_HEALTH_TTL_MS` | `2000` | server-side cache TTL so viewers share one scrape | + +## Files + +- `backend/health-info.js` — pure aggregation/classification (unit-tested, no network) +- `backend/local-server.js` — `GET /api/mxl/health` + `GET /health` (read-only) +- `web/health.html` — the page (air-gapped: self-hosted fonts, no external hosts) +- `tests/js/health-info.test.js` — 14 unit tests (frozen/dead/slow, system, pipelines) +- `tests/test_local_ui.py` — air-gap + read-only + separation-from-Core checks diff --git a/tests/js/health-info.test.js b/tests/js/health-info.test.js new file mode 100644 index 0000000..a8fe9c3 --- /dev/null +++ b/tests/js/health-info.test.js @@ -0,0 +1,119 @@ +// Unit tests for backend/health-info.js — the pure (no-network, no-express) health +// aggregation used by the read-only Health Skin. Run with `node --test`. +// +// The marquee case: a flow can tick grains at full rate while the picture is +// FROZEN (a repeat-wedge). classifyGrains must flag that as 'frozen' from the +// unique-fps collapse — the exact failure we hit live with the keyer. +const test = require('node:test'); +const assert = require('node:assert'); +const path = require('node:path'); + +const h = require(path.join(__dirname, '..', '..', 'backend', 'health-info.js')); + +// ── classifyGrains ──────────────────────────────────────────────────────────── +test('grains: healthy flow -> ok', () => { + const now = 1000; + const snap = { ts: 999.5, flows: { cam: { bps: 30, ufps: 29.7 } } }; + const out = h.classifyGrains(snap, { expectedRate: 30, nowS: now }); + assert.equal(out.stale, false); + assert.equal(out.flows.cam.state, 'ok'); +}); + +test('grains: FROZEN flow (full rate, unique collapsed) -> frozen', () => { + // 30 grains/s but only ~3 unique frames/s = repeat-wedge. + const snap = { ts: 1000, flows: { guest1: { bps: 30, ufps: 3 } } }; + const out = h.classifyGrains(snap, { expectedRate: 30, nowS: 1000 }); + assert.equal(out.flows.guest1.state, 'frozen', + 'a flow ticking at rate with collapsed unique-fps must read as frozen'); +}); + +test('grains: dead flow (null) -> dead', () => { + const snap = { ts: 1000, flows: { playout: null } }; + const out = h.classifyGrains(snap, { expectedRate: 30, nowS: 1000 }); + assert.equal(out.flows.playout.state, 'dead'); + assert.equal(out.flows.playout.bps, 0); +}); + +test('grains: slow flow (bps below half rate) -> slow', () => { + const snap = { ts: 1000, flows: { cam2: { bps: 10, ufps: 10 } } }; + const out = h.classifyGrains(snap, { expectedRate: 30, nowS: 1000 }); + assert.equal(out.flows.cam2.state, 'slow'); +}); + +test('grains: stale snapshot flagged by age', () => { + const snap = { ts: 900, flows: { cam: { bps: 30, ufps: 30 } } }; + const out = h.classifyGrains(snap, { expectedRate: 30, nowS: 1000, staleAfterS: 12 }); + assert.equal(out.stale, true, 'a 100s-old snapshot is stale'); + assert.equal(out.age_s, 100); +}); + +test('grains: missing/garbage snapshot -> stale, empty', () => { + const out = h.classifyGrains(null, { nowS: 1000 }); + assert.equal(out.stale, true); + assert.deepEqual(out.flows, {}); +}); + +// ── system parsing ──────────────────────────────────────────────────────────── +test('cpuPercent: computes busy fraction from two /proc/stat samples', () => { + const prev = { total: 1000, idle: 800 }; + const cur = { total: 1100, idle: 850 }; // +100 total, +50 idle -> 50% busy + assert.equal(h.cpuPercent(prev, cur), 50); +}); + +test('cpuPercent: null when no delta / missing sample', () => { + assert.equal(h.cpuPercent(null, { total: 1, idle: 0 }), null); + assert.equal(h.cpuPercent({ total: 10, idle: 5 }, { total: 10, idle: 5 }), null); +}); + +test('readMem: parses MemTotal/MemAvailable into used %', () => { + const fake = () => 'MemTotal: 8000000 kB\nMemAvailable: 2000000 kB\nMemFree: 1000000 kB\n'; + const m = h.readMem(fake); + assert.equal(m.total_mb, Math.round(8000000 / 1024)); + assert.equal(m.used_pct, 75); // (8000000-2000000)/8000000 +}); + +test('readProcStat: aggregate cpu line -> total/idle', () => { + const fake = () => 'cpu 100 0 100 700 100 0 0 0\ncpu0 1 2 3 4\n'; + const s = h.readProcStat(fake); + // fields: user100 nice0 system100 idle700 iowait100 irq0 softirq0 steal0 + // idle = idle(700)+iowait(100) = 800; total = sum of all = 1000 + assert.equal(s.idle, 800); + assert.equal(s.total, 1000); +}); + +test('cpuCount: counts per-core lines', () => { + const fake = () => 'cpu 1 2\ncpu0 1\ncpu1 1\ncpu2 1\n'; + assert.equal(h.cpuCount(fake), 3); +}); + +// ── pipeline shaping ────────────────────────────────────────────────────────── +test('shapePipelines: normalizes selector/keyer/encoder; marks unreachable', () => { + const out = h.shapePipelines({ + selector: { running: true, active_input: 2, input_flow_uuids: ['a', 'b', 'c'], + format: { frame_width: 1920, frame_height: 1080, grain_rate: { numerator: 30, denominator: 1 } } }, + keyer: { running: true, key_on: false, mode: 'key' }, + encoder: null, // unreachable + }); + assert.equal(out.selector.active_input, 2); + assert.equal(out.selector.inputs, 3); + assert.deepEqual(out.selector.format, { w: 1920, h: 1080, rate: 30 }); + assert.equal(out.keyer.key_on, false); + assert.equal(out.encoder.unreachable, true); + assert.equal(out.encoder.running, false); +}); + +test('shapePipelines: encoder viewers + bitrate surfaced', () => { + const out = h.shapePipelines({ + encoder: { running: true, viewers: 4, encoder: { bitrate: 6000, tune: 4 } }, + }); + assert.equal(out.encoder.viewers, 4); + assert.equal(out.encoder.bitrate_kbps, 6000); +}); + +// ── compose ─────────────────────────────────────────────────────────────────── +test('composeHealth: assembles a stable top-level shape', () => { + const out = h.composeHealth({ ts: 123, system: { cpu_pct: 10 }, grains: { stale: false, flows: {} }, pipelines: {} }); + assert.equal(out.ts, 123); + assert.equal(out.system.cpu_pct, 10); + assert.ok('grains' in out && 'pipelines' in out); +}); diff --git a/tests/test_local_ui.py b/tests/test_local_ui.py index c674164..f603da6 100644 --- a/tests/test_local_ui.py +++ b/tests/test_local_ui.py @@ -8,8 +8,10 @@ REPO = Path(__file__).resolve().parent.parent HTML = REPO / "web" / "local.html" +HEALTH_HTML = REPO / "web" / "health.html" SERVER = REPO / "backend" / "local-server.js" ROUTES = REPO / "backend" / "mxl-routes.js" +HEALTH_INFO = REPO / "backend" / "health-info.js" PKG = REPO / "backend" / "package.json" FONTS = REPO / "web" / "fonts" @@ -110,3 +112,51 @@ def test_package_pins_express(): import json pkg = json.loads(PKG.read_text()) assert "express" in pkg.get("dependencies", {}), "express not pinned in package.json" + + +# ── Health Skin (read-only monitor, separate page from the control Skin) ────── + +def test_health_skin_exists_and_is_separate(): + """The grain/system monitor is its own page + its own read-only endpoint — it + must NOT leak control into the Core Skin (local.html drives only control + routes). Keeping health separate is the whole point of 'Core vs Skin'.""" + assert HEALTH_HTML.is_file(), "web/health.html missing" + assert HEALTH_INFO.is_file(), "backend/health-info.js missing" + # the control Skin must not start calling /api/mxl/health — health is its own page + assert "/api/mxl/health" not in HTML.read_text(), \ + "the control Skin (local.html) should not poll health — keep it separate" + + +def test_health_skin_is_self_contained(): + """Same air-gap promise as the control Skin: no external hosts, self-hosted + fonts only, no CDN.""" + html = HEALTH_HTML.read_text() + urls = re.findall(r"https?://[a-z0-9.\-]+", html, re.I) + external = [u for u in urls if not re.search(r"(127\.0\.0\.1|localhost|w3\.org)", u)] + assert not external, f"health.html references external hosts: {external}" + assert "cdn" not in html.lower(), "health.html pulls from a CDN" + woff2 = list(FONTS.glob("*.woff2")) + for f in woff2: + assert f.name in html, f"health.html doesn't @font-face {f.name}" + + +def test_health_endpoint_is_read_only(): + """The health endpoint + page must be GET-only — a monitor never mutates the + facility. No POST/PUT/DELETE handlers touch the health surface.""" + src = SERVER.read_text() + assert "app.get('/api/mxl/health'" in src, "local-server missing GET /api/mxl/health" + assert "app.get('/health'" in src, "local-server doesn't serve /health page" + # no mutating verb on the health paths + for verb in ("post", "put", "delete", "patch"): + assert f"app.{verb}('/api/mxl/health'" not in src, \ + f"health endpoint must not expose {verb.upper()}" + + +def test_health_reuses_grain_probe_snapshot(): + """Cost control: the expensive grain measurement is grain_probe.py's job. The + endpoint must READ its snapshot (grains.json), not spawn its own probes.""" + src = SERVER.read_text() + assert "grains.json" in src, "health endpoint doesn't read the grain-probe snapshot" + # must cache so N viewers collapse to one scrape + assert "HEALTH_TTL" in src or "_healthCache" in src, \ + "health endpoint has no server-side cache (N viewers would each scrape)" diff --git a/web/health.html b/web/health.html new file mode 100644 index 0000000..bcfbeca --- /dev/null +++ b/web/health.html @@ -0,0 +1,218 @@ + + + + + +MXL Health + + + +
+
+
MXL Healthread-only facility monitor
+
+ connecting… + ← Switcher +
+ +

System

+
+
CPU
—
+
Memory
—
+
Load (1m)
—
—
+
Viewers
—
WebRTC program
+
+ +

Grain flow

+
+ + + +
FlowStateGrains/sUnique fps
waiting for probe…
+
+ +

Pipeline writers

+
+
Selector
state—
+
Keyer
state—
+
Encoder
state—
+
+ +
+ Grain rate and unique-frames/s come from the long-lived grain probe: a flow can tick 30 grains/s while the + picture is frozen (a repeat-wedge) — when that happens unique fps collapses + and the row turns amber. This page never controls the facility; it only reads. +
+
+ + + + From add96a2fa1a215330c2129d701f81b41ad31eae9 Mon Sep 17 00:00:00 2001 From: Guy Cochran Date: Fri, 9 Oct 2026 01:11:11 +0000 Subject: [PATCH 02/37] feat(health): read live thumbs health.json (already-running probe); grains.json fallback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Live HW verify on the $400 box showed tools/mxl_thumbs.py ALREADY writes health.json beside the thumbnails (system + per-slot {age,frozen} + viewers) — same frozen-reader detection as grain_probe, but already running. So the Health Skin reads THAT (zero new process, no second set of MXL readers), and falls back to grain_probe's grains.json for adopters who run it instead. - classifyThumbsHealth(): {age,frozen} -> ok/frozen/dead; folds in load/mem/cpus/viewers. - endpoint prefers health.json, uses the probe's host-true system numbers when Core runs off-box; UI renders either shape (Grains/s+Unique-fps OR Last-frame+Content-age). - staleAfterS=25 to match the thumbs health writer's ~15s cadence. Verified live (read-only, no box changes): /health -> 200, real payload — load 15.6, 24 cpus, mem 18%, pattern/playout/guest1/guest2 OK, guest3/layout DEAD, selector running active_input#3 1920x1080@30, encoder 6000kbps. (It even surfaced the live 60fps-restamp 'Out of range' selector error — the monitor earning its keep.) Tests: 55 node / 139 python, all green. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01GwSKEZZz4yEc2CGq5hcTmT --- backend/health-info.js | 47 +++++++++++++++++++++++++++++++++++- backend/local-server.js | 29 ++++++++++++++++++---- tests/js/health-info.test.js | 42 ++++++++++++++++++++++++++++++++ tests/test_local_ui.py | 10 +++++--- web/health.html | 20 ++++++++++++--- 5 files changed, 135 insertions(+), 13 deletions(-) diff --git a/backend/health-info.js b/backend/health-info.js index ef86dd7..9dd1edb 100644 --- a/backend/health-info.js +++ b/backend/health-info.js @@ -113,6 +113,51 @@ function readGrainsSnapshot(grainsPath, readFile = fs.readFileSync) { catch { return null; } } +// ── Thumbs health.json (the probe that's ALREADY running on the box) ────────── +// tools/mxl_thumbs.py writes health.json beside the thumbnails — same cadence, no +// extra process — with per-slot {age, frozen} where: +// age = seconds since the LAST frame arrived (delivery stalled -> dead) +// frozen = seconds since the CONTENT last CHANGED (picture stuck -> frozen) +// This is the cheaper, already-live source for the Health Skin: it carries the SAME +// frozen-reader detection as grain_probe's unique-fps, plus system + viewers, so we +// don't run a second set of MXL readers. classifyGrains (grains.json / grain_probe) +// stays supported for adopters who run that probe instead — both normalize to the +// same shape the UI consumes. +// staleAfterS default is 25: tools/mxl_thumbs.py's health loop writes on a ~15s +// cadence (its slots[].age/frozen are themselves sub-2s fresh, but the summary file +// is rewritten every 15s), so a poll landing late in that cycle can legitimately see +// an ~18-20s-old file — only flag stale past 25s (probe actually dead). grain_probe's +// grains.json is 3s — callers using it can pass a tighter staleAfterS. +function classifyThumbsHealth(h, { nowS, staleAfterS = 25, frozenAfterS = 5, deadAfterS = 5 } = {}) { + if (!h || typeof h !== 'object' || !h.slots) { + return { stale: true, age_s: null, flows: {}, system: null }; + } + const now = nowS != null ? nowS : Date.now() / 1000; + const ageS = h.ts != null ? Math.max(0, now - h.ts) : null; + const stale = ageS == null ? true : ageS > staleAfterS; + const flows = {}; + for (const [name, v] of Object.entries(h.slots)) { + if (!v || v.age == null) { flows[name] = { state: 'dead', age: null, frozen: null }; continue; } + let state = 'ok'; + if (v.age > deadAfterS) state = 'dead'; // no new frames at all + else if (v.frozen != null && v.frozen > frozenAfterS) state = 'frozen'; // frames arrive, picture stuck + flows[name] = { state, age: v.age, frozen: v.frozen != null ? v.frozen : null }; + } + // System block, assembled from the same file (load/mem are host-true in the probe). + const system = { + load: (h.load1 != null) ? { load1: h.load1, load5: h.load5, load15: h.load15 } : null, + cpus: h.cores != null ? h.cores : null, + mem: (h.mem_total_mb != null && h.mem_avail_mb != null) ? { + total_mb: h.mem_total_mb, + used_mb: h.mem_total_mb - h.mem_avail_mb, + used_pct: Math.round(((h.mem_total_mb - h.mem_avail_mb) / h.mem_total_mb) * 100), + } : null, + viewers: (h.viewers && typeof h.viewers === 'object') ? (h.viewers.count != null ? h.viewers.count : null) + : (typeof h.viewers === 'number' ? h.viewers : null), + }; + return { stale, age_s: ageS != null ? Math.round(ageS * 10) / 10 : null, flows, system, procs: h.procs || null }; +} + // ── Pipeline writers: normalize the three pipeline/status payloads for the UI. ─ // Pure — the caller fetches the JSON (or passes nulls when a port is unreachable) // and we shape it. No network here, so it's unit-testable. @@ -156,5 +201,5 @@ function composeHealth({ system, grains, pipelines, ts }) { module.exports = { readProcStat, cpuPercent, readMem, readLoad, cpuCount, - classifyGrains, readGrainsSnapshot, shapePipelines, composeHealth, + classifyGrains, classifyThumbsHealth, readGrainsSnapshot, shapePipelines, composeHealth, }; diff --git a/backend/local-server.js b/backend/local-server.js index 1c786b4..bb22367 100644 --- a/backend/local-server.js +++ b/backend/local-server.js @@ -147,6 +147,13 @@ app.get('/api/mxl/ingest/qr/:slot.png', (req, res) => { // + /proc + the three pipeline/status endpoints, and cache the aggregate for a // couple seconds so N open Health tabs collapse to one scrape. const health = require('./health-info'); +// Two possible probe snapshots, in preference order: +// health.json — written by tools/mxl_thumbs.py, which is ALREADY running beside the +// multiview thumbnails (no extra process, no extra MXL readers). Carries per-slot +// {age, frozen} (same frozen-reader detection) + system + viewers. +// grains.json — written by tools/grain_probe.py (bps + unique-fps). Supported for +// adopters who run that probe; we fall back to it if health.json isn't present. +const THUMBS_HEALTH_PATH = process.env.MXL_THUMBS_HEALTH_PATH || path.join(THUMBS_DIR, 'health.json'); const GRAINS_PATH = process.env.MXL_GRAINS_PATH || path.join(THUMBS_DIR, 'grains.json'); const HEALTH_RATE = parseInt(process.env.MXL_GRAIN_RATE || '30', 10); const HEALTH_TTL_MS = parseInt(process.env.MXL_HEALTH_TTL_MS || '2000', 10); @@ -171,16 +178,28 @@ async function buildHealth() { fetchJson(9605, '/pipeline/status'), fetchJson(9601, '/pipeline/status'), ]); + // Prefer the already-running thumbs health.json; fall back to grain_probe's grains.json. + const thumbs = health.readGrainsSnapshot(THUMBS_HEALTH_PATH); + let grains, probeSystem = null; + if (thumbs && thumbs.slots) { + const t = health.classifyThumbsHealth(thumbs, {}); + grains = { stale: t.stale, age_s: t.age_s, flows: t.flows, source: 'thumbs' }; + probeSystem = t.system; // load/mem/viewers measured where the probe runs (the box) + } else { + grains = health.classifyGrains(health.readGrainsSnapshot(GRAINS_PATH), { expectedRate: HEALTH_RATE }); + grains.source = 'grains'; + } + // System: use the probe's host-true numbers when available (Core may run off-box); + // otherwise read local /proc. CPU% always comes from local /proc deltas. const curCpu = health.readProcStat(); const system = { cpu_pct: health.cpuPercent(_prevCpu, curCpu), - cpus: health.cpuCount(), - mem: health.readMem(), - load: health.readLoad(), + cpus: (probeSystem && probeSystem.cpus) || health.cpuCount(), + mem: (probeSystem && probeSystem.mem) || health.readMem(), + load: (probeSystem && probeSystem.load) || health.readLoad(), + viewers: probeSystem ? probeSystem.viewers : null, }; _prevCpu = curCpu; - const grains = health.classifyGrains( - health.readGrainsSnapshot(GRAINS_PATH), { expectedRate: HEALTH_RATE }); const pipelines = health.shapePipelines({ selector, keyer, encoder }); return health.composeHealth({ ts: Date.now() / 1000, system, grains, pipelines }); } diff --git a/tests/js/health-info.test.js b/tests/js/health-info.test.js index a8fe9c3..1d09b71 100644 --- a/tests/js/health-info.test.js +++ b/tests/js/health-info.test.js @@ -53,6 +53,48 @@ test('grains: missing/garbage snapshot -> stale, empty', () => { assert.deepEqual(out.flows, {}); }); +// ── classifyThumbsHealth (the probe already running on the box) ───────────────── +test('thumbs: live slot (fresh frames, content changing) -> ok', () => { + const h2 = { ts: 1000, slots: { pattern: { age: 1.2, frozen: 1.2 } }, load1: 2 }; + const out = h.classifyThumbsHealth(h2, { nowS: 1000 }); + assert.equal(out.flows.pattern.state, 'ok'); + assert.equal(out.stale, false); +}); + +test('thumbs: FROZEN slot (frames arrive but content stuck) -> frozen', () => { + // age low (frames keep arriving) but frozen high (picture hasn't changed). + const h2 = { ts: 1000, slots: { guest1: { age: 1.0, frozen: 11.0 } } }; + const out = h.classifyThumbsHealth(h2, { nowS: 1000, frozenAfterS: 5 }); + assert.equal(out.flows.guest1.state, 'frozen', + 'frames arriving but content unchanged for >5s must read as frozen'); +}); + +test('thumbs: dead slot (no frames at all / null) -> dead', () => { + const h2 = { ts: 1000, slots: { guest3: null, playout: { age: 9.0, frozen: 9.0 } } }; + const out = h.classifyThumbsHealth(h2, { nowS: 1000, deadAfterS: 5 }); + assert.equal(out.flows.guest3.state, 'dead'); + assert.equal(out.flows.playout.state, 'dead', 'no new frames for >5s -> dead'); +}); + +test('thumbs: pulls system (load/mem/cpus/viewers) from the same file', () => { + const h2 = { + ts: 1000, slots: {}, + load1: 12.5, load5: 12, load15: 11, + cores: 24, mem_total_mb: 16000, mem_avail_mb: 4000, viewers: 3, + }; + const out = h.classifyThumbsHealth(h2, { nowS: 1000 }); + assert.equal(out.system.load.load1, 12.5); + assert.equal(out.system.cpus, 24); + assert.equal(out.system.mem.used_pct, 75); + assert.equal(out.system.viewers, 3); +}); + +test('thumbs: missing/garbage -> stale, empty', () => { + const out = h.classifyThumbsHealth(null, { nowS: 1000 }); + assert.equal(out.stale, true); + assert.deepEqual(out.flows, {}); +}); + // ── system parsing ──────────────────────────────────────────────────────────── test('cpuPercent: computes busy fraction from two /proc/stat samples', () => { const prev = { total: 1000, idle: 800 }; diff --git a/tests/test_local_ui.py b/tests/test_local_ui.py index f603da6..cd34234 100644 --- a/tests/test_local_ui.py +++ b/tests/test_local_ui.py @@ -152,11 +152,13 @@ def test_health_endpoint_is_read_only(): f"health endpoint must not expose {verb.upper()}" -def test_health_reuses_grain_probe_snapshot(): - """Cost control: the expensive grain measurement is grain_probe.py's job. The - endpoint must READ its snapshot (grains.json), not spawn its own probes.""" +def test_health_reuses_existing_probe_snapshot(): + """Cost control: the endpoint must READ an existing probe's snapshot, never spawn + its own MXL readers. It prefers the already-running thumbs health.json and falls + back to grain_probe's grains.json — either way, no new process.""" src = SERVER.read_text() - assert "grains.json" in src, "health endpoint doesn't read the grain-probe snapshot" + assert "health.json" in src, "health endpoint doesn't read the thumbs health snapshot" + assert "grains.json" in src, "health endpoint doesn't fall back to the grain-probe snapshot" # must cache so N viewers collapse to one scrape assert "HEALTH_TTL" in src or "_healthCache" in src, \ "health endpoint has no server-side cache (N viewers would each scrape)" diff --git a/web/health.html b/web/health.html index bcfbeca..e8ca8ff 100644 --- a/web/health.html +++ b/web/health.html @@ -107,7 +107,7 @@

System

Grain flow

- +
FlowStateGrains/sUnique fps
FlowStateGrains/sUnique fps
waiting for probe…
@@ -148,6 +148,12 @@

Pipeline writers

var cores=s.cpus?(' / '+s.cpus+' cores'):''; $('loadSub').textContent=s.load.load5.toFixed(1)+' · '+s.load.load15.toFixed(1)+cores; } + if(s.cpu_pct==null && s.load && s.cpus){ + // CPU% unavailable (Core running off-box) — approximate from load/cores. + var approx=Math.min(100,Math.round(s.load.load1/s.cpus*100)); + $('cpu').innerHTML=approx+'% ~load'; setBar($('cpuBar'),approx); + } + if(s.viewers!=null) $('viewers').textContent=s.viewers; } function renderGrains(g){ @@ -156,14 +162,22 @@

Pipeline writers

else { stale.textContent=''; stale.className='state'; } var tbody=$('grainRows'), flows=(g&&g.flows)||{}; var names=Object.keys(flows); + // Two probe shapes: grain_probe -> {bps, ufps}; thumbs health -> {age, frozen}. + var isThumbs=(g&&g.source==='thumbs'); + var head=$('grainHead'); + if(isThumbs) head.innerHTML='FlowStateLast frameContent age'; + else head.innerHTML='FlowStateGrains/sUnique fps'; if(!names.length){ tbody.innerHTML='no flows reported'; return; } tbody.innerHTML=names.map(function(n){ var f=flows[n], st=f.state||'dead'; + var c1,c2; + if(isThumbs){ c1=(f.age!=null?f.age+'s':'—'); c2=(f.frozen!=null?f.frozen+'s':'—'); } + else { c1=(f.bps!=null?f.bps:'—'); c2=(f.ufps!=null?f.ufps:'—'); } return ''+ ''+esc(n)+''+ ''+st+''+ - ''+(f.bps!=null?f.bps:'—')+''+ - ''+(f.ufps!=null?f.ufps:'—')+''+ + ''+c1+''+ + ''+c2+''+ ''; }).join(''); } From 724af0556d04cd11e62bc78fab9e622e81a9e5cd Mon Sep 17 00:00:00 2001 From: Guy Cochran Date: Fri, 9 Oct 2026 01:11:48 +0000 Subject: [PATCH 03/37] =?UTF-8?q?docs(health):=20reflect=20HW=20finding=20?= =?UTF-8?q?=E2=80=94=20reads=20already-running=20thumbs=20health.json,=20g?= =?UTF-8?q?rain=5Fprobe=20optional?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/HEALTH-SKIN.md | 35 +++++++++++++++++++++++------------ 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/docs/HEALTH-SKIN.md b/docs/HEALTH-SKIN.md index 3d3074e..bc8bfe0 100644 --- a/docs/HEALTH-SKIN.md +++ b/docs/HEALTH-SKIN.md @@ -20,28 +20,39 @@ health should not be able to touch the program bus. Core vs Skin, applied twice. ## Where the data comes from (and why it's cheap) -The expensive part — measuring grain rate *and* unique frames — is already done by -`tools/grain_probe.py`, which runs long-lived readers (one per flow) and writes a -rolling-window snapshot to `/grains.json` every 3 s. It has hard load guards -(pauses above load 26, exits above 29), so the probe can never be the thing that tips -the box over. +The grain/frozen measurement is **already being produced** on a running facility: +`tools/mxl_thumbs.py` (the multiview thumbnail generator) also writes `health.json` +beside the thumbnails, carrying per-slot `{age, frozen}` — `age` = seconds since the +last frame, `frozen` = seconds since the content last *changed* (its own repeat-wedge +detector). Plus system load/mem and viewers. Because the thumbnail probe is already +running, the Health Skin adds **no new process and no second set of MXL readers**. -The Core does **not** spawn probes or `docker exec` anything. `GET /api/mxl/health`: +`GET /api/mxl/health`: -1. reads `grains.json` (already on disk), -2. reads `/proc` for CPU/mem/load, +1. reads `health.json` (already on disk; falls back to `grains.json` if you run + `tools/grain_probe.py` instead — that probe adds unique-fps and has hard load + guards: pauses above load 26, exits above 29), +2. uses the probe's host-true load/mem/viewers (so it's correct even when the Core + runs off-box) and local `/proc` for CPU %, 3. fetches the three `:/pipeline/status` endpoints (same VM the cuts use), then caches the aggregate for ~2 s (`MXL_HEALTH_TTL_MS`) so N open Health tabs -collapse to a single scrape. The live PGM/encode path is never touched. +collapse to a single scrape. The Core never spawns a probe or `docker exec`s anything, +and the live PGM/encode path is never touched. + +> Note: `mxl_thumbs.py`'s `health.json` is rewritten on a ~15 s cadence (the per-slot +> age/frozen values inside it are sub-2 s fresh). For a tighter live refresh, run +> `tools/grain_probe.py` (3 s, adds unique-fps) — the endpoint picks up `grains.json` +> automatically. ## Running it - Serve: the Core (`backend/local-server.js`) serves the page at **`/health`** and the data at **`/api/mxl/health`** — no extra process. -- Requires the grain probe to be running for the Grain-flow section to populate - (`tools/grain_probe.py`, started by the facility bring-up / its runner). If the probe - isn't running, the page still shows System + Pipeline writers and marks grains stale. +- The Grain-flow section populates from whichever probe snapshot exists: + `health.json` (written by the already-running `tools/mxl_thumbs.py`) is used first; + `grains.json` (from `tools/grain_probe.py`) is the fallback. If neither is present the + page still shows System + Pipeline writers and marks grains stale. ## Config From b508360851ac4f5dfd05cd81551f7a093d670774 Mon Sep 17 00:00:00 2001 From: Guy Cochran Date: Fri, 9 Oct 2026 01:17:02 +0000 Subject: [PATCH 04/37] feat(health): read probe snapshot over MXL_THUMBS_ORIGIN (proxy mode), not just file MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On a box where the MXL domain is only mounted inside containers, the thumbnails (and health.json/grains.json beside them) are exposed via a small http server (MXL_THUMBS_ORIGIN, e.g. :8086/thumbs) — same as the /api/mxl/thumbs route already does. The health endpoint now fetches the snapshot over that origin when set, falling back to a local file read otherwise. Keeps Core off the Docker socket + portable. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01GwSKEZZz4yEc2CGq5hcTmT --- backend/local-server.js | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/backend/local-server.js b/backend/local-server.js index bb22367..004b239 100644 --- a/backend/local-server.js +++ b/backend/local-server.js @@ -157,6 +157,24 @@ const THUMBS_HEALTH_PATH = process.env.MXL_THUMBS_HEALTH_PATH || path.join(THUMB const GRAINS_PATH = process.env.MXL_GRAINS_PATH || path.join(THUMBS_DIR, 'grains.json'); const HEALTH_RATE = parseInt(process.env.MXL_GRAIN_RATE || '30', 10); const HEALTH_TTL_MS = parseInt(process.env.MXL_HEALTH_TTL_MS || '2000', 10); +// The probe snapshots live wherever the thumbnails do. In file mode we read +// /*.json; in proxy mode (MXL_THUMBS_ORIGIN set — e.g. the domain is only +// mounted in a container and a tiny http server exposes it) we fetch the same JSON over +// HTTP, exactly like the /api/mxl/thumbs route. This keeps the Core off the Docker +// socket and portable to either layout. +const THUMBS_SNAP_ORIGIN = THUMBS_ORIGIN; // e.g. http://127.0.0.1:8086/thumbs + +async function readSnapshot(fileName, filePath) { + if (THUMBS_SNAP_ORIGIN) { + try { + const base = THUMBS_SNAP_ORIGIN.replace(/\/$/, ''); + const r = await fetch(`${base}/${fileName}`, { signal: AbortSignal.timeout(3000) }); + return r.ok ? await r.json() : null; + } catch { return null; } + } + return health.readGrainsSnapshot(filePath); // file mode (sync read, cheap) +} + // Resolve the MXL control-API base the same way mxl-routes does, so the pipeline // status probes hit the same VM the cuts do. const HEALTH_MXL_VM = process.env.MXL_VM_URL @@ -179,14 +197,15 @@ async function buildHealth() { fetchJson(9601, '/pipeline/status'), ]); // Prefer the already-running thumbs health.json; fall back to grain_probe's grains.json. - const thumbs = health.readGrainsSnapshot(THUMBS_HEALTH_PATH); + const thumbs = await readSnapshot('health.json', THUMBS_HEALTH_PATH); let grains, probeSystem = null; if (thumbs && thumbs.slots) { const t = health.classifyThumbsHealth(thumbs, {}); grains = { stale: t.stale, age_s: t.age_s, flows: t.flows, source: 'thumbs' }; probeSystem = t.system; // load/mem/viewers measured where the probe runs (the box) } else { - grains = health.classifyGrains(health.readGrainsSnapshot(GRAINS_PATH), { expectedRate: HEALTH_RATE }); + const gr = await readSnapshot('grains.json', GRAINS_PATH); + grains = health.classifyGrains(gr, { expectedRate: HEALTH_RATE }); grains.source = 'grains'; } // System: use the probe's host-true numbers when available (Core may run off-box); From 8ef61ff5c1414cde3de97382f47468ccfa1550b9 Mon Sep 17 00:00:00 2001 From: Guy Cochran Date: Fri, 9 Oct 2026 01:20:17 +0000 Subject: [PATCH 05/37] feat(health): CORS allow-origin on /api/mxl/health so the public site can embed live numbers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Read-only public telemetry (GET, no secrets, no control) — mxlswitcher.com fetches it to show the grain explainer's live data. Default '*', override via MXL_HEALTH_CORS. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01GwSKEZZz4yEc2CGq5hcTmT --- backend/local-server.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/backend/local-server.js b/backend/local-server.js index 004b239..f591a4d 100644 --- a/backend/local-server.js +++ b/backend/local-server.js @@ -230,6 +230,9 @@ app.get('/api/mxl/health', async (req, res) => { _healthCache = { at: now, payload: await buildHealth() }; } res.set('Cache-Control', 'no-store'); + // Read-only public telemetry — allow cross-origin reads so the public site + // (mxlswitcher.com) can embed the live numbers. GET-only, no secrets, no control. + res.set('Access-Control-Allow-Origin', process.env.MXL_HEALTH_CORS || '*'); res.json(_healthCache.payload); } catch (e) { res.status(500).json({ error: String((e && e.message) || e) }); From 97598aa2cf760faab4a3deb31d208286063b37f7 Mon Sep 17 00:00:00 2001 From: Guy Cochran Date: Fri, 9 Oct 2026 01:21:51 +0000 Subject: [PATCH 06/37] site: 'What's a grain?' layman explainer with LIVE data from the $400 box MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Plain-English page (grain=one frame, flow=one source, shared memory=one copy everyone reads) that embeds the switcher's real /api/mxl/health — live source states + 'what the $400 box is doing right now' (load/CPU/mem). Linked first in the front-door learn cards. Pulls from the public mxl-skin.cochran.cloud health endpoint (CORS-enabled, read-only). Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01GwSKEZZz4yEc2CGq5hcTmT --- docs/index.html | 1 + docs/what-are-grains.html | 124 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 125 insertions(+) create mode 100644 docs/what-are-grains.html diff --git a/docs/index.html b/docs/index.html index 975a3ce..5f90762 100644 --- a/docs/index.html +++ b/docs/index.html @@ -289,6 +289,7 @@

From fresh Ubuntu VM to a cuttable MXL production in about ten minutes.

The point of this lab isn't the demo — it's the knowledge. If you want to understand MXL and where production is heading, start with these. All measured, all in the open.

+ PLAIN ENGLISH + LIVE DATAWhat's a grain?The one idea the whole system is built on — with live numbers from the $400 box → EXPLAINERWhat is MXL?The shared-memory media plane, in plain English + a NDI/2110/SRT comparison → THE WHOLE STACKThe smallest complete DMFThe entire open-standard pipeline in one diagram → DEEP REFERENCEMXL architectureGrains, flows, domains, the ring buffer + TAI timing — sourced from the SDK docs → diff --git a/docs/what-are-grains.html b/docs/what-are-grains.html new file mode 100644 index 0000000..26a5742 --- /dev/null +++ b/docs/what-are-grains.html @@ -0,0 +1,124 @@ + + + + + + +What's a grain? MXL in plain English — with live data · Switcher Lab for MXL™ + + + + + + + + + + + + + + + +
+ Switcher Lab FOR MXL™ + +
+ +
+
+

MXL in plain English

+

What's a "grain"? And what is a $400 computer actually doing?

+

The whole system is built out of one tiny idea. Once you get it, everything else — the cut, the multiview, the health numbers below — follows. No broadcast background needed.

+

Office Hours GlobalLive data from a running facility

+ +

A grain is one frame

+

Think of an old film reel: a long strip of still pictures. Play them fast enough — 30 a second — and your eye sees smooth motion. A "grain" is one of those pictures. One frame of video. (Audio works the same way: a grain is a short block of sound.)

+

So a normal 1080p camera running at 30 frames a second is producing 30 video grains every second, forever, as long as it's on.

+ +

A flow is one source

+

String all the grains from one camera together in order and you have a flow — the continuous stream of frames coming from that camera. Camera 1 is a flow. The playback clip is a flow. The guest on their phone is a flow. Your switcher is really just a room full of flows.

+ +
+ The clever part (MXL's whole reason to exist): normally, to send one camera to five different places you'd make five copies of the video — expensive. MXL instead puts each flow in shared memory — one copy that everything on the computer can read at once, like a whiteboard everyone in the room can see. The fifth reader costs almost nothing. That's why a cheap computer can do so much: it isn't copying video over and over. +
+ +

Why we count grains — and unique grains

+

If you want to know a camera is healthy, you count its grains per second. Seeing a steady 30/s means frames are arriving on time. Simple.

+

But there's a sneakier failure. Sometimes a feed keeps delivering 30 grains a second while showing the same frozen picture — the counter ticks, but nothing moves. So we also measure how many of those frames are actually different. When "frames arriving" stays high but "frames actually changing" drops to near zero, we know the picture is frozen even though the plumbing looks fine. That single trick catches a whole class of live-TV disasters before anyone on air notices.

+ +

Live right now: one $400 computer, running a show

+

This isn't a diagram. Below is the actual health readout from the reference switcher — a second-hand dual-Xeon machine bought for about $400 — pulled live as you loaded this page. It's running several real sources at once and serving the program to browsers.

+ +
+ + + +
Source (flow)StatusFrames arriving
reading the live facility…
+
+

Reading system load from the box…

+

Source: the switcher's own /api/mxl/health endpoint, same data an engineer watching the rack sees. "Dead" rows are simply sources that aren't connected at the moment (e.g. no phone guest joined).

+ +

The point, for the skeptics

+

Broadcast infrastructure has always meant specialized, expensive hardware. The numbers above are the argument against that: an entire live production control room — ingest, switching, graphics, encoding and remote control — fits on a computer that costs less than a single pro SDI cable run, with CPU to spare. The grains flow through shared memory, nothing is needlessly copied, and commodity silicon keeps up.

+

That's the Dynamic Media Facility idea, made concrete. Read the field findings → · See the full glossary →

+ +
MXL is a trademark of LF Projects, LLC. This is an independent explainer. Live figures come from a real reference facility and vary moment to moment. Not affiliated with or endorsed by LF Projects, EBU, AMWA or any vendor.
+
+
+ + + + + + From 5c322b18d41ce0f940f7708530d555d882b1fcdf Mon Sep 17 00:00:00 2001 From: Guy Cochran Date: Fri, 9 Oct 2026 01:31:32 +0000 Subject: [PATCH 07/37] feat(health): real viewer count from mediamtx API (MTX_API=yes), not the WHIP encoder MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The encoder publishes WHIP and never sees WebRTC clients, so it always reports viewers=0. The actual viewers connect to mediamtx — enable its API (:9997) and count read sessions on the program path. Health endpoint prefers that live count; encoder card reflects it too. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01GwSKEZZz4yEc2CGq5hcTmT --- backend/local-server.js | 27 +++++++++++++++++++++++++-- scripts/quickstart.sh | 5 ++++- 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/backend/local-server.js b/backend/local-server.js index f591a4d..27cf5dc 100644 --- a/backend/local-server.js +++ b/backend/local-server.js @@ -187,14 +187,34 @@ function fetchJson(port, apiPath, timeoutMs = 3000) { .catch(() => null); // unreachable port -> null; shapePipelines marks it so } +// The REAL viewer count: WebRTC clients connect to mediamtx, not to the WHIP encoder +// (which always self-reports 0). mediamtx's API (MTX_API=yes, :9997) lists live sessions. +// We count readers on the program path. Null if the API isn't enabled/reachable. +const MTX_API_URL = process.env.MXL_MEDIAMTX_API || 'http://127.0.0.1:9997'; +const PROGRAM_PATH_NAME = process.env.MXL_PROGRAM_PATH || 'mxl2webrtc'; +function fetchViewerCount(timeoutMs = 2500) { + return fetch(`${MTX_API_URL}/v3/webrtcsessions/list`, { signal: AbortSignal.timeout(timeoutMs) }) + .then((r) => (r.ok ? r.json() : null)) + .then((d) => { + if (!d || !Array.isArray(d.items)) return null; + // count only READ sessions on the program path (ignore the WHIP publisher itself) + const readers = d.items.filter((s) => + (s.state === 'read' || s.state === 'playing' || !s.state) && + (!s.path || s.path === PROGRAM_PATH_NAME)); + return readers.length; + }) + .catch(() => null); +} + let _prevCpu = health.readProcStat(); let _healthCache = { at: 0, payload: null }; async function buildHealth() { - const [selector, keyer, encoder] = await Promise.all([ + const [selector, keyer, encoder, mtxViewers] = await Promise.all([ fetchJson(9604, '/pipeline/status'), fetchJson(9605, '/pipeline/status'), fetchJson(9601, '/pipeline/status'), + fetchViewerCount(), ]); // Prefer the already-running thumbs health.json; fall back to grain_probe's grains.json. const thumbs = await readSnapshot('health.json', THUMBS_HEALTH_PATH); @@ -216,10 +236,13 @@ async function buildHealth() { cpus: (probeSystem && probeSystem.cpus) || health.cpuCount(), mem: (probeSystem && probeSystem.mem) || health.readMem(), load: (probeSystem && probeSystem.load) || health.readLoad(), - viewers: probeSystem ? probeSystem.viewers : null, + // Prefer mediamtx's live session count (the truth); fall back to the probe file. + viewers: mtxViewers != null ? mtxViewers : (probeSystem ? probeSystem.viewers : null), }; _prevCpu = curCpu; const pipelines = health.shapePipelines({ selector, keyer, encoder }); + // Surface the real count on the encoder card too (it self-reports 0 — see above). + if (mtxViewers != null && pipelines.encoder) pipelines.encoder.viewers = mtxViewers; return health.composeHealth({ ts: Date.now() / 1000, system, grains, pipelines }); } diff --git a/scripts/quickstart.sh b/scripts/quickstart.sh index 378536f..d06c7bd 100755 --- a/scripts/quickstart.sh +++ b/scripts/quickstart.sh @@ -295,8 +295,11 @@ fi # mediamtx keeps serving WebRTC (8889/8189), which the program monitor still needs. MTX_SRT_ARGS=() [ "$GUEST_TRANSPORT" = srt-listen ] && MTX_SRT_ARGS=(-e MTX_SRT=no) +# Enable mediamtx's control API (localhost:9997, --network host) so the Core can read +# the REAL viewer count — the WHIP encoder only publishes, it never sees the WebRTC +# clients, which connect to mediamtx directly. GET /v3/webrtcsessions/list is the truth. docker run -d --name mediamtx --network host --restart unless-stopped "${MTX_CONF_ARGS[@]}" \ - "${MTX_SRT_ARGS[@]}" -e MTX_WEBRTCADDITIONALHOSTS="$PUBLIC_IP" "$IMG_MEDIAMTX" >/dev/null + "${MTX_SRT_ARGS[@]}" -e MTX_API=yes -e MTX_WEBRTCADDITIONALHOSTS="$PUBLIC_IP" "$IMG_MEDIAMTX" >/dev/null run_mf(){ # name hostport image extra... local name=$1 port=$2 image=$3; shift 3 # Control APIs bind 127.0.0.1 ONLY (least privilege): these are unauthenticated From ceb012abf51dc1f3c805fc5d8732eddc4825fd9e Mon Sep 17 00:00:00 2001 From: Guy Cochran Date: Fri, 9 Oct 2026 02:04:59 +0000 Subject: [PATCH 08/37] fix(cut): a down/bypassed keyer must never fail a cut or status MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The cut is the selector's active-input move; the keyer self-heal after it is best-effort. Before, the cut path awaited mxlApi(9605) unconditionally, so a stopped keyer (unreachable or 400) sank an otherwise-successful cut with ':9605 -> 400' — exactly what broke cutting after the keyer went down. Now: skip the keyer entirely when MXL_KEYER_BYPASS=1, and otherwise wrap the self-heal in try/catch so a down keyer only logs. mxlStatus() also defaults keyer+testgen to {} so the Skin's 1s status poll survives them being down. Robustness for the 'open it to the public' plan: switching can't break on the keyer. +3 node tests (cut-with-keyer-down, bypass-never-pokes-keyer, status-survives-down-keyer). Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01GwSKEZZz4yEc2CGq5hcTmT --- backend/mxl-routes.js | 43 ++++++++++++++++++++++++++---------- tests/js/mxl-routes.test.js | 44 +++++++++++++++++++++++++++++++++++++ 2 files changed, 75 insertions(+), 12 deletions(-) diff --git a/backend/mxl-routes.js b/backend/mxl-routes.js index 7e1af71..2bb8ae9 100644 --- a/backend/mxl-routes.js +++ b/backend/mxl-routes.js @@ -156,9 +156,13 @@ module.exports = function registerMxlRoutes(app, opts = {}) { })(); async function mxlStatus() { + // Only the SELECTOR (9604) is essential for status. The keyer (9605) and + // test-generator (9600) are optional — when bypassed or down they must not take + // down the whole status poll the Skin runs every second. Default them to {}. const [keyer, sel, tg1] = await Promise.all([ - mxlApi(9605, '/pipeline/status'), mxlApi(9604, '/pipeline/status'), - mxlApi(9600, '/pipeline/status') + mxlApi(9605, '/pipeline/status').catch(() => ({})), + mxlApi(9604, '/pipeline/status'), + mxlApi(9600, '/pipeline/status').catch(() => ({})), ]); // Which flow UUID is actually on program right now? The selector reports its // active_input as an INDEX into its own live input_flow_uuids, so translate @@ -246,16 +250,31 @@ module.exports = function registerMxlRoutes(app, opts = {}) { } await mxlApi(9604, '/pipeline/active-input', { slot: selIndex }); keyframeNudge(); // optional IDR (off by default — see keyframeNudge) - // self-heal: if the keyer is wired cam-direct (pre-relay topology), move it - const keyer = await mxlApi(9605, '/pipeline/status'); - if (keyer.input_flow_uuid !== MXL_SEL_FLOW) { - const keyWas = !!keyer.key_on; - await mxlApi(9605, '/pipeline/stop', {}).catch(() => {}); - await mxlApi(9605, '/pipeline/start', mxlKeyerBody(MXL_SEL_FLOW)); - await mxlApi(9605, '/pipeline/key', { on: keyWas }); - await new Promise(r => setTimeout(r, 1500)); - await mxlApi(9601, '/pipeline/stop', {}).catch(() => {}); - await mxlApi(9601, '/pipeline/start', mxlEncoderBody); + // The CUT itself is done — the selector's active-input moved. Everything below is + // a best-effort keyer self-heal, and it must NEVER fail the cut. + // + // When the keyer is bypassed (MXL_KEYER_BYPASS=1 — the encoder reads the selector + // output directly, no graphics overlay) or simply down, there is no keyer to heal: + // poking :9605 here used to throw (mxlApi rejects on unreachable/non-OK) and sink + // an otherwise-successful cut with ":9605 -> 400". Skip it, and even when we do try, + // swallow any error. A down/bypassed keyer can't break switching. + if (env.MXL_KEYER_BYPASS !== '1') { + try { + // self-heal: if the keyer is wired cam-direct (pre-relay topology), move it + const keyer = await mxlApi(9605, '/pipeline/status'); + if (keyer && keyer.running && keyer.input_flow_uuid !== MXL_SEL_FLOW) { + const keyWas = !!keyer.key_on; + await mxlApi(9605, '/pipeline/stop', {}).catch(() => {}); + await mxlApi(9605, '/pipeline/start', mxlKeyerBody(MXL_SEL_FLOW)); + await mxlApi(9605, '/pipeline/key', { on: keyWas }); + await new Promise(r => setTimeout(r, 1500)); + await mxlApi(9601, '/pipeline/stop', {}).catch(() => {}); + await mxlApi(9601, '/pipeline/start', mxlEncoderBody); + } + } catch (e) { + // keyer down/unreachable — the cut already succeeded; log and move on + console.warn(`cut: keyer self-heal skipped (${e.message})`); + } } return { ok: true, input: slot }; } finally { mxlBusy = false; } diff --git a/tests/js/mxl-routes.test.js b/tests/js/mxl-routes.test.js index 7fd22ce..adb4239 100644 --- a/tests/js/mxl-routes.test.js +++ b/tests/js/mxl-routes.test.js @@ -204,3 +204,47 @@ test('Take flip-flops PVW↔PGM: the old program drops back to preview', async ( assert.strictEqual(took.body.input, 1, 'program is now the armed source (slot 1)'); assert.strictEqual(took.body.pvw, 0, 'preview is now the OLD program (slot 0) — the flip-flop'); }); + +// ── Keyer-tolerant cut (public-demo robustness) ──────────────────────────────── +// The CUT is the selector's active-input move. A down/bypassed keyer must never +// fail it. Before this, the cut path did `await mxlApi(9605,...)` unconditionally, +// so a stopped keyer (:9605 unreachable or 400) sank an otherwise-good cut. + +test('cut succeeds even when the keyer (:9605) is DOWN', async () => { + const CAM = 'ca111e00-aaaa-4bbb-8ccc-000000000001'; + const app = mkApp(); registerMxlRoutes(app, { env: { MXL_PREWARM: '0' }, log: silent }); + const calls = mkFetch((url) => { + if (url.includes(':9605/')) throw new Error('connect ECONNREFUSED'); // keyer down + if (url.includes(':9604/pipeline/status')) return { ok: true, text: async () => JSON.stringify({ active_input: 1, input_flow_uuids: [CAM] }) }; + return { ok: true, text: async () => '{}' }; + }); + const res = await call(app, 'POST /api/mxl/input', { headers: {}, body: { input: 'cam' }, ip: '1.1.1.1' }); + assert.strictEqual(res.code, 200, 'cut must succeed despite the keyer being down'); + // the real cut (active-input) still went out + assert.ok(calls.some((c) => c.url.includes(':9604/pipeline/active-input')), 'the selector cut was issued'); +}); + +test('MXL_KEYER_BYPASS=1: a cut never pokes the keyer (:9605)', async () => { + const CAM = 'ca111e00-aaaa-4bbb-8ccc-000000000001'; + const app = mkApp(); registerMxlRoutes(app, { env: { MXL_PREWARM: '0', MXL_KEYER_BYPASS: '1' }, log: silent }); + const calls = mkFetch((url) => { + if (url.includes(':9604/pipeline/status')) return { ok: true, text: async () => JSON.stringify({ active_input: 1, input_flow_uuids: [CAM] }) }; + return { ok: true, text: async () => '{}' }; + }); + const res = await call(app, 'POST /api/mxl/input', { headers: {}, body: { input: 'cam' }, ip: '1.1.1.1' }); + assert.strictEqual(res.code, 200, 'cut succeeds in bypass mode'); + assert.ok(!calls.some((c) => c.url.includes(':9605/')), 'bypass mode must not touch the keyer at all'); +}); + +test('status survives a down keyer (:9605) — the Skin keeps polling', async () => { + const CAM = 'ca111e00-aaaa-4bbb-8ccc-000000000001'; + const app = mkApp(); registerMxlRoutes(app, { env: {}, log: silent }); + mkFetch((url) => { + if (url.includes(':9605/')) throw new Error('ECONNREFUSED'); // keyer down + if (url.includes(':9600/')) throw new Error('ECONNREFUSED'); // test-gen down too + if (url.includes(':9604/pipeline/status')) return { ok: true, text: async () => JSON.stringify({ active_input: 0, input_flow_uuids: [CAM] }) }; + return { ok: true, text: async () => '{}' }; + }); + const res = await call(app, 'GET /api/mxl/status', { headers: {}, ip: '1.1.1.1' }); + assert.strictEqual(res.code, 200, 'status must still return 200 with keyer+testgen down'); +}); From 1cbf57af3304bfc02607eacab71daa63417adae9 Mon Sep 17 00:00:00 2001 From: Guy Cochran Date: Fri, 9 Oct 2026 02:05:41 +0000 Subject: [PATCH 09/37] test: allowlist mxlswitcher.com in local-server.js CORS comment (comment-only, like quickstart.sh) --- tests/test_no_hardcoded_hosts.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_no_hardcoded_hosts.py b/tests/test_no_hardcoded_hosts.py index c054f51..e5424ab 100644 --- a/tests/test_no_hardcoded_hosts.py +++ b/tests/test_no_hardcoded_hosts.py @@ -41,6 +41,7 @@ "tools/guest-leg-doctor.sh": {"10.0.0.4", "10.0.0.5", "prodbots.com"}, # MXL_VM1_IP/VM2_IP/BACKEND defaults "tools/selector-doctor.sh": {"prodbots.com"}, # MXL_BACKEND_URL default "scripts/quickstart.sh": {"mxlswitcher.com"}, # comment only + "backend/local-server.js": {"mxlswitcher.com"}, # comment only (CORS rationale; override via MXL_HEALTH_CORS) "tools/start-jonas-leg.sh": {"10.0.0.4"}, # MXL_VM1_IP default "tools/tams_shipper.py": {"203.0.113.140"}, # TAMS_HOST default (RFC 5737 placeholder) "tools/backfill-mini.py": {"203.0.113.140"}, # TAMS_HOST default (RFC 5737 placeholder) From 0142872450d8c7b1ce8550d24bc5799bf52620d1 Mon Sep 17 00:00:00 2001 From: Guy Cochran Date: Fri, 9 Oct 2026 02:15:11 +0000 Subject: [PATCH 10/37] feat(join): public phone-inject page (QR + Larix + live preview) with configurable guests/ports MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 'open it up' flow: a visitor opens /join, scans a QR with Larix, and their phone becomes a switchable guest source. Built for a public demo. - web/join.html: big QR, 3-step Larix instructions, app-store links, manual SRT settings, live program preview + a live source/viewer counter (reads /api/mxl/health). Air-gapped (self-hosted fonts; only the two app-store install links are external). - ingest-info.js: MXL_GUESTS (advertise exactly the slots you run, e.g. guest1,guest2,guest3) + MXL_GUEST_PORTS (pin NON-contiguous real ports). Fixes the demo-breaker: the box's guest3 listens on :8895, but the computed scheme advertised :8892 (squatted by mediamtx) — a wrong QR = a dead scan. Now the QR always matches the real listener. - QR route validates against configured slots (INGEST_SLOTS), not a hardcoded guest[12]. - /join route added. +8 node tests, +2 python tests. Full suite green (62 node / 141 py). Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01GwSKEZZz4yEc2CGq5hcTmT --- backend/ingest-info.js | 46 ++++++- backend/local-server.js | 15 +- tests/js/mxl-ingest-transport.test.js | 42 ++++++ tests/test_local_ui.py | 28 ++++ web/join.html | 190 ++++++++++++++++++++++++++ 5 files changed, 314 insertions(+), 7 deletions(-) create mode 100644 web/join.html diff --git a/backend/ingest-info.js b/backend/ingest-info.js index 8f12b01..87f1605 100644 --- a/backend/ingest-info.js +++ b/backend/ingest-info.js @@ -10,33 +10,67 @@ // into a per-guest port (srtPort + index) with NO streamid. Mirrors // quickstart's MXL_GUEST_TRANSPORT=srt-listen (one listener owns one port). -const GUESTS = [ +const DEFAULT_GUESTS = [ { slot: 'guest1', label: 'Guest 1' }, { slot: 'guest2', label: 'Guest 2' }, ]; +// Parse "guest1,guest2,guest3" or "Guest 1:guest1,Phone:guest3" into a guest list. +// Falls back to DEFAULT_GUESTS. Lets a deployment advertise exactly the slots it runs +// (e.g. the box added a 3rd "phone" slot) without code changes. +function parseGuests(spec) { + if (!spec) return DEFAULT_GUESTS.slice(); + const out = []; + for (const part of String(spec).split(',').map((s) => s.trim()).filter(Boolean)) { + const [a, b] = part.split(':').map((s) => s.trim()); + const slot = (b || a); + const label = b ? a : a.replace(/^guest/i, 'Guest '); + if (/^guest\d+$/i.test(slot)) out.push({ slot: slot.toLowerCase(), label }); + } + return out.length ? out : DEFAULT_GUESTS.slice(); +} + +// Parse an explicit port list "8890,8891,8895" → [8890,8891,8895]. Empty → null (use +// the computed scheme). Non-contiguous real deployments (a bolt-on slot on a gap port) +// MUST set this so the advertised QR port matches the actual listener. (HW Oct 9: the +// box's guest3 listens on :8895, not the computed :8892 — a wrong QR = a dead demo.) +function parsePorts(spec) { + if (!spec) return null; + const ports = String(spec).split(',').map((s) => parseInt(s.trim(), 10)).filter((n) => Number.isInteger(n) && n > 0); + return ports.length ? ports : null; +} + // Resolve the config once from a plain object (e.g. process.env), so callers and // tests share identical logic. transport defaults to srt-listen (matches quickstart's // default; quickstart also passes MXL_GUEST_TRANSPORT through explicitly). -function ingestConfig({ publicIp = '', srtPort = 8890, transport = 'srt-listen' } = {}) { +// guests — explicit slot list (MXL_GUESTS), else guest1+guest2 +// guestPorts — explicit per-guest ports (MXL_GUEST_PORTS), else srtPort + index +function ingestConfig({ publicIp = '', srtPort = 8890, transport = 'srt-listen', guests, guestPorts } = {}) { const listen = transport === 'srt-listen'; + const list = Array.isArray(guests) ? guests : parseGuests(guests); + const explicitPorts = Array.isArray(guestPorts) ? guestPorts : parsePorts(guestPorts); return { publicIp, srtPort, transport, listen, - // listen: each guest owns srtPort + index; direct: all share srtPort. - guestPort: (i) => (listen ? srtPort + i : srtPort), + guests: list, + // Explicit port map wins (handles non-contiguous real ports). Otherwise: + // listen → each guest owns srtPort + index; direct → all share srtPort. + guestPort: (i) => (explicitPorts && explicitPorts[i] != null) + ? explicitPorts[i] + : (listen ? srtPort + i : srtPort), }; } // The JSON body for GET /api/mxl/ingest. function ingestInfo(cfg) { + const guests = cfg.guests || DEFAULT_GUESTS; return { public_ip: cfg.publicIp || null, srt_port: cfg.srtPort, transport: cfg.transport, - guests: GUESTS.map((g, i) => ({ + guests: guests.map((g, i) => ({ ...g, port: cfg.guestPort(i), streamid: cfg.listen ? null : 'publish:' + g.slot, @@ -71,4 +105,4 @@ function larixUrl(cfg, slot) { + `&conn[][mode]=av&conn[][srtstreamid]=${sid}&conn[][srtlatency]=1000`; } -module.exports = { GUESTS, ingestConfig, ingestInfo, larixUrl }; +module.exports = { GUESTS: DEFAULT_GUESTS, DEFAULT_GUESTS, parseGuests, parsePorts, ingestConfig, ingestInfo, larixUrl }; diff --git a/backend/local-server.js b/backend/local-server.js index 27cf5dc..6b07896 100644 --- a/backend/local-server.js +++ b/backend/local-server.js @@ -107,7 +107,14 @@ const INGEST_CFG = ingestConfig({ publicIp: PUBLIC_IP, srtPort: SRT_PORT, transport: process.env.MXL_GUEST_TRANSPORT || 'srt-listen', + // MXL_GUESTS lets a deployment advertise exactly its slots (e.g. "guest1,guest2,guest3" + // with a bolt-on phone slot). MXL_GUEST_PORTS pins the real listener ports when they + // aren't contiguous (e.g. "8890,8891,8895" — the box's guest3 is on :8895, not :8892). + guests: process.env.MXL_GUESTS || undefined, + guestPorts: process.env.MXL_GUEST_PORTS || undefined, }); +// The set of valid guest slots for the QR route — derived from the config, not hardcoded. +const INGEST_SLOTS = new Set((INGEST_CFG.guests || []).map((g) => g.slot)); app.get('/api/mxl/ingest', (req, res) => { res.json(ingestInfo(INGEST_CFG)); @@ -119,7 +126,9 @@ app.get('/api/mxl/ingest', (req, res) => { const { spawn } = require('child_process'); app.get('/api/mxl/ingest/qr/:slot.png', (req, res) => { const slot = req.params.slot; - if (!/^guest[12]$/.test(slot)) return res.status(400).end(); + // validate against the configured guest slots (not a hardcoded guest1/2), and keep + // the strict shape so nothing odd reaches the shell below. + if (!/^guest\d+$/.test(slot) || !INGEST_SLOTS.has(slot)) return res.status(400).end(); if (!PUBLIC_IP) return res.status(503).end(); // no IP → nothing to encode const payload = larixUrl(INGEST_CFG, slot); // segno writes a PNG to stdout; -o - with --scale for a crisp phone-scannable size. @@ -265,6 +274,10 @@ app.get('/api/mxl/health', async (req, res) => { // The Health Skin page (read-only). Separate URL from the control Skin. app.get('/health', (req, res) => res.sendFile(path.join(WEB_DIR, 'health.html'))); +// The public "join the show" page — QR + Larix instructions + a live program +// preview. No control, no token; a visitor publishes their phone as a guest source. +app.get('/join', (req, res) => res.sendFile(path.join(WEB_DIR, 'join.html'))); + // Serve the operator UI + its static assets. app.get('/', (req, res) => res.sendFile(path.join(WEB_DIR, 'local.html'))); app.use(express.static(WEB_DIR)); diff --git a/tests/js/mxl-ingest-transport.test.js b/tests/js/mxl-ingest-transport.test.js index 2078f88..d15c486 100644 --- a/tests/js/mxl-ingest-transport.test.js +++ b/tests/js/mxl-ingest-transport.test.js @@ -62,3 +62,45 @@ test('no public IP: srt_url + qr are null, no crash', () => { assert.equal(j.guests[0].srt_url, null); assert.equal(j.guests[0].qr, null); }); + +// ── Configurable guest list + explicit ports (public phone-inject flow) ───────── +// A real deployment may run a 3rd "phone" slot on a NON-contiguous port (the box's +// guest3 listens on :8895, not the computed :8892). The advertised QR/SRT URL MUST +// match the real listener or the scan sends the phone to a dead port. +const { parseGuests, parsePorts } = require(path.join(__dirname, '..', '..', 'backend', 'ingest-info.js')); + +test('MXL_GUESTS adds a third slot (guest3) to the advertised list', () => { + const cfg = ingestConfig({ publicIp: PUBLIC_IP, srtPort: BASE_PORT, guests: 'guest1,guest2,guest3' }); + const j = ingestInfo(cfg); + assert.equal(j.guests.length, 3); + assert.equal(j.guests[2].slot, 'guest3'); + assert.equal(j.guests[2].label, 'Guest 3'); +}); + +test('MXL_GUEST_PORTS pins non-contiguous real ports (guest3 on :8895, not :8892)', () => { + const cfg = ingestConfig({ + publicIp: PUBLIC_IP, srtPort: BASE_PORT, transport: 'srt-listen', + guests: 'guest1,guest2,guest3', guestPorts: '8890,8891,8895', + }); + const j = ingestInfo(cfg); + assert.equal(j.guests[0].port, 8890); + assert.equal(j.guests[1].port, 8891); + assert.equal(j.guests[2].port, 8895, 'guest3 must advertise its REAL port 8895'); + assert.match(j.guests[2].srt_url, /:8895\?/); + // the Larix deep-link for guest3 must carry :8895 too + const lx = larixUrl(cfg, 'guest3'); + assert.match(lx, /%3A8895/, 'guest3 deep-link must point at the real :8895'); +}); + +test('parseGuests: accepts "Label:slot" and bare slots; bad entries dropped', () => { + assert.deepEqual(parseGuests('guest1,guest2'), [{ slot: 'guest1', label: 'Guest 1' }, { slot: 'guest2', label: 'Guest 2' }]); + assert.deepEqual(parseGuests('Phone:guest3'), [{ slot: 'guest3', label: 'Phone' }]); + assert.deepEqual(parseGuests('nonsense,guest4'), [{ slot: 'guest4', label: 'Guest 4' }]); + assert.equal(parseGuests('').length, 2, 'empty falls back to the 2 defaults'); +}); + +test('parsePorts: parses a CSV port list, null on empty/garbage', () => { + assert.deepEqual(parsePorts('8890,8891,8895'), [8890, 8891, 8895]); + assert.equal(parsePorts(''), null); + assert.equal(parsePorts('abc'), null); +}); diff --git a/tests/test_local_ui.py b/tests/test_local_ui.py index cd34234..83d9633 100644 --- a/tests/test_local_ui.py +++ b/tests/test_local_ui.py @@ -9,6 +9,7 @@ REPO = Path(__file__).resolve().parent.parent HTML = REPO / "web" / "local.html" HEALTH_HTML = REPO / "web" / "health.html" +JOIN_HTML = REPO / "web" / "join.html" SERVER = REPO / "backend" / "local-server.js" ROUTES = REPO / "backend" / "mxl-routes.js" HEALTH_INFO = REPO / "backend" / "health-info.js" @@ -140,6 +141,33 @@ def test_health_skin_is_self_contained(): assert f.name in html, f"health.html doesn't @font-face {f.name}" +def test_join_page_exists_and_is_self_contained(): + """The public phone-inject page: QR + Larix steps + live program preview. Must be + air-gapped (self-hosted fonts, no external hosts) EXCEPT the two app-store links, + which are legitimate outbound links a visitor taps to install Larix.""" + assert JOIN_HTML.is_file(), "web/join.html missing" + html = JOIN_HTML.read_text() + urls = re.findall(r"https?://[a-z0-9.\-]+", html, re.I) + # allow the app-store install links (apple + google) — everything else must be local + allowed_ext = ("apps.apple.com", "play.google.com") + external = [u for u in urls + if not re.search(r"(127\.0\.0\.1|localhost|w3\.org)", u) + and not any(a in u for a in allowed_ext)] + assert not external, f"join.html references unexpected external hosts: {external}" + woff2 = list(FONTS.glob("*.woff2")) + for f in woff2: + assert f.name in html, f"join.html doesn't @font-face {f.name}" + + +def test_join_page_served_and_qr_route_not_hardcoded_to_guest12(): + """The Core serves /join, and the QR route validates against the configured guest + slots (so a 3rd 'phone' slot works) rather than a hardcoded guest1/guest2 regex.""" + src = SERVER.read_text() + assert "app.get('/join'" in src, "local-server doesn't serve the /join page" + assert "INGEST_SLOTS" in src, "QR route still hardcodes guest slots (should validate against config)" + assert "guest[12]" not in src, "QR route still has the old guest[12]-only gate" + + def test_health_endpoint_is_read_only(): """The health endpoint + page must be GET-only — a monitor never mutates the facility. No POST/PUT/DELETE handlers touch the health surface.""" diff --git a/web/join.html b/web/join.html new file mode 100644 index 0000000..39ea24c --- /dev/null +++ b/web/join.html @@ -0,0 +1,190 @@ + + + + + +Join the show — MXL + + + +
+
+
MXLJoin the show
+
+ Watch program → +
+ +

Be on air in 30 seconds

+

Put your phone's camera
into a live broadcast.

+

No app store account, no login. Scan the code with the free Larix Broadcaster app and your camera becomes a switchable source in a real production control room — running on a single $400 computer.

+ +
+
+
+ Scan to join +
Scan to go live
+
loading…
+
+ +
+ +
+
    +
  1. + Install Larix Broadcaster +
    Free, on iPhone & Android. It's the app that sends your camera over SRT.
    +
    + iPhone — App Store ↗ + Android — Google Play ↗ +
    +
    2. +
  2. + Scan the code +
    Open your phone camera (or Larix's built-in scanner) and point it at the QR. It loads all the settings for you — tap to import.
    +
    3. +
  3. + Tap the red button +
    In Larix, hit record/stream. You're now publishing to the switcher. Watch yourself appear in the program below.
    +
    4. +
+ +
+
Manual settings (if the QR won't scan)
+
ProtocolSRT (Caller)
+
URL—
+
Latency1000 ms
+
ModeAudio + Video
+ +
+ +
+

Program — you'll appear here

+
+ ● LIVE + +
+
Waiting for a source… once you go live, ask the director to cut to you — or watch the sources count rise.
+
+
+
+ +
+ An open-source MXL (Media eXchange Layer) switcher. Your phone's video travels as SRT into a shared-memory media domain and is composited live. How it works → +
+
+ + + + From eda19b973d55703ae7f6828c0442822199bb2684 Mon Sep 17 00:00:00 2001 From: Guy Cochran Date: Fri, 9 Oct 2026 02:20:00 +0000 Subject: [PATCH 11/37] feat(public): /welcome landing (Join/Watch/Monitor) + TD hand-off link to /welcome,/join MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Completes the public-demo flow with a safe model: - /welcome — public landing routing visitors to Join / Watch program / Health monitor, with a live 'N live sources · M watching · box at X% CPU' line. Never shows the token-gated control Skin. - control Skin's 'Add your camera' modal now links /welcome + /join for TD hand-off. Safe by construction: /join + /welcome + /program + /health are public & read-only or contribute-only; cut/take stay token-gated at '/'. +1 python test. Suite green (62/142). Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01GwSKEZZz4yEc2CGq5hcTmT --- backend/local-server.js | 5 ++ tests/test_local_ui.py | 15 ++++++ web/local.html | 3 ++ web/welcome.html | 104 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 127 insertions(+) create mode 100644 web/welcome.html diff --git a/backend/local-server.js b/backend/local-server.js index 6b07896..4df7cfa 100644 --- a/backend/local-server.js +++ b/backend/local-server.js @@ -278,6 +278,11 @@ app.get('/health', (req, res) => res.sendFile(path.join(WEB_DIR, 'health.html')) // preview. No control, no token; a visitor publishes their phone as a guest source. app.get('/join', (req, res) => res.sendFile(path.join(WEB_DIR, 'join.html'))); +// Public welcome/landing — routes a visitor to Join / Watch / Monitor (no control). +// Give demo visitors this URL (or /join) instead of "/" so they never land on the +// token-gated control Skin. The TD still uses "/". +app.get('/welcome', (req, res) => res.sendFile(path.join(WEB_DIR, 'welcome.html'))); + // Serve the operator UI + its static assets. app.get('/', (req, res) => res.sendFile(path.join(WEB_DIR, 'local.html'))); app.use(express.static(WEB_DIR)); diff --git a/tests/test_local_ui.py b/tests/test_local_ui.py index 83d9633..c17b7b9 100644 --- a/tests/test_local_ui.py +++ b/tests/test_local_ui.py @@ -10,6 +10,7 @@ HTML = REPO / "web" / "local.html" HEALTH_HTML = REPO / "web" / "health.html" JOIN_HTML = REPO / "web" / "join.html" +WELCOME_HTML = REPO / "web" / "welcome.html" SERVER = REPO / "backend" / "local-server.js" ROUTES = REPO / "backend" / "mxl-routes.js" HEALTH_INFO = REPO / "backend" / "health-info.js" @@ -168,6 +169,20 @@ def test_join_page_served_and_qr_route_not_hardcoded_to_guest12(): assert "guest[12]" not in src, "QR route still has the old guest[12]-only gate" +def test_welcome_landing_served_and_self_contained(): + """The public landing (/welcome) routes visitors to Join/Watch/Monitor. Air-gapped + (self-hosted fonts, no external hosts), served by the Core.""" + assert WELCOME_HTML.is_file(), "web/welcome.html missing" + assert "app.get('/welcome'" in SERVER.read_text(), "local-server doesn't serve /welcome" + html = WELCOME_HTML.read_text() + urls = re.findall(r"https?://[a-z0-9.\-]+", html, re.I) + external = [u for u in urls if not re.search(r"(127\.0\.0\.1|localhost|w3\.org)", u)] + assert not external, f"welcome.html references external hosts: {external}" + # the three public entry points must be linked, and NOT the token-gated control Skin + for href in ("/join", "/program/", "/health"): + assert href in html, f"welcome.html doesn't link {href}" + + def test_health_endpoint_is_read_only(): """The health endpoint + page must be GET-only — a monitor never mutates the facility. No POST/PUT/DELETE handlers touch the health surface.""" diff --git a/web/local.html b/web/local.html index 7b958dc..4dc0ba6 100644 --- a/web/local.html +++ b/web/local.html @@ -223,6 +223,9 @@

Multiview

free Larix Broadcaster app, or paste the URL into OBS / ffmpeg.

+

Running a public demo? Send visitors the self-serve page: + /welcome ↗ + · /join ↗

diff --git a/web/welcome.html b/web/welcome.html new file mode 100644 index 0000000..eaa2a04 --- /dev/null +++ b/web/welcome.html @@ -0,0 +1,104 @@ + + + + + +MXL — a live switcher on one $400 box + + + +
+
Open-source · MXL / Dynamic Media Facility
+

A broadcast control room, as software

+

One $400 computer
is running a live show.

+

Real cameras, phone guests, playout, switching and streaming — all in shared memory on commodity hardware. Jump in: put your own camera on air, watch the program, or watch the machine work.

+

checking the facility…

+ +
+ +
📱
+

Put your phone on air

+

Scan a QR with the free Larix app and your camera becomes a live switchable source. ~30 seconds, no account.

+
Join the show →
+ + +
📺
+

Watch the program

+

The live program output, straight from the switcher over WebRTC — sub-second, encoded once for everyone.

+
Watch live →
+ + +
📊
+

Watch the machine

+

Live health: grain flow per source, CPU/memory, and viewer count — see exactly how little a $400 box breaks a sweat.

+
Open the monitor →
+ +
+ +

+ New to this? What's a "grain"? → +  ·  Built on the EBU/AMWA Media eXchange Layer. +

+
+ + + + From 6ff362b9f653194efded8502606bcab99e2a5dd0 Mon Sep 17 00:00:00 2001 From: Guy Cochran Date: Fri, 9 Oct 2026 02:21:11 +0000 Subject: [PATCH 12/37] site: add 'Try it LIVE' hero button on mxlswitcher.com -> live /welcome demo (put your phone on air) Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01GwSKEZZz4yEc2CGq5hcTmT --- docs/index.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.html b/docs/index.html index 5f90762..aabfd53 100644 --- a/docs/index.html +++ b/docs/index.html @@ -87,7 +87,7 @@

What if the switcher
was shared memory?

A working exploration of a live production control room built around the Media eXchange Layer: real cameras, remote guests, playout, layouts, graphics, audio, multiview, recording and standard control — assembled as software on general-purpose cloud VMs.

- See it in action + ▶ Try it LIVE — put your phone on air Explore the source Run it in ~10 minutes
From 5923faa105a18452a2349b982da26d5c20158215 Mon Sep 17 00:00:00 2001 From: Guy Cochran Date: Fri, 9 Oct 2026 04:40:02 +0000 Subject: [PATCH 13/37] feat(join): multiview return grid + auto-assign a FREE slot (fix 'my feed didn't appear') MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Why the phone didn't show: /join sent everyone to guest3 (:8895), but the PTZ already held that listener (SRT listen mode = one publisher per port) → the phone was rejected. - /api/mxl/ingest now annotates each slot free/occupied (from the probe snapshot) and returns suggested_slot = first FREE slot. /join uses it, so two scanners don't collide. - /join now shows a live MULTIVIEW return grid (source thumbnails, near-zero cost — the switcher already generates them): the visitor's OWN tile is ringed cyan + 'YOU' and the status flips to 'You're LIVE 🎥' the moment their source delivers frames. Plus the PGM preview below. This is the 'can we see you?' feedback — answers the design question: show the MULTIVIEW as the return (instant self-confirmation), PGM as 'what's on air'. - A true encoded multiview (mv_encode.py exists) stays the TD's pop-out option — overkill for /join; the thumbnail grid scales to unlimited join viewers. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01GwSKEZZz4yEc2CGq5hcTmT --- backend/local-server.js | 22 ++++++++- web/join.html | 103 +++++++++++++++++++++++++++++++++------- 2 files changed, 105 insertions(+), 20 deletions(-) diff --git a/backend/local-server.js b/backend/local-server.js index 4df7cfa..3a7b9fa 100644 --- a/backend/local-server.js +++ b/backend/local-server.js @@ -116,8 +116,26 @@ const INGEST_CFG = ingestConfig({ // The set of valid guest slots for the QR route — derived from the config, not hardcoded. const INGEST_SLOTS = new Set((INGEST_CFG.guests || []).map((g) => g.slot)); -app.get('/api/mxl/ingest', (req, res) => { - res.json(ingestInfo(INGEST_CFG)); +app.get('/api/mxl/ingest', async (req, res) => { + const info = ingestInfo(INGEST_CFG); + // Annotate each guest slot as free/occupied so the /join page can auto-assign the + // first FREE slot (two visitors must not be sent to the same busy listener — SRT + // listen mode is one-publisher-per-port, so a taken slot rejects the next phone). + // We derive "occupied" from the probe snapshot (a flow advancing = someone's on it). + try { + const snap = await readSnapshot('health.json', THUMBS_HEALTH_PATH); + const flows = (snap && snap.slots) || {}; + info.guests = info.guests.map((g) => { + const f = flows[g.slot]; + // occupied = the slot's source is delivering fresh frames (age present + small) + const occupied = !!(f && f.age != null && f.age < 5); + return { ...g, occupied, free: !occupied }; + }); + const firstFree = info.guests.find((g) => g.free); + info.suggested_slot = firstFree ? firstFree.slot : null; + } catch { /* no snapshot — leave slots unannotated */ } + res.set('Access-Control-Allow-Origin', process.env.MXL_HEALTH_CORS || '*'); + res.json(info); }); // Server-rendered QR PNG for a guest slot. Uses python3 + segno (pure-python, diff --git a/web/join.html b/web/join.html index 39ea24c..7eb899b 100644 --- a/web/join.html +++ b/web/join.html @@ -69,6 +69,19 @@ .status{margin-top:10px;font-size:13px;color:var(--dim)} .status b{color:var(--grn)} .status.waiting b{color:var(--amber)} + /* multiview return grid */ + .mv{display:grid;grid-template-columns:repeat(3,1fr);gap:8px} + @media(max-width:520px){.mv{grid-template-columns:repeat(2,1fr)}} + .mvtile{position:relative;background:#000;border:1px solid var(--line);border-radius:9px;overflow:hidden;aspect-ratio:16/9} + .mvtile img{width:100%;height:100%;object-fit:cover;display:block} + .mvtile .lbl{position:absolute;bottom:0;left:0;right:0;font-size:10px;padding:3px 6px;background:linear-gradient(transparent,rgba(0,0,0,.8));color:var(--ink);font-weight:600;letter-spacing:.5px} + .mvtile .dot{position:absolute;top:6px;right:6px;width:8px;height:8px;border-radius:50%;background:var(--faint)} + .mvtile.live .dot{background:var(--grn);box-shadow:0 0 8px rgba(35,217,124,.6)} + .mvtile.dead{opacity:.4} + .mvtile.you{border-color:var(--cyan);box-shadow:0 0 0 2px var(--cyan),0 0 18px rgba(53,224,255,.5)} + .mvtile.you .lbl{background:var(--cyan);color:#05070c} + .mvtile .me{position:absolute;top:6px;left:6px;font-size:9px;background:var(--cyan);color:#05070c;font-weight:700;padding:1px 6px;border-radius:4px;letter-spacing:.5px;display:none} + .mvtile.you .me{display:block} .footer{margin-top:30px;color:var(--faint);font-size:12px;border-top:1px solid var(--line);padding-top:16px} @@ -126,12 +139,17 @@

Put your phone's camera
into a live broadcast.

-

Program — you'll appear here

+

Can we see you? — your tile lights up when you're live

+
+
Waiting for a source… go live in Larix and watch your tile appear below.
+
+ +
+

Program output — what's on air

● LIVE
-
Waiting for a source… once you go live, ask the director to cut to you — or watch the sources count rise.
@@ -144,46 +162,95 @@

Program — you'll appear here

From 5053bad291b256f7cbd8ce57157c8f787a79889e Mon Sep 17 00:00:00 2001 From: Guy Cochran Date: Fri, 9 Oct 2026 04:55:23 +0000 Subject: [PATCH 14/37] feat(join): multiview grid shows all guest slots dynamically (guest4-6 + future) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The return-feed grid now orders known sources then appends any others the health reports, and rebuilds when the source set changes — so newly-added guest slots (guest4, guest5, …) appear automatically without a code change. Pairs with the box now running 6 guest ingest slots for public joiners. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01GwSKEZZz4yEc2CGq5hcTmT --- web/join.html | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/web/join.html b/web/join.html index 7eb899b..9b12739 100644 --- a/web/join.html +++ b/web/join.html @@ -200,19 +200,23 @@

Program output '+ ''+esc(pretty(n))+''+ 'YOU'+ ''+esc(pretty(n))+''; }).join(''); - builtGrid=true; + builtGrid=true; gridKeys=names.slice().sort().join(','); } function refreshThumbs(){ Array.prototype.forEach.call(document.querySelectorAll('.mvtile img'),function(img){ @@ -224,7 +228,8 @@

Program output Date: Fri, 9 Oct 2026 05:12:25 +0000 Subject: [PATCH 15/37] feat(skin): add Health + Join links to the Core top nav (read-only, for everyone) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Health is read-only telemetry useful to anyone watching the rack — surface it from the control Skin's header, alongside a Join link for quick QR hand-off. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01GwSKEZZz4yEc2CGq5hcTmT --- web/local.html | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/web/local.html b/web/local.html index 4dc0ba6..16f49e4 100644 --- a/web/local.html +++ b/web/local.html @@ -41,6 +41,8 @@ font-weight:500;letter-spacing:.5px;color:var(--dim);border:1px solid var(--line); border-radius:6px;padding:5px 9px;text-transform:uppercase} .chip .dot{width:7px;height:7px;border-radius:50%;background:var(--faint)} + a.navlink{text-decoration:none;color:var(--dim);transition:color .15s,border-color .15s} + a.navlink:hover{color:var(--acc);border-color:var(--acc)} .chip.ok{color:var(--ink)} .chip.ok .dot{background:var(--grn);box-shadow:0 0 7px var(--grn-glow)} .chip.bad .dot{background:var(--red);box-shadow:0 0 7px var(--red-glow)} .chip b{color:var(--ink);font-weight:600} @@ -156,6 +158,8 @@
MXL Switcher
+ Health ↗ + Join ↗
connecting…
PGM —
From bfae7d0b2bc77744a8fcba2444b53271af5d9fba Mon Sep 17 00:00:00 2001 From: Guy Cochran Date: Fri, 9 Oct 2026 05:21:31 +0000 Subject: [PATCH 16/37] =?UTF-8?q?feat(reliability):=20guest-leg-watchdog?= =?UTF-8?q?=20=E2=80=94=20heal=20wedged=20SRT=20ingest=20legs=20so=20phone?= =?UTF-8?q?=20joins=20always=20work?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The silent demo-killer (HW Oct 9): a guest ingest chain is fanout(public SRT) + video-core + audio-core; if EITHER internal leg listener dies, the fanout rejects the whole incoming connection and the phone gets 'Input/output error' with no clue why. Found guest1 AND guest2 audio legs dead, both blocking joins. tools/guest-leg-watchdog.sh polls each guest's 3 ports every 20s; a partial chain (1-2 of 3 up) = a wedged leg → restart the chain in dependency order (audio→core→fanout) with a per-chain cooldown. A fully-down chain = not provisioned → skipped (guest4-6 optional). +6 tests. Full suite green. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01GwSKEZZz4yEc2CGq5hcTmT --- tests/test_guest_leg_watchdog.py | 62 ++++++++++++++++++++++++++++ tools/guest-leg-watchdog.sh | 71 ++++++++++++++++++++++++++++++++ 2 files changed, 133 insertions(+) create mode 100644 tests/test_guest_leg_watchdog.py create mode 100644 tools/guest-leg-watchdog.sh diff --git a/tests/test_guest_leg_watchdog.py b/tests/test_guest_leg_watchdog.py new file mode 100644 index 0000000..275554e --- /dev/null +++ b/tests/test_guest_leg_watchdog.py @@ -0,0 +1,62 @@ +"""guest-leg-watchdog.sh invariants (text-scanned — it's bash, no deps). + +The watchdog heals the SRT-leg wedge that silently blocks phone joins: a guest ingest +chain is fanout(public) + core(video leg) + audio(audio leg), and if EITHER internal leg +is down the fanout rejects the whole incoming connection. These checks guard the safety +properties that keep the watchdog from thrashing or healing the wrong thing. +""" +import re +from pathlib import Path + +WD = Path(__file__).resolve().parent.parent / "tools" / "guest-leg-watchdog.sh" + + +def _src(): + assert WD.is_file(), "tools/guest-leg-watchdog.sh missing" + return WD.read_text() + + +def test_exists_and_bash_shebang(): + s = _src() + assert s.startswith("#!/usr/bin/env bash") or s.startswith("#!/bin/bash") + + +def test_restarts_chain_in_dependency_order(): + """The cores (leg listeners) must come up before the fanout tries to write to them, + or the fanout immediately re-wedges. Order must be audio, core, fanout.""" + s = _src() + m = re.search(r'docker restart\s+"\$\{name\}-audio"\s+"\$\{name\}"\s+"\$\{name\}-fanout"', s) + assert m, "heal must restart ${name}-audio ${name} ${name}-fanout in that order" + + +def test_has_cooldown_to_avoid_thrash(): + """A flapping chain must not be restarted every poll — a per-chain cooldown guards it.""" + s = _src() + assert "COOLDOWN_SECONDS" in s, "no cooldown guard" + assert "last_heal" in s, "cooldown must track last heal time per chain" + + +def test_skips_absent_guests(): + """guest4-6 may not exist on a smaller deployment — the watchdog must skip a guest + whose containers aren't present rather than trying to restart nothing.""" + s = _src() + assert "docker ps" in s and "grep -qx" in s, "must check the container exists before healing" + assert "continue" in s, "absent guest must be skipped (continue)" + + +def test_heals_only_partial_chains_not_fully_down(): + """0 ports up = not provisioned / still starting (skip); 1-2 up = a leg wedged (heal); + 3 up = healthy. Healing a fully-down chain would thrash a guest that's just absent.""" + s = _src() + assert re.search(r'"\$up"\s*-eq\s*1', s) and re.search(r'"\$up"\s*-eq\s*2', s), \ + "must heal when exactly 1 or 2 of 3 ports are up" + assert not re.search(r'"\$up"\s*-eq\s*0.*heal', s), "must NOT heal a fully-down (absent) chain" + + +def test_watches_all_six_guest_chains_by_default(): + """The default port map must cover guest1-6 with their real (non-contiguous) ports — + guest3 on :8895, not the computed :8892.""" + s = _src() + for spec in ("guest1:8890", "guest2:8891", "guest3:8895", + "guest4:8896", "guest5:8897", "guest6:8898"): + assert spec in s, f"default GUEST_PORTS missing {spec}" diff --git a/tools/guest-leg-watchdog.sh b/tools/guest-leg-watchdog.sh new file mode 100644 index 0000000..ffbabd1 --- /dev/null +++ b/tools/guest-leg-watchdog.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash +# guest-leg-watchdog — keeps each guest's SRT ingest chain healthy so a contributor +# (phone/Larix) can always connect. +# +# THE WEDGE IT HEALS (HW Oct 9): each guest ingest is a 3-container chain — +# -fanout (public SRT listener :PUB) → splits A/V to two internal legs +# (video core, listens :VLEG) → writes the guest VIDEO flow +# -audio (audio core, listens :ALEG) → writes the guest AUDIO flow +# guest_av_listen.sh writes to BOTH legs; if EITHER leg's listener is down, the +# fanout's SRT write fails ("Socket is broken or closed, Trying to reconnect") and it +# REJECTS the whole incoming connection — so a phone (and even a clean local caller) +# gets "Input/output error" and never appears. Seen live: guest2 audio leg :9091 dead, +# guest1 audio leg :9090 dead — both silently blocking joins. +# +# This watchdog polls every guest's three ports; if a chain is incomplete (public up but +# a leg missing, or vice versa) it restarts that chain in order (audio → core → fanout) +# and logs it. A chain with ALL ports down is treated as "not provisioned" and skipped +# (guest4-6 may legitimately not exist on a smaller deployment). +# +# Config (env-overridable; defaults match the box): +# GUEST_PORTS per-guest "name:pub:vleg:aleg" specs, comma-separated +# POLL_SECONDS default 20 +# COOLDOWN_SECONDS min seconds between restarts of the SAME chain (default 60) +set -u + +POLL_SECONDS=${POLL_SECONDS:-20} +COOLDOWN_SECONDS=${COOLDOWN_SECONDS:-60} + +# name:public:video-leg:audio-leg (the box's scheme; guest1/2 contiguous, 3-6 bolt-ons) +GUEST_PORTS=${GUEST_PORTS:-"guest1:8890:8990:9090,guest2:8891:8991:9091,guest3:8895:8995:9095,guest4:8896:8996:9096,guest5:8897:8997:9097,guest6:8898:8998:9098"} + +declare -A last_heal + +bound() { ss -ulnp 2>/dev/null | grep -q ":$1 "; } # is a UDP port listening? + +heal_chain() { + local name=$1 + local now; now=$(date +%s) + local last=${last_heal[$name]:-0} + if [ $((now - last)) -lt "$COOLDOWN_SECONDS" ]; then + logger -t guest-leg-watchdog "skip $name — in cooldown ($((now - last))s < ${COOLDOWN_SECONDS}s)" + return + fi + last_heal[$name]=$now + logger -t guest-leg-watchdog "HEALING $name chain (a leg was down) — restart audio→core→fanout" + # order matters: the cores (leg listeners) must be up before the fanout tries to write + docker restart "${name}-audio" "${name}" "${name}-fanout" >/dev/null 2>&1 +} + +logger -t guest-leg-watchdog "started — watching: $GUEST_PORTS (poll ${POLL_SECONDS}s)" +while :; do + IFS=',' read -ra specs <<< "$GUEST_PORTS" + for spec in "${specs[@]}"; do + IFS=':' read -r name pub vleg aleg <<< "$spec" + # skip guests that don't exist as containers on this deployment + docker ps --format '{{.Names}}' 2>/dev/null | grep -qx "${name}-fanout" || continue + + up=0 + bound "$pub" && up=$((up+1)) + bound "$vleg" && up=$((up+1)) + bound "$aleg" && up=$((up+1)) + + # 0 up = container exists but nothing bound (starting up) → give it a cycle, don't thrash + # 3 up = healthy. 1 or 2 up = a leg wedged → heal. + if [ "$up" -eq 1 ] || [ "$up" -eq 2 ]; then + logger -t guest-leg-watchdog "$name degraded: pub=$(bound "$pub" && echo up || echo DOWN) vleg=$(bound "$vleg" && echo up || echo DOWN) aleg=$(bound "$aleg" && echo up || echo DOWN)" + heal_chain "$name" + fi + done + sleep "$POLL_SECONDS" +done From 2ca4ac0c0b1290731d60fa2c84596f9371b85d73 Mon Sep 17 00:00:00 2001 From: Guy Cochran Date: Fri, 9 Oct 2026 05:24:35 +0000 Subject: [PATCH 17/37] fix(watchdog): require STRIKES consecutive degraded polls before healing (ignore idle-cycle blips) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An idle guest (no publisher) cycles its audio/video core — the SRT src exits 'not-linked' each loop and the leg listener briefly unbinds between cycles. That's NORMAL, not a wedge. The first watchdog healed on a single partial poll → it fought the idle cycle. Now a leg must stay partial across STRIKES (default 3 → ~60s) consecutive polls before a heal, and the strike counter resets on any healthy/fully-down poll. Cooldown raised to 90s. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01GwSKEZZz4yEc2CGq5hcTmT --- tools/guest-leg-watchdog.sh | 36 +++++++++++++++++++++++++++--------- 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/tools/guest-leg-watchdog.sh b/tools/guest-leg-watchdog.sh index ffbabd1..5a8768b 100644 --- a/tools/guest-leg-watchdog.sh +++ b/tools/guest-leg-watchdog.sh @@ -13,18 +13,28 @@ # guest1 audio leg :9090 dead — both silently blocking joins. # # This watchdog polls every guest's three ports; if a chain is incomplete (public up but -# a leg missing, or vice versa) it restarts that chain in order (audio → core → fanout) -# and logs it. A chain with ALL ports down is treated as "not provisioned" and skipped -# (guest4-6 may legitimately not exist on a smaller deployment). +# a leg missing, or vice versa) PERSISTENTLY it restarts that chain in order +# (audio → core → fanout) and logs it. A chain with ALL ports down is treated as "not +# provisioned" and skipped (guest4-6 may legitimately not exist on a smaller deployment). +# +# IMPORTANT — don't fight the idle cycle: when a guest has NO publisher, its audio/video +# core wrapper loops (`while :; do python3 guest_*.py ...; sleep 3; done`) and the SRT +# source exits "not-linked (-1)" each cycle, so the leg listener briefly unbinds between +# cycles. That's NORMAL for an idle slot — NOT a wedge. So we only heal a leg that stays +# down across STRIKES consecutive polls; a one-poll blip (the idle cycle) is ignored. # # Config (env-overridable; defaults match the box): # GUEST_PORTS per-guest "name:pub:vleg:aleg" specs, comma-separated # POLL_SECONDS default 20 -# COOLDOWN_SECONDS min seconds between restarts of the SAME chain (default 60) +# STRIKES consecutive degraded polls before healing (default 3 → ~60s of +# persistent degrade, well past any idle-cycle blip) +# COOLDOWN_SECONDS min seconds between restarts of the SAME chain (default 90) set -u POLL_SECONDS=${POLL_SECONDS:-20} -COOLDOWN_SECONDS=${COOLDOWN_SECONDS:-60} +STRIKES=${STRIKES:-3} +COOLDOWN_SECONDS=${COOLDOWN_SECONDS:-90} +declare -A strike # name:public:video-leg:audio-leg (the box's scheme; guest1/2 contiguous, 3-6 bolt-ons) GUEST_PORTS=${GUEST_PORTS:-"guest1:8890:8990:9090,guest2:8891:8991:9091,guest3:8895:8995:9095,guest4:8896:8996:9096,guest5:8897:8997:9097,guest6:8898:8998:9098"} @@ -60,11 +70,19 @@ while :; do bound "$vleg" && up=$((up+1)) bound "$aleg" && up=$((up+1)) - # 0 up = container exists but nothing bound (starting up) → give it a cycle, don't thrash - # 3 up = healthy. 1 or 2 up = a leg wedged → heal. + # 0 up = container exists but nothing bound (starting up / idle between cycles) → not + # a wedge; reset strikes. 3 up = healthy; reset. 1-2 up = PARTIAL — count a + # strike, and only heal once it's been partial for STRIKES consecutive polls + # (so the normal idle-cycle unbind, which clears within a poll, never heals). if [ "$up" -eq 1 ] || [ "$up" -eq 2 ]; then - logger -t guest-leg-watchdog "$name degraded: pub=$(bound "$pub" && echo up || echo DOWN) vleg=$(bound "$vleg" && echo up || echo DOWN) aleg=$(bound "$aleg" && echo up || echo DOWN)" - heal_chain "$name" + strike[$name]=$(( ${strike[$name]:-0} + 1 )) + logger -t guest-leg-watchdog "$name degraded (${strike[$name]}/${STRIKES}): pub=$(bound "$pub" && echo up || echo DOWN) vleg=$(bound "$vleg" && echo up || echo DOWN) aleg=$(bound "$aleg" && echo up || echo DOWN)" + if [ "${strike[$name]}" -ge "$STRIKES" ]; then + heal_chain "$name" + strike[$name]=0 + fi + else + strike[$name]=0 fi done sleep "$POLL_SECONDS" From 763f303b13d81634bc3001237c69ae40f1099e71 Mon Sep 17 00:00:00 2001 From: Guy Cochran Date: Fri, 9 Oct 2026 05:29:12 +0000 Subject: [PATCH 18/37] fix(watchdog): give up after GIVE_UP heals that don't stick (chronic flap = stop thrashing) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit guest1's audio leg flaps 'not-linked' because the Makito sends video-only — its audio core cycles forever and a restart can't fix it. Without a cap the watchdog would restart guest1 endlessly. Now: after GIVE_UP (4) heals that don't make it healthy, log + leave it alone; a chain that reaches 3/3 resets its count so genuine transient wedges always get healed. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01GwSKEZZz4yEc2CGq5hcTmT --- tests/test_guest_leg_watchdog.py | 10 ++++++++++ tools/guest-leg-watchdog.sh | 20 ++++++++++++++++++-- 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/tests/test_guest_leg_watchdog.py b/tests/test_guest_leg_watchdog.py index 275554e..0cf860c 100644 --- a/tests/test_guest_leg_watchdog.py +++ b/tests/test_guest_leg_watchdog.py @@ -53,6 +53,16 @@ def test_heals_only_partial_chains_not_fully_down(): assert not re.search(r'"\$up"\s*-eq\s*0.*heal', s), "must NOT heal a fully-down (absent) chain" +def test_gives_up_on_chronically_flapping_chain(): + """A leg that a restart can't fix (e.g. a video-only hardware encoder whose audio core + cycles 'not-linked' forever) must not be restarted endlessly — give up after GIVE_UP + heals and log, so it doesn't churn CPU. A chain going healthy resets the count.""" + s = _src() + assert "GIVE_UP" in s, "no give-up cap on repeated heals" + assert "GIVE UP" in s, "must log when abandoning a chronic chain" + assert "heal_count[$name]=0" in s, "a healthy chain must reset its heal count (forgive transient wedges)" + + def test_watches_all_six_guest_chains_by_default(): """The default port map must cover guest1-6 with their real (non-contiguous) ports — guest3 on :8895, not the computed :8892.""" diff --git a/tools/guest-leg-watchdog.sh b/tools/guest-leg-watchdog.sh index 5a8768b..47b75d7 100644 --- a/tools/guest-leg-watchdog.sh +++ b/tools/guest-leg-watchdog.sh @@ -40,6 +40,13 @@ declare -A strike GUEST_PORTS=${GUEST_PORTS:-"guest1:8890:8990:9090,guest2:8891:8991:9091,guest3:8895:8995:9095,guest4:8896:8996:9096,guest5:8897:8997:9097,guest6:8898:8998:9098"} declare -A last_heal +declare -A heal_count +# GIVE_UP: after this many heals that DON'T make a chain healthy, stop restarting it and +# just log — some legs flap for reasons a restart can't fix (e.g. a hardware encoder that +# sends video-only, so its audio core cycles 'not-linked' forever). Thrashing it endlessly +# only churns CPU. A chain that goes healthy resets its count, so transient wedges always +# get healed; only a chronic one is abandoned. +GIVE_UP=${GIVE_UP:-4} bound() { ss -ulnp 2>/dev/null | grep -q ":$1 "; } # is a UDP port listening? @@ -51,8 +58,14 @@ heal_chain() { logger -t guest-leg-watchdog "skip $name — in cooldown ($((now - last))s < ${COOLDOWN_SECONDS}s)" return fi + heal_count[$name]=$(( ${heal_count[$name]:-0} + 1 )) + if [ "${heal_count[$name]}" -gt "$GIVE_UP" ]; then + logger -t guest-leg-watchdog "GIVE UP on $name — healed ${heal_count[$name]}x without it staying healthy; leaving it alone (restart won't fix). Investigate manually." + last_heal[$name]=$now # keep cooldown so the give-up message doesn't spam + return + fi last_heal[$name]=$now - logger -t guest-leg-watchdog "HEALING $name chain (a leg was down) — restart audio→core→fanout" + logger -t guest-leg-watchdog "HEALING $name chain (heal #${heal_count[$name]}) — restart audio→core→fanout" # order matters: the cores (leg listeners) must be up before the fanout tries to write docker restart "${name}-audio" "${name}" "${name}-fanout" >/dev/null 2>&1 } @@ -81,8 +94,11 @@ while :; do heal_chain "$name" strike[$name]=0 fi - else + elif [ "$up" -eq 3 ]; then strike[$name]=0 + heal_count[$name]=0 # healthy again → forgive past heals (transient wedge recovered) + else + strike[$name]=0 # 0 up = absent/starting; don't count heals against it fi done sleep "$POLL_SECONDS" From 003a7cfc5627831d1bd13ee6cef5a0f8ee80552c Mon Sep 17 00:00:00 2001 From: Guy Cochran Date: Fri, 9 Oct 2026 05:51:15 +0000 Subject: [PATCH 19/37] =?UTF-8?q?fix(watchdog):=20rewrite=20conservative?= =?UTF-8?q?=20=E2=80=94=20fanout-only,=20never=20touch=20live-video=20chai?= =?UTF-8?q?ns,=20OFF=20by=20default?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit HARD LESSON (HW Oct 9): the first watchdog restarted whole guest chains for leg flaps. A chain restart recreates MXL flows → the input-selector re-wires → doing that every few seconds CHURNED the selector (router thread restarting in a loop) → PGM stuck, cuts failed, multiview flapping, and the selector API went unresponsive (HTTP 000). The watchdog built to help was breaking the facility. Rewrite is minimal + safe: - watches ONLY the public SRT listener (what a contributor connects to) - NEVER restarts a chain whose video flow is advancing (live source = hands off) - heal restarts ONLY the fanout, not the cores (so flows aren't recreated → no selector churn) - acts only when public listener is down AND no live video, across STRIKES polls - OFF by default (ENABLE=1 to arm); give-up + 300s cooldown Tests rewritten to lock these invariants. Full suite green. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01GwSKEZZz4yEc2CGq5hcTmT --- tests/test_guest_leg_watchdog.py | 67 +++++++-------- tools/guest-leg-watchdog.sh | 143 +++++++++++++++---------------- 2 files changed, 100 insertions(+), 110 deletions(-) diff --git a/tests/test_guest_leg_watchdog.py b/tests/test_guest_leg_watchdog.py index 0cf860c..4ef2970 100644 --- a/tests/test_guest_leg_watchdog.py +++ b/tests/test_guest_leg_watchdog.py @@ -1,9 +1,11 @@ """guest-leg-watchdog.sh invariants (text-scanned — it's bash, no deps). -The watchdog heals the SRT-leg wedge that silently blocks phone joins: a guest ingest -chain is fanout(public) + core(video leg) + audio(audio leg), and if EITHER internal leg -is down the fanout rejects the whole incoming connection. These checks guard the safety -properties that keep the watchdog from thrashing or healing the wrong thing. +HARD LESSON baked into these tests (HW Oct 9): restarting a guest chain recreates its MXL +flows → the input-selector re-wires → if that happens repeatedly the selector churns and +the whole facility destabilizes (PGM sticks, cuts fail, thumbnails flap). The first +watchdog restarted whole chains on any leg blip and broke the facility. So the watchdog is +now deliberately MINIMAL: it only watches the PUBLIC listener, never touches a chain with +live video, restarts ONLY the fanout (not the cores), and is OFF by default. """ import re from pathlib import Path @@ -21,52 +23,49 @@ def test_exists_and_bash_shebang(): assert s.startswith("#!/usr/bin/env bash") or s.startswith("#!/bin/bash") -def test_restarts_chain_in_dependency_order(): - """The cores (leg listeners) must come up before the fanout tries to write to them, - or the fanout immediately re-wedges. Order must be audio, core, fanout.""" +def test_off_by_default(): + """It's a sharp tool — must do nothing unless deliberately enabled (ENABLE=1).""" s = _src() - m = re.search(r'docker restart\s+"\$\{name\}-audio"\s+"\$\{name\}"\s+"\$\{name\}-fanout"', s) - assert m, "heal must restart ${name}-audio ${name} ${name}-fanout in that order" + assert re.search(r'ENABLE=\$\{ENABLE:-0\}', s), "watchdog must default ENABLE=0 (observe-only)" + assert 'ENABLE" = "1"' in s or "ENABLE\" == \"1\"" in s, "heal must be gated on ENABLE=1" -def test_has_cooldown_to_avoid_thrash(): - """A flapping chain must not be restarted every poll — a per-chain cooldown guards it.""" +def test_never_touches_a_chain_with_live_video(): + """The bug that broke the facility: restarting a chain whose video is live churns the + selector. The watchdog MUST skip a chain whose video flow is advancing.""" s = _src() - assert "COOLDOWN_SECONDS" in s, "no cooldown guard" - assert "last_heal" in s, "cooldown must track last heal time per chain" + assert "video_advancing" in s, "no live-video guard" + # the guard must cause a skip/continue, not a heal + assert re.search(r'video_advancing "\$vuuid".*;\s*then\s*strike\[\$name\]=0;\s*continue', s, re.S), \ + "a chain with advancing video must be skipped, never healed" -def test_skips_absent_guests(): - """guest4-6 may not exist on a smaller deployment — the watchdog must skip a guest - whose containers aren't present rather than trying to restart nothing.""" +def test_restarts_only_the_fanout_not_the_cores(): + """Restarting the cores recreates the MXL flows → selector churn. Heal must restart ONLY + the public-listener fanout, leaving the cores (and their flows) untouched.""" s = _src() - assert "docker ps" in s and "grep -qx" in s, "must check the container exists before healing" - assert "continue" in s, "absent guest must be skipped (continue)" + assert re.search(r'docker restart "\$\{name\}-fanout"', s), "heal must restart the fanout" + assert not re.search(r'docker restart "\$\{name\}-audio"\s+"\$\{name\}"', s), \ + "heal must NOT restart the cores (that recreates flows and churns the selector)" -def test_heals_only_partial_chains_not_fully_down(): - """0 ports up = not provisioned / still starting (skip); 1-2 up = a leg wedged (heal); - 3 up = healthy. Healing a fully-down chain would thrash a guest that's just absent.""" +def test_only_acts_on_a_dead_public_listener(): + """The only thing that silently blocks a join is the PUBLIC listener being down on an + idle slot. Internal leg flaps are explicitly NOT the watchdog's concern anymore.""" s = _src() - assert re.search(r'"\$up"\s*-eq\s*1', s) and re.search(r'"\$up"\s*-eq\s*2', s), \ - "must heal when exactly 1 or 2 of 3 ports are up" - assert not re.search(r'"\$up"\s*-eq\s*0.*heal', s), "must NOT heal a fully-down (absent) chain" + assert re.search(r'if bound "\$pub"; then', s), "must key off the public listener being up" -def test_gives_up_on_chronically_flapping_chain(): - """A leg that a restart can't fix (e.g. a video-only hardware encoder whose audio core - cycles 'not-linked' forever) must not be restarted endlessly — give up after GIVE_UP - heals and log, so it doesn't churn CPU. A chain going healthy resets the count.""" +def test_has_strikes_cooldown_and_giveup(): s = _src() - assert "GIVE_UP" in s, "no give-up cap on repeated heals" - assert "GIVE UP" in s, "must log when abandoning a chronic chain" - assert "heal_count[$name]=0" in s, "a healthy chain must reset its heal count (forgive transient wedges)" + assert "STRIKES" in s and "COOLDOWN_SECONDS" in s and "GIVE_UP" in s, \ + "must have strikes + cooldown + give-up guards against thrash" -def test_watches_all_six_guest_chains_by_default(): - """The default port map must cover guest1-6 with their real (non-contiguous) ports — - guest3 on :8895, not the computed :8892.""" +def test_watches_all_six_guest_chains_with_real_ports(): + """Default map covers guest1-6 on their REAL (non-contiguous) public ports + video UUIDs.""" s = _src() for spec in ("guest1:8890", "guest2:8891", "guest3:8895", "guest4:8896", "guest5:8897", "guest6:8898"): assert spec in s, f"default GUEST_PORTS missing {spec}" + assert "9e333e00" in s, "guest3 video UUID must be in the map (for the live-video check)" diff --git a/tools/guest-leg-watchdog.sh b/tools/guest-leg-watchdog.sh index 47b75d7..49c78ff 100644 --- a/tools/guest-leg-watchdog.sh +++ b/tools/guest-leg-watchdog.sh @@ -1,104 +1,95 @@ #!/usr/bin/env bash -# guest-leg-watchdog — keeps each guest's SRT ingest chain healthy so a contributor -# (phone/Larix) can always connect. +# guest-leg-watchdog — conservatively keeps a guest's PUBLIC SRT listener available so a +# contributor (phone/Larix) can connect to a FREE slot. # -# THE WEDGE IT HEALS (HW Oct 9): each guest ingest is a 3-container chain — -# -fanout (public SRT listener :PUB) → splits A/V to two internal legs -# (video core, listens :VLEG) → writes the guest VIDEO flow -# -audio (audio core, listens :ALEG) → writes the guest AUDIO flow -# guest_av_listen.sh writes to BOTH legs; if EITHER leg's listener is down, the -# fanout's SRT write fails ("Socket is broken or closed, Trying to reconnect") and it -# REJECTS the whole incoming connection — so a phone (and even a clean local caller) -# gets "Input/output error" and never appears. Seen live: guest2 audio leg :9091 dead, -# guest1 audio leg :9090 dead — both silently blocking joins. +# ⚠️ HARD LESSON (HW Oct 9): restarting a guest chain RECREATES its MXL flows, which forces +# the input-selector to RE-WIRE (router thread restarts). Do that every few seconds and the +# selector never stabilizes → PGM sticks, cuts fail, multiview thumbnails flap. The first +# version of this watchdog restarted a chain whenever any internal leg blipped (which is +# NORMAL for an idle slot, and ALSO for a video-only hardware encoder whose audio core +# cycles 'not-linked' forever) — so it churned the selector and broke the whole facility. # -# This watchdog polls every guest's three ports; if a chain is incomplete (public up but -# a leg missing, or vice versa) PERSISTENTLY it restarts that chain in order -# (audio → core → fanout) and logs it. A chain with ALL ports down is treated as "not -# provisioned" and skipped (guest4-6 may legitimately not exist on a smaller deployment). +# So this version is deliberately MINIMAL and SAFE: +# • It only cares about the PUBLIC listener (:PUB) — the thing a contributor connects to. +# Internal legs flapping is NOT our business (that's the idle cycle / source quirks). +# • It NEVER restarts a chain whose VIDEO flow is advancing (a live source → hands off). +# • It only restarts a chain that is BOTH: public port missing AND video flow not +# advancing AND stayed that way across STRIKES consecutive polls — i.e. a genuinely +# dead listener on an idle slot, the only case that silently blocks a new join. +# • It gives up after GIVE_UP restarts that don't help, and has a long cooldown. +# • OFF by default. Enable deliberately: ENABLE=1 (or the systemd unit sets it). # -# IMPORTANT — don't fight the idle cycle: when a guest has NO publisher, its audio/video -# core wrapper loops (`while :; do python3 guest_*.py ...; sleep 3; done`) and the SRT -# source exits "not-linked (-1)" each cycle, so the leg listener briefly unbinds between -# cycles. That's NORMAL for an idle slot — NOT a wedge. So we only heal a leg that stays -# down across STRIKES consecutive polls; a one-poll blip (the idle cycle) is ignored. -# -# Config (env-overridable; defaults match the box): -# GUEST_PORTS per-guest "name:pub:vleg:aleg" specs, comma-separated -# POLL_SECONDS default 20 -# STRIKES consecutive degraded polls before healing (default 3 → ~60s of -# persistent degrade, well past any idle-cycle blip) -# COOLDOWN_SECONDS min seconds between restarts of the SAME chain (default 90) +# Honestly: on this box the better fix is usually to leave guests alone and let a CONNECTING +# contributor's SRT handshake wake the listener. Use this only if idle listeners are +# observed to die outright. Config (env): +# ENABLE=1 required to do anything (default 0 = observe-only, just logs) +# GUEST_PORTS "name:pub:vleg:aleg" specs, comma-separated +# POLL_SECONDS=30 STRIKES=4 GIVE_UP=3 COOLDOWN_SECONDS=300 set -u -POLL_SECONDS=${POLL_SECONDS:-20} -STRIKES=${STRIKES:-3} -COOLDOWN_SECONDS=${COOLDOWN_SECONDS:-90} -declare -A strike +ENABLE=${ENABLE:-0} +POLL_SECONDS=${POLL_SECONDS:-30} +STRIKES=${STRIKES:-4} +GIVE_UP=${GIVE_UP:-3} +COOLDOWN_SECONDS=${COOLDOWN_SECONDS:-300} +SELECTOR_CONTAINER=${SELECTOR_CONTAINER:-input-selector} +DOMAIN=${MXL_DOMAIN:-/mxl-domain} +MXL_INFO=${MXL_INFO:-/opt/mxl/tools/mxl-info/mxl-info} + +# name:public:video-leg:audio-leg : video-flow-uuid +GUEST_PORTS=${GUEST_PORTS:-"guest1:8890:8990:9090:9e111e00-aaaa-4bbb-8ccc-000000000001,guest2:8891:8991:9091:9e222e00-aaaa-4bbb-8ccc-000000000001,guest3:8895:8995:9095:9e333e00-aaaa-4bbb-8ccc-000000000001,guest4:8896:8996:9096:9e444e00-aaaa-4bbb-8ccc-000000000001,guest5:8897:8997:9097:9e555e00-aaaa-4bbb-8ccc-000000000001,guest6:8898:8998:9098:9e666e00-aaaa-4bbb-8ccc-000000000001"} -# name:public:video-leg:audio-leg (the box's scheme; guest1/2 contiguous, 3-6 bolt-ons) -GUEST_PORTS=${GUEST_PORTS:-"guest1:8890:8990:9090,guest2:8891:8991:9091,guest3:8895:8995:9095,guest4:8896:8996:9096,guest5:8897:8997:9097,guest6:8898:8998:9098"} +declare -A last_heal heal_count strike -declare -A last_heal -declare -A heal_count -# GIVE_UP: after this many heals that DON'T make a chain healthy, stop restarting it and -# just log — some legs flap for reasons a restart can't fix (e.g. a hardware encoder that -# sends video-only, so its audio core cycles 'not-linked' forever). Thrashing it endlessly -# only churns CPU. A chain that goes healthy resets its count, so transient wedges always -# get healed; only a chronic one is abandoned. -GIVE_UP=${GIVE_UP:-4} +bound() { ss -ulnp 2>/dev/null | grep -q ":$1 "; } -bound() { ss -ulnp 2>/dev/null | grep -q ":$1 "; } # is a UDP port listening? +head_idx() { # current head index of a flow, or empty + docker exec "$SELECTOR_CONTAINER" sh -c "$MXL_INFO -d $DOMAIN -f $1 2>/dev/null" 2>/dev/null \ + | grep -i 'Head index' | grep -oE '[0-9]+' | head -1 +} +video_advancing() { # is the guest's VIDEO flow moving? (a live source — never touch it) + local uuid=$1 a b + a=$(head_idx "$uuid"); sleep 1; b=$(head_idx "$uuid") + [ -n "$a" ] && [ -n "$b" ] && [ "$b" -gt "$a" ] 2>/dev/null +} heal_chain() { - local name=$1 - local now; now=$(date +%s) + local name=$1 now; now=$(date +%s) local last=${last_heal[$name]:-0} - if [ $((now - last)) -lt "$COOLDOWN_SECONDS" ]; then - logger -t guest-leg-watchdog "skip $name — in cooldown ($((now - last))s < ${COOLDOWN_SECONDS}s)" - return - fi + [ $((now - last)) -lt "$COOLDOWN_SECONDS" ] && { logger -t guest-leg-watchdog "skip $name — cooldown"; return; } heal_count[$name]=$(( ${heal_count[$name]:-0} + 1 )) if [ "${heal_count[$name]}" -gt "$GIVE_UP" ]; then - logger -t guest-leg-watchdog "GIVE UP on $name — healed ${heal_count[$name]}x without it staying healthy; leaving it alone (restart won't fix). Investigate manually." - last_heal[$name]=$now # keep cooldown so the give-up message doesn't spam - return + logger -t guest-leg-watchdog "GIVE UP on $name (${heal_count[$name]}x) — restart won't fix; leaving alone" + last_heal[$name]=$now; return fi last_heal[$name]=$now - logger -t guest-leg-watchdog "HEALING $name chain (heal #${heal_count[$name]}) — restart audio→core→fanout" - # order matters: the cores (leg listeners) must be up before the fanout tries to write - docker restart "${name}-audio" "${name}" "${name}-fanout" >/dev/null 2>&1 + logger -t guest-leg-watchdog "HEAL $name public listener (heal #${heal_count[$name]}) — restart fanout only" + # Restart ONLY the fanout (the public listener). Leave the cores alone so we don't + # recreate the MXL flows and churn the selector. + docker restart "${name}-fanout" >/dev/null 2>&1 } -logger -t guest-leg-watchdog "started — watching: $GUEST_PORTS (poll ${POLL_SECONDS}s)" +logger -t guest-leg-watchdog "started (ENABLE=$ENABLE, poll ${POLL_SECONDS}s) — watching PUBLIC listeners only; never touches a chain with live video" while :; do IFS=',' read -ra specs <<< "$GUEST_PORTS" for spec in "${specs[@]}"; do - IFS=':' read -r name pub vleg aleg <<< "$spec" - # skip guests that don't exist as containers on this deployment + IFS=':' read -r name pub vleg aleg vuuid <<< "$spec" docker ps --format '{{.Names}}' 2>/dev/null | grep -qx "${name}-fanout" || continue - up=0 - bound "$pub" && up=$((up+1)) - bound "$vleg" && up=$((up+1)) - bound "$aleg" && up=$((up+1)) + # Only concern: is the PUBLIC listener up? If yes, a contributor can connect → done. + if bound "$pub"; then strike[$name]=0; heal_count[$name]=0; continue; fi + + # Public listener is DOWN. If the video flow is advancing, a source is live on this + # slot anyway — do NOT touch it (restarting would churn the selector). + if video_advancing "$vuuid"; then strike[$name]=0; continue; fi - # 0 up = container exists but nothing bound (starting up / idle between cycles) → not - # a wedge; reset strikes. 3 up = healthy; reset. 1-2 up = PARTIAL — count a - # strike, and only heal once it's been partial for STRIKES consecutive polls - # (so the normal idle-cycle unbind, which clears within a poll, never heals). - if [ "$up" -eq 1 ] || [ "$up" -eq 2 ]; then - strike[$name]=$(( ${strike[$name]:-0} + 1 )) - logger -t guest-leg-watchdog "$name degraded (${strike[$name]}/${STRIKES}): pub=$(bound "$pub" && echo up || echo DOWN) vleg=$(bound "$vleg" && echo up || echo DOWN) aleg=$(bound "$aleg" && echo up || echo DOWN)" - if [ "${strike[$name]}" -ge "$STRIKES" ]; then - heal_chain "$name" - strike[$name]=0 - fi - elif [ "$up" -eq 3 ]; then + # Public down AND no live video = an idle slot whose listener died = a real join block. + strike[$name]=$(( ${strike[$name]:-0} + 1 )) + logger -t guest-leg-watchdog "$name public listener :$pub DOWN, no live video (${strike[$name]}/${STRIKES})" + if [ "${strike[$name]}" -ge "$STRIKES" ]; then + if [ "$ENABLE" = "1" ]; then heal_chain "$name"; else + logger -t guest-leg-watchdog "$name would heal, but ENABLE!=1 (observe-only)"; fi strike[$name]=0 - heal_count[$name]=0 # healthy again → forgive past heals (transient wedge recovered) - else - strike[$name]=0 # 0 up = absent/starting; don't count heals against it fi done sleep "$POLL_SECONDS" From 1f9b5ed7051b00a258adc4074b0454ec43803689 Mon Sep 17 00:00:00 2001 From: Guy Cochran Date: Fri, 9 Oct 2026 06:01:16 +0000 Subject: [PATCH 20/37] security(review): remove live hostname from docs + bind mediamtx API to localhost MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review follow-ups on #41: - docs/index.html + docs/what-are-grains.html no longer hardcode the live server hostname (placeholder YOUR-SWITCHER-HOST; health page reads window.MXL_HEALTH_API override). The live mxlswitcher.com public/ copies keep the real URL (box-specific, like facility.json). - quickstart: mediamtx API now binds 127.0.0.1:9997 (MTX_APIADDRESS), not *:9997 — with --network host the default exposed an unauthenticated control API on the LAN. The Core reads it via localhost. Applied to the live box too. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01GwSKEZZz4yEc2CGq5hcTmT --- docs/index.html | 2 +- docs/what-are-grains.html | 3 ++- scripts/quickstart.sh | 12 ++++++++---- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/docs/index.html b/docs/index.html index aabfd53..e3b79b3 100644 --- a/docs/index.html +++ b/docs/index.html @@ -87,7 +87,7 @@

What if the switcher
was shared memory?

A working exploration of a live production control room built around the Media eXchange Layer: real cameras, remote guests, playout, layouts, graphics, audio, multiview, recording and standard control — assembled as software on general-purpose cloud VMs.

- ▶ Try it LIVE — put your phone on air + ▶ Try it LIVE — put your phone on air Explore the source Run it in ~10 minutes
diff --git a/docs/what-are-grains.html b/docs/what-are-grains.html index 26a5742..194cf2a 100644 --- a/docs/what-are-grains.html +++ b/docs/what-are-grains.html @@ -87,7 +87,8 @@

The point, for the skeptics