diff --git a/backend/health-info.js b/backend/health-info.js new file mode 100644 index 0000000..8384073 --- /dev/null +++ b/backend/health-info.js @@ -0,0 +1,249 @@ +// backend/health-info.js — read-only health aggregation for the Health Skin. +// +// This is deliberately SEPARATE from the control plane (mxl-routes.js). The Core +// Skin (web/local.html) is what a TD touches on air — cut/arm/take. Grain +// internals, CPU/mem and pipeline-writer state belong on their own read-only +// page (web/health.html, served at /health). Nothing here mutates the facility. +// +// Cost note: the expensive part — measuring per-flow grain rate AND unique-frames +// /s (to catch repeat-wedges where the head advances but the picture is frozen) — +// is already done by tools/grain_probe.py, which writes a rolling-window snapshot +// to /grains.json every 3s and has hard load guards (pauses >26, exits +// >29). We only READ that file here, so N health viewers add ~one file read each, +// never a container spawn. The live PGM/encode path is never touched. + +'use strict'; + +const fs = require('fs'); + +// ── System: CPU / mem / load from /proc (Linux). Pure reads, microseconds. ── +// CPU % needs two samples of /proc/stat; the caller holds the previous snapshot +// so a single scrape is cheap and the delta is computed across polls. +function readProcStat(readFile = fs.readFileSync) { + // Returns {total, idle} jiffies for the aggregate "cpu " line, or null. + try { + const line = String(readFile('/proc/stat', 'utf8')).split('\n') + .find((l) => l.startsWith('cpu ')); + if (!line) return null; + const n = line.trim().split(/\s+/).slice(1).map(Number); + // user nice system idle iowait irq softirq steal ... + const idle = (n[3] || 0) + (n[4] || 0); // idle + iowait + const total = n.reduce((a, b) => a + (Number.isFinite(b) ? b : 0), 0); + return { total, idle }; + } catch { return null; } +} + +function cpuPercent(prev, cur) { + if (!prev || !cur) return null; + const dt = cur.total - prev.total; + const di = cur.idle - prev.idle; + if (dt <= 0) return null; + return Math.max(0, Math.min(100, Math.round((1 - di / dt) * 100))); +} + +function readMem(readFile = fs.readFileSync) { + try { + const info = {}; + String(readFile('/proc/meminfo', 'utf8')).split('\n').forEach((l) => { + const m = l.match(/^(\w+):\s+(\d+)/); + if (m) info[m[1]] = Number(m[2]); // kB + }); + const totalKb = info.MemTotal || 0; + const availKb = info.MemAvailable != null ? info.MemAvailable : info.MemFree || 0; + if (!totalKb) return null; + const usedKb = totalKb - availKb; + return { + total_mb: Math.round(totalKb / 1024), + used_mb: Math.round(usedKb / 1024), + used_pct: Math.round((usedKb / totalKb) * 100), + }; + } catch { return null; } +} + +function readLoad(readFile = fs.readFileSync) { + try { + const p = String(readFile('/proc/loadavg', 'utf8')).trim().split(/\s+/); + return { load1: Number(p[0]), load5: Number(p[1]), load15: Number(p[2]) }; + } catch { return null; } +} + +function cpuCount(readFile = fs.readFileSync) { + try { + return String(readFile('/proc/stat', 'utf8')).split('\n') + .filter((l) => /^cpu\d+ /.test(l)).length || null; + } catch { return null; } +} + +// ── Grain flow: read grain_probe's snapshot (already on disk, atomic-renamed). ─ +// Shape written by tools/grain_probe.py: {ts, flows: {name: {bps, ufps}|null}}. +// We classify each flow so the UI can paint green/amber/red without logic: +// dead — null (probe read nothing in the window) +// frozen — bps healthy but ufps collapsed (repeat-wedge: head moves, picture +// doesn't). This is the exact keyer-bypass symptom we hit live. +// ok — bps and ufps both near the expected rate +// expectedRate defaults to 30 (the facility is 1080p30). stale = snapshot older +// than staleAfterS (probe died / not running). +function classifyGrains(snapshot, { expectedRate = 30, nowS, staleAfterS = 12 } = {}) { + if (!snapshot || typeof snapshot !== 'object' || !snapshot.flows) { + return { stale: true, age_s: null, flows: {} }; + } + const now = nowS != null ? nowS : Date.now() / 1000; + const ageS = snapshot.ts != null ? Math.max(0, now - snapshot.ts) : null; + const stale = ageS == null ? true : ageS > staleAfterS; + const flows = {}; + for (const [name, v] of Object.entries(snapshot.flows)) { + if (!v || v.bps == null) { + flows[name] = { state: 'dead', bps: 0, ufps: 0 }; + continue; + } + const bps = v.bps; + const ufps = v.ufps != null ? v.ufps : null; + // frozen: grains tick at ~rate but unique frames are a fraction of them. + // Use 60% of expected as the unique-fps floor for "moving". + let state = 'ok'; + if (bps < expectedRate * 0.5) state = 'slow'; + if (ufps != null && ufps < expectedRate * 0.6 && bps >= expectedRate * 0.5) state = 'frozen'; + flows[name] = { state, bps, ufps }; + } + return { stale, age_s: ageS != null ? Math.round(ageS * 10) / 10 : null, flows }; +} + +function readGrainsSnapshot(grainsPath, readFile = fs.readFileSync) { + try { return JSON.parse(String(readFile(grainsPath, 'utf8'))); } + catch { return null; } +} + +// ── Thumbs health.json (the probe that's ALREADY running on the box) ────────── +// tools/mxl_thumbs.py writes health.json beside the thumbnails — same cadence, no +// extra process — with per-slot {age, frozen} where: +// age = seconds since the LAST frame arrived (delivery stalled -> dead) +// frozen = seconds since the CONTENT last CHANGED (picture stuck -> frozen) +// This is the cheaper, already-live source for the Health Skin: it carries the SAME +// frozen-reader detection as grain_probe's unique-fps, plus system + viewers, so we +// don't run a second set of MXL readers. classifyGrains (grains.json / grain_probe) +// stays supported for adopters who run that probe instead — both normalize to the +// same shape the UI consumes. +// staleAfterS default is 25: tools/mxl_thumbs.py's health loop writes on a ~15s +// cadence (its slots[].age/frozen are themselves sub-2s fresh, but the summary file +// is rewritten every 15s), so a poll landing late in that cycle can legitimately see +// an ~18-20s-old file — only flag stale past 25s (probe actually dead). grain_probe's +// grains.json is 3s — callers using it can pass a tighter staleAfterS. +function classifyThumbsHealth(h, { nowS, staleAfterS = 25, frozenAfterS = 5, deadAfterS = 5 } = {}) { + if (!h || typeof h !== 'object' || !h.slots) { + return { stale: true, age_s: null, flows: {}, system: null }; + } + const now = nowS != null ? nowS : Date.now() / 1000; + const ageS = h.ts != null ? Math.max(0, now - h.ts) : null; + const stale = ageS == null ? true : ageS > staleAfterS; + const flows = {}; + for (const [name, v] of Object.entries(h.slots)) { + // v == null / age == null means the PROBE has no reader for this slot (rebuilding, + // or it couldn't attach) — NOT that the source is confirmed dead. Report 'unknown' so + // the UI doesn't paint a live-but-unprobed camera red. A slot with real stale `age` + // data below IS classified dead/frozen. (HW Oct 10: the thumbs reader for a live + // Makito/PTZ can sit null while mxl-info shows the flow advancing — false-dead fixed.) + if (!v || v.age == null) { flows[name] = { state: 'unknown', age: null, frozen: null }; continue; } + let state = 'ok'; + if (v.age > deadAfterS) state = 'dead'; // no new frames at all + else if (v.frozen != null && v.frozen > frozenAfterS) state = 'frozen'; // frames arrive, picture stuck + flows[name] = { state, age: v.age, frozen: v.frozen != null ? v.frozen : null }; + } + // System block, assembled from the same file (load/mem are host-true in the probe). + const system = { + load: (h.load1 != null) ? { load1: h.load1, load5: h.load5, load15: h.load15 } : null, + cpus: h.cores != null ? h.cores : null, + mem: (h.mem_total_mb != null && h.mem_avail_mb != null) ? { + total_mb: h.mem_total_mb, + used_mb: h.mem_total_mb - h.mem_avail_mb, + used_pct: Math.round(((h.mem_total_mb - h.mem_avail_mb) / h.mem_total_mb) * 100), + } : null, + viewers: (h.viewers && typeof h.viewers === 'object') ? (h.viewers.count != null ? h.viewers.count : null) + : (typeof h.viewers === 'number' ? h.viewers : null), + }; + return { stale, age_s: ageS != null ? Math.round(ageS * 10) / 10 : null, flows, system, procs: h.procs || null }; +} + +// ── Pipeline writers: normalize the three pipeline/status payloads for the UI. ─ +// Pure — the caller fetches the JSON (or passes nulls when a port is unreachable) +// and we shape it. No network here, so it's unit-testable. +function shapePipelines({ selector, keyer, encoder } = {}) { + const sel = selector || {}; + const key = keyer || {}; + const enc = encoder || {}; + const fmt = (f) => (f && f.format + ? { + w: f.format.frame_width, h: f.format.frame_height, + rate: f.format.grain_rate + ? Math.round((f.format.grain_rate.numerator || 0) / (f.format.grain_rate.denominator || 1)) + : null, + } + : null); + return { + selector: selector ? { + running: !!sel.running, error: sel.error || null, + active_input: sel.active_input != null ? sel.active_input : null, + inputs: Array.isArray(sel.input_flow_uuids) ? sel.input_flow_uuids.length : null, + input_flow_uuids: Array.isArray(sel.input_flow_uuids) ? sel.input_flow_uuids : [], + format: fmt(sel), + } : { running: false, unreachable: true }, + keyer: keyer ? { + running: !!key.running, error: key.error || null, + key_on: !!key.key_on, mode: key.mode || null, + format: fmt(key), + } : { running: false, unreachable: true }, + encoder: encoder ? { + running: !!enc.running, error: enc.error || null, + viewers: enc.viewers != null ? enc.viewers : null, + bitrate_kbps: enc.encoder && enc.encoder.bitrate != null ? enc.encoder.bitrate : null, + tune: enc.encoder ? enc.encoder.tune : null, + } : { running: false, unreachable: true }, + }; +} + +// ── Sources panel: one row per wired selector input, joined with liveness. ──── +// Pure. Joins three things we ALREADY have — no new probes: +// • the selector's wired UUID list + active_input (which slot is PGM) +// • the thumbs per-flow {state} map (keyed by thumbs slot name, e.g. "guest1") +// • a caller-supplied roleMap: { "": {name, role}, ... } for friendly labels +// and a flowSlot map { "": "" } to look up liveness. +// roleMap/flowSlot are built from config on the server (box-specific), so this stays +// generic/committable. A uuid starting with the stable prefix (57ab) is flagged as a +// never-interrupt correspondent slot. Unknown uuids still render (short uuid + "input"). +function buildSources({ selector, flows = {}, roleMap = {}, flowSlot = {}, stablePrefix = '57ab' } = {}) { + const uuids = (selector && Array.isArray(selector.input_flow_uuids)) ? selector.input_flow_uuids : []; + const active = selector && selector.active_input != null ? selector.active_input : null; + return uuids.map((uuid, i) => { + const meta = roleMap[uuid] || {}; + const slotName = flowSlot[uuid] || meta.slot || null; + const liveness = slotName && flows[slotName] ? flows[slotName].state : null; + const stable = typeof uuid === 'string' && uuid.startsWith(stablePrefix); + // A stable slot with no live correspondent reads 'dead'/none from thumbs but is + // correctly "standing by", not broken — surface that distinctly. + let state = liveness || 'unknown'; + if (stable && (state === 'dead' || state === 'unknown')) state = 'standby'; + return { + slot: i, + uuid: typeof uuid === 'string' ? uuid.slice(0, 8) : null, + name: meta.name || (stable ? `Correspondent ${i}` : `Input ${i}`), + role: meta.role || (stable ? 'correspondent' : 'input'), + state, + is_pgm: active != null && i === active, + stable, + }; + }); +} + +// ── Compose the whole payload (pure; takes already-read inputs). ────────────── +function composeHealth({ system, grains, pipelines, sources, ts }) { + return { + ts, system: system || null, + grains: grains || { stale: true, flows: {} }, + pipelines: pipelines || {}, + sources: sources || [], + }; +} + +module.exports = { + readProcStat, cpuPercent, readMem, readLoad, cpuCount, + classifyGrains, classifyThumbsHealth, readGrainsSnapshot, shapePipelines, buildSources, composeHealth, +}; diff --git a/backend/ingest-info.js b/backend/ingest-info.js index 8f12b01..d113c98 100644 --- a/backend/ingest-info.js +++ b/backend/ingest-info.js @@ -10,40 +10,80 @@ // into a per-guest port (srtPort + index) with NO streamid. Mirrors // quickstart's MXL_GUEST_TRANSPORT=srt-listen (one listener owns one port). -const GUESTS = [ +const DEFAULT_GUESTS = [ { slot: 'guest1', label: 'Guest 1' }, { slot: 'guest2', label: 'Guest 2' }, ]; +// Parse "guest1,guest2,guest3" or "Guest 1:guest1,Phone:guest3" into a guest list. +// Falls back to DEFAULT_GUESTS. Lets a deployment advertise exactly the slots it runs +// (e.g. the box added a 3rd "phone" slot) without code changes. +function parseGuests(spec) { + if (!spec) return DEFAULT_GUESTS.slice(); + const out = []; + for (const part of String(spec).split(',').map((s) => s.trim()).filter(Boolean)) { + const [a, b] = part.split(':').map((s) => s.trim()); + const slot = (b || a); + const label = b ? a : a.replace(/^guest/i, 'Guest '); + if (/^guest\d+$/i.test(slot)) out.push({ slot: slot.toLowerCase(), label }); + } + return out.length ? out : DEFAULT_GUESTS.slice(); +} + +// Parse an explicit port list "8890,8891,8895" → [8890,8891,8895]. Empty → null (use +// the computed scheme). Non-contiguous real deployments (a bolt-on slot on a gap port) +// MUST set this so the advertised QR port matches the actual listener. (HW Oct 9: the +// box's guest3 listens on :8895, not the computed :8892 — a wrong QR = a dead demo.) +function parsePorts(spec) { + if (!spec) return null; + const ports = String(spec).split(',').map((s) => parseInt(s.trim(), 10)).filter((n) => Number.isInteger(n) && n > 0); + return ports.length ? ports : null; +} + // Resolve the config once from a plain object (e.g. process.env), so callers and // tests share identical logic. transport defaults to srt-listen (matches quickstart's // default; quickstart also passes MXL_GUEST_TRANSPORT through explicitly). -function ingestConfig({ publicIp = '', srtPort = 8890, transport = 'srt-listen' } = {}) { +// guests — explicit slot list (MXL_GUESTS), else guest1+guest2 +// guestPorts — explicit per-guest ports (MXL_GUEST_PORTS), else srtPort + index +// Parse "guest5=HOST,guest3=HOST2" → {guest5:HOST,...}. Lets a deployment +// advertise a RELAY host (e.g. an Azure SRT relay) for a specific guest slot so the +// contributor publishes there instead of the box's own IP — keeping the home IP private. +function parseHosts(spec){ if(!spec) return {}; const m={}; for(const p of String(spec).split(',').map(x=>x.trim()).filter(Boolean)){ const [k,v]=p.split('='); if(k&&v) m[k.trim()]=v.trim(); } return m; } +function ingestConfig({ publicIp = '', srtPort = 8890, transport = 'srt-listen', guests, guestPorts, guestHosts } = {}) { const listen = transport === 'srt-listen'; + const list = Array.isArray(guests) ? guests : parseGuests(guests); + const explicitPorts = Array.isArray(guestPorts) ? guestPorts : parsePorts(guestPorts); + const hostMap = (guestHosts && typeof guestHosts==='object' && !Array.isArray(guestHosts)) ? guestHosts : parseHosts(guestHosts); return { publicIp, srtPort, transport, listen, - // listen: each guest owns srtPort + index; direct: all share srtPort. - guestPort: (i) => (listen ? srtPort + i : srtPort), + guests: list, + // Explicit port map wins (handles non-contiguous real ports). Otherwise: + // listen → each guest owns srtPort + index; direct → all share srtPort. + hostFor: (slot) => (hostMap[slot] || publicIp), + guestPort: (i) => (explicitPorts && explicitPorts[i] != null) + ? explicitPorts[i] + : (listen ? srtPort + i : srtPort), }; } // The JSON body for GET /api/mxl/ingest. function ingestInfo(cfg) { + const guests = cfg.guests || DEFAULT_GUESTS; return { public_ip: cfg.publicIp || null, srt_port: cfg.srtPort, transport: cfg.transport, - guests: GUESTS.map((g, i) => ({ + guests: guests.map((g, i) => ({ ...g, port: cfg.guestPort(i), streamid: cfg.listen ? null : 'publish:' + g.slot, srt_url: !cfg.publicIp ? null : cfg.listen - ? `srt://${cfg.publicIp}:${cfg.guestPort(i)}?latency=200` + ? `srt://${cfg.hostFor ? cfg.hostFor(g.slot) : cfg.publicIp}:${cfg.guestPort(i)}?latency=200` : `srt://${cfg.publicIp}:${cfg.srtPort}?streamid=publish:${g.slot}&latency=200`, qr: cfg.publicIp ? `/api/mxl/ingest/qr/${g.slot}.png` : null, })), @@ -61,7 +101,7 @@ function larixUrl(cfg, slot) { const name = encodeURIComponent('MXL ' + slot.replace(/^guest/, 'Guest ')); const idx = Math.max(0, (parseInt(slot.replace(/^guest/, ''), 10) || 1) - 1); if (cfg.listen) { - const srt = encodeURIComponent(`srt://${cfg.publicIp}:${cfg.guestPort(idx)}`); + const srt = encodeURIComponent(`srt://${cfg.hostFor ? cfg.hostFor(slot) : cfg.publicIp}:${cfg.guestPort(idx)}`); return `larix://set/v1?conn[][name]=${name}&conn[][url]=${srt}` + `&conn[][mode]=av&conn[][srtlatency]=1000`; } @@ -71,4 +111,4 @@ function larixUrl(cfg, slot) { + `&conn[][mode]=av&conn[][srtstreamid]=${sid}&conn[][srtlatency]=1000`; } -module.exports = { GUESTS, ingestConfig, ingestInfo, larixUrl }; +module.exports = { GUESTS: DEFAULT_GUESTS, DEFAULT_GUESTS, parseGuests, parsePorts, ingestConfig, ingestInfo, larixUrl }; diff --git a/backend/local-server.js b/backend/local-server.js index 5dea2c4..29d5744 100644 --- a/backend/local-server.js +++ b/backend/local-server.js @@ -107,10 +107,90 @@ const INGEST_CFG = ingestConfig({ publicIp: PUBLIC_IP, srtPort: SRT_PORT, transport: process.env.MXL_GUEST_TRANSPORT || 'srt-listen', + // MXL_GUESTS lets a deployment advertise exactly its slots (e.g. "guest1,guest2,guest3" + // with a bolt-on phone slot). MXL_GUEST_PORTS pins the real listener ports when they + // aren't contiguous (e.g. "8890,8891,8895" — the box's guest3 is on :8895, not :8892). + guests: process.env.MXL_GUESTS || undefined, + guestPorts: process.env.MXL_GUEST_PORTS || undefined, + guestHosts: process.env.MXL_GUEST_HOSTS || undefined, }); +// The set of valid guest slots for the QR route — derived from the config, not hardcoded. +const INGEST_SLOTS = new Set((INGEST_CFG.guests || []).map((g) => g.slot)); -app.get('/api/mxl/ingest', (req, res) => { - res.json(ingestInfo(INGEST_CFG)); +// ── Health "Sources" panel role map ─────────────────────────────────────────── +// Friendly name + role for each wired flow, so the health page can label inputs +// ("Makito", "PTZ", "Correspondent 5") instead of raw UUIDs, and mark which is PGM. +// Keyed by UUID PREFIX (first 8 chars) because that's what survives runtime discovery +// for the deterministic flows (guest/stable = 9eNNN…/57abN…); pattern/playout get random +// suffixes so we match them by prefix too (d3e15194/9998da48 on this box). A deployment +// overrides via MXL_HEALTH_ROLES="=:,..." — all box-specific values +// live in env, so this stays generic. Defaults cover the common lab roles by prefix. +function parseHealthRoles(spec) { + const map = {}; + String(spec || '').split(',').map((s) => s.trim()).filter(Boolean).forEach((e) => { + const [pfx, rest] = e.split('='); + if (!pfx || !rest) return; + const [name, role] = rest.split(':'); + map[pfx.trim()] = { name: (name || '').trim(), role: (role || 'input').trim() }; + }); + return map; +} +// Generic defaults: studio roles keyed by the conventional flow-id prefixes. These are +// NOT secrets or hostnames (just the demo flow-id scheme), and are overridable. +const HEALTH_ROLES = Object.assign({ + 'd3e15194': { name: 'Pattern', role: 'generator' }, + '9998da48': { name: 'Playout', role: 'playout' }, + '9e111e00': { name: 'Makito', role: 'camera' }, + '9e222e00': { name: 'Cam 2', role: 'camera' }, + '9e333e00': { name: 'PTZ', role: 'camera' }, + '9e444e00': { name: 'Guest 4', role: 'guest' }, + '9e555e00': { name: 'Guest 5', role: 'guest' }, + '9e666e00': { name: 'Guest 6', role: 'guest' }, + '57ab5e00': { name: 'Correspondent 5', role: 'correspondent' }, + '57ab6e00': { name: 'Correspondent 6', role: 'correspondent' }, +}, parseHealthRoles(process.env.MXL_HEALTH_ROLES)); +// Build the per-UUID roleMap + flowSlot (uuid -> thumbs slot name) for a given wiring. +function sourceMaps(uuids) { + const roleMap = {}; const flowSlot = {}; + (uuids || []).forEach((u) => { + if (typeof u !== 'string') return; + const meta = HEALTH_ROLES[u.slice(0, 8)]; + if (meta) roleMap[u] = meta; + // Map each flow to the thumbs probe slot name that carries its liveness. The thumbs + // probe keys by guestN for contributions and pattern/playout for the always-on + // generators; a stable correspondent flow (57abN) reflects its volatile guestN source. + const byPrefix = { + 'd3e15': 'pattern', '9998d': 'playout', + '9e111': 'guest1', '9e222': 'guest2', '9e333': 'guest3', + '9e444': 'guest4', '9e555': 'guest5', '9e666': 'guest6', + '57ab5': 'guest5', '57ab6': 'guest6', + }; + const slot = byPrefix[u.slice(0, 5)]; + if (slot) flowSlot[u] = slot; + }); + return { roleMap, flowSlot }; +} + +app.get('/api/mxl/ingest', async (req, res) => { + const info = ingestInfo(INGEST_CFG); + // Annotate each guest slot as free/occupied so the /join page can auto-assign the + // first FREE slot (two visitors must not be sent to the same busy listener — SRT + // listen mode is one-publisher-per-port, so a taken slot rejects the next phone). + // We derive "occupied" from the probe snapshot (a flow advancing = someone's on it). + try { + const snap = await readSnapshot('health.json', THUMBS_HEALTH_PATH); + const flows = (snap && snap.slots) || {}; + info.guests = info.guests.map((g) => { + const f = flows[g.slot]; + // occupied = the slot's source is delivering fresh frames (age present + small) + const occupied = !!(f && f.age != null && f.age < 5); + return { ...g, occupied, free: !occupied }; + }); + const firstFree = info.guests.find((g) => g.free); + info.suggested_slot = firstFree ? firstFree.slot : null; + } catch { /* no snapshot — leave slots unannotated */ } + res.set('Access-Control-Allow-Origin', process.env.MXL_HEALTH_CORS || '*'); + res.json(info); }); // Server-rendered QR PNG for a guest slot. Uses python3 + segno (pure-python, @@ -119,7 +199,9 @@ app.get('/api/mxl/ingest', (req, res) => { const { spawn } = require('child_process'); app.get('/api/mxl/ingest/qr/:slot.png', (req, res) => { const slot = req.params.slot; - if (!/^guest[12]$/.test(slot)) return res.status(400).end(); + // validate against the configured guest slots (not a hardcoded guest1/2), and keep + // the strict shape so nothing odd reaches the shell below. + if (!/^guest\d+$/.test(slot) || !INGEST_SLOTS.has(slot)) return res.status(400).end(); if (!PUBLIC_IP) return res.status(503).end(); // no IP → nothing to encode const payload = larixUrl(INGEST_CFG, slot); // segno writes a PNG to stdout; -o - with --scale for a crisp phone-scannable size. @@ -137,6 +219,149 @@ app.get('/api/mxl/ingest/qr/:slot.png', (req, res) => { }); }); +// ── Health Skin (read-only) ────────────────────────────────────────────────── +// A SEPARATE page from the control Skin: System (CPU/mem/load), Grain flow (bps + +// unique-fps per flow, catching repeat-wedges), per-source fps and pipeline-writer +// state. Nothing here mutates the facility — the Core stays lean and on-air-focused. +// +// The grain measurement is already done by tools/grain_probe.py (writes +// /grains.json every 3s, with its own load guards). We only read that file +// + /proc + the three pipeline/status endpoints, and cache the aggregate for a +// couple seconds so N open Health tabs collapse to one scrape. +const health = require('./health-info'); +// Two possible probe snapshots, in preference order: +// health.json — written by tools/mxl_thumbs.py, which is ALREADY running beside the +// multiview thumbnails (no extra process, no extra MXL readers). Carries per-slot +// {age, frozen} (same frozen-reader detection) + system + viewers. +// grains.json — written by tools/grain_probe.py (bps + unique-fps). Supported for +// adopters who run that probe; we fall back to it if health.json isn't present. +const THUMBS_HEALTH_PATH = process.env.MXL_THUMBS_HEALTH_PATH || path.join(THUMBS_DIR, 'health.json'); +const GRAINS_PATH = process.env.MXL_GRAINS_PATH || path.join(THUMBS_DIR, 'grains.json'); +const HEALTH_RATE = parseInt(process.env.MXL_GRAIN_RATE || '30', 10); +const HEALTH_TTL_MS = parseInt(process.env.MXL_HEALTH_TTL_MS || '2000', 10); +// The probe snapshots live wherever the thumbnails do. In file mode we read +// /*.json; in proxy mode (MXL_THUMBS_ORIGIN set — e.g. the domain is only +// mounted in a container and a tiny http server exposes it) we fetch the same JSON over +// HTTP, exactly like the /api/mxl/thumbs route. This keeps the Core off the Docker +// socket and portable to either layout. +const THUMBS_SNAP_ORIGIN = THUMBS_ORIGIN; // e.g. http://127.0.0.1:8086/thumbs + +async function readSnapshot(fileName, filePath) { + if (THUMBS_SNAP_ORIGIN) { + try { + const base = THUMBS_SNAP_ORIGIN.replace(/\/$/, ''); + const r = await fetch(`${base}/${fileName}`, { signal: AbortSignal.timeout(3000) }); + return r.ok ? await r.json() : null; + } catch { return null; } + } + return health.readGrainsSnapshot(filePath); // file mode (sync read, cheap) +} + +// Resolve the MXL control-API base the same way mxl-routes does, so the pipeline +// status probes hit the same VM the cuts do. +const HEALTH_MXL_VM = process.env.MXL_VM_URL + || (process.env.MXL_VM_FROM_MANIFEST === '1' && facility && facility.network && `http://${facility.network.mxl_vm}`) + || 'http://127.0.0.1'; + +function fetchJson(port, apiPath, timeoutMs = 3000) { + return fetch(`${HEALTH_MXL_VM}:${port}${apiPath}`, { signal: AbortSignal.timeout(timeoutMs) }) + .then((r) => (r.ok ? r.json() : null)) + .catch(() => null); // unreachable port -> null; shapePipelines marks it so +} + +// The REAL viewer count: WebRTC clients connect to mediamtx, not to the WHIP encoder +// (which always self-reports 0). mediamtx's API (MTX_API=yes, :9997) lists live sessions. +// We count readers on the program path. Null if the API isn't enabled/reachable. +const MTX_API_URL = process.env.MXL_MEDIAMTX_API || 'http://127.0.0.1:9997'; +const PROGRAM_PATH_NAME = process.env.MXL_PROGRAM_PATH || 'mxl2webrtc'; +function fetchViewerCount(timeoutMs = 2500) { + return fetch(`${MTX_API_URL}/v3/webrtcsessions/list`, { signal: AbortSignal.timeout(timeoutMs) }) + .then((r) => (r.ok ? r.json() : null)) + .then((d) => { + if (!d || !Array.isArray(d.items)) return null; + // count only READ sessions on the program path (ignore the WHIP publisher itself) + const readers = d.items.filter((s) => + (s.state === 'read' || s.state === 'playing' || !s.state) && + (!s.path || s.path === PROGRAM_PATH_NAME)); + return readers.length; + }) + .catch(() => null); +} + +let _prevCpu = health.readProcStat(); +let _healthCache = { at: 0, payload: null }; + +async function buildHealth() { + const [selector, keyer, encoder, mtxViewers] = await Promise.all([ + fetchJson(9604, '/pipeline/status'), + fetchJson(9605, '/pipeline/status'), + fetchJson(9601, '/pipeline/status'), + fetchViewerCount(), + ]); + // Prefer the already-running thumbs health.json; fall back to grain_probe's grains.json. + const thumbs = await readSnapshot('health.json', THUMBS_HEALTH_PATH); + let grains, probeSystem = null; + if (thumbs && thumbs.slots) { + const t = health.classifyThumbsHealth(thumbs, {}); + grains = { stale: t.stale, age_s: t.age_s, flows: t.flows, source: 'thumbs' }; + probeSystem = t.system; // load/mem/viewers measured where the probe runs (the box) + } else { + const gr = await readSnapshot('grains.json', GRAINS_PATH); + grains = health.classifyGrains(gr, { expectedRate: HEALTH_RATE }); + grains.source = 'grains'; + } + // System: use the probe's host-true numbers when available (Core may run off-box); + // otherwise read local /proc. CPU% always comes from local /proc deltas. + const curCpu = health.readProcStat(); + const system = { + cpu_pct: health.cpuPercent(_prevCpu, curCpu), + cpus: (probeSystem && probeSystem.cpus) || health.cpuCount(), + mem: (probeSystem && probeSystem.mem) || health.readMem(), + load: (probeSystem && probeSystem.load) || health.readLoad(), + // Prefer mediamtx's live session count (the truth); fall back to the probe file. + viewers: mtxViewers != null ? mtxViewers : (probeSystem ? probeSystem.viewers : null), + }; + _prevCpu = curCpu; + const pipelines = health.shapePipelines({ selector, keyer, encoder }); + // Surface the real count on the encoder card too (it self-reports 0 — see above). + if (mtxViewers != null && pipelines.encoder) pipelines.encoder.viewers = mtxViewers; + // Sources panel: join the wired selector inputs with thumbs liveness + friendly roles. + const wired = (pipelines.selector && pipelines.selector.input_flow_uuids) || []; + const { roleMap, flowSlot } = sourceMaps(wired); + const sources = health.buildSources({ + selector: pipelines.selector, flows: grains.flows || {}, roleMap, flowSlot, + }); + return health.composeHealth({ ts: Date.now() / 1000, system, grains, pipelines, sources }); +} + +app.get('/api/mxl/health', async (req, res) => { + try { + const now = Date.now(); + if (!_healthCache.payload || now - _healthCache.at > HEALTH_TTL_MS) { + _healthCache = { at: now, payload: await buildHealth() }; + } + res.set('Cache-Control', 'no-store'); + // Read-only public telemetry — allow cross-origin reads so the public site + // (mxlswitcher.com) can embed the live numbers. GET-only, no secrets, no control. + res.set('Access-Control-Allow-Origin', process.env.MXL_HEALTH_CORS || '*'); + res.json(_healthCache.payload); + } catch (e) { + res.status(500).json({ error: String((e && e.message) || e) }); + } +}); + +// The Health Skin page (read-only). Separate URL from the control Skin. +app.get('/health', (req, res) => res.sendFile(path.join(WEB_DIR, 'health.html'))); + +// The public "join the show" page — QR + Larix instructions + a live program +// preview. No control, no token; a visitor publishes their phone as a guest source. +app.get('/join', (req, res) => res.sendFile(path.join(WEB_DIR, 'join.html'))); + +// Public welcome/landing — routes a visitor to Join / Watch / Monitor (no control). +// Give demo visitors this URL (or /join) instead of "/" so they never land on the +// token-gated control Skin. The TD still uses "/". +app.get('/welcome', (req, res) => res.sendFile(path.join(WEB_DIR, 'welcome.html'))); + // Serve the operator UI + its static assets. app.get('/', (req, res) => res.sendFile(path.join(WEB_DIR, 'local.html'))); app.use(express.static(WEB_DIR)); diff --git a/backend/mxl-routes.js b/backend/mxl-routes.js index 7e1af71..b6dae7e 100644 --- a/backend/mxl-routes.js +++ b/backend/mxl-routes.js @@ -156,9 +156,13 @@ module.exports = function registerMxlRoutes(app, opts = {}) { })(); async function mxlStatus() { + // Only the SELECTOR (9604) is essential for status. The keyer (9605) and + // test-generator (9600) are optional — when bypassed or down they must not take + // down the whole status poll the Skin runs every second. Default them to {}. const [keyer, sel, tg1] = await Promise.all([ - mxlApi(9605, '/pipeline/status'), mxlApi(9604, '/pipeline/status'), - mxlApi(9600, '/pipeline/status') + mxlApi(9605, '/pipeline/status').catch(() => ({})), + mxlApi(9604, '/pipeline/status'), + mxlApi(9600, '/pipeline/status').catch(() => ({})), ]); // Which flow UUID is actually on program right now? The selector reports its // active_input as an INDEX into its own live input_flow_uuids, so translate @@ -246,16 +250,31 @@ module.exports = function registerMxlRoutes(app, opts = {}) { } await mxlApi(9604, '/pipeline/active-input', { slot: selIndex }); keyframeNudge(); // optional IDR (off by default — see keyframeNudge) - // self-heal: if the keyer is wired cam-direct (pre-relay topology), move it - const keyer = await mxlApi(9605, '/pipeline/status'); - if (keyer.input_flow_uuid !== MXL_SEL_FLOW) { - const keyWas = !!keyer.key_on; - await mxlApi(9605, '/pipeline/stop', {}).catch(() => {}); - await mxlApi(9605, '/pipeline/start', mxlKeyerBody(MXL_SEL_FLOW)); - await mxlApi(9605, '/pipeline/key', { on: keyWas }); - await new Promise(r => setTimeout(r, 1500)); - await mxlApi(9601, '/pipeline/stop', {}).catch(() => {}); - await mxlApi(9601, '/pipeline/start', mxlEncoderBody); + // The CUT itself is done — the selector's active-input moved. Everything below is + // a best-effort keyer self-heal, and it must NEVER fail the cut. + // + // When the keyer is bypassed (MXL_KEYER_BYPASS=1 — the encoder reads the selector + // output directly, no graphics overlay) or simply down, there is no keyer to heal: + // poking :9605 here used to throw (mxlApi rejects on unreachable/non-OK) and sink + // an otherwise-successful cut with ":9605 -> 400". Skip it, and even when we do try, + // swallow any error. A down/bypassed keyer can't break switching. + if (env.MXL_KEYER_BYPASS !== '1') { + try { + // self-heal: if the keyer is wired cam-direct (pre-relay topology), move it + const keyer = await mxlApi(9605, '/pipeline/status'); + if (keyer && keyer.running && keyer.input_flow_uuid !== MXL_SEL_FLOW) { + const keyWas = !!keyer.key_on; + await mxlApi(9605, '/pipeline/stop', {}).catch(() => {}); + await mxlApi(9605, '/pipeline/start', mxlKeyerBody(MXL_SEL_FLOW)); + await mxlApi(9605, '/pipeline/key', { on: keyWas }); + await new Promise(r => setTimeout(r, 1500)); + await mxlApi(9601, '/pipeline/stop', {}).catch(() => {}); + await mxlApi(9601, '/pipeline/start', mxlEncoderBody); + } + } catch (e) { + // keyer down/unreachable — the cut already succeeded; log and move on + console.warn(`cut: keyer self-heal skipped (${e.message})`); + } } return { ok: true, input: slot }; } finally { mxlBusy = false; } @@ -378,6 +397,26 @@ module.exports = function registerMxlRoutes(app, opts = {}) { mxlBusy = true; try { const slot = [0, 1, 2, 3].includes(req.body.slot) ? req.body.slot : 0; + // NEVER-INTERRUPT (docs/NEVER-INTERRUPT-PGM.md): a contributor's ingest fires an + // AUTO announce ({auto:1}) on its first locked frame — including every reconnect. + // The full cascade below STOPS+restarts the selector from a FIXED input list, which + // (a) drops the pre-wired stable correspondent slots and (b) interrupts PGM. That is + // exactly what never-interrupt must avoid. So for an AUTO announce, first check + // whether the selector even needs rebuilding: if it's already running cleanly with + // inputs wired, the reconnecting source is served by its pre-wired slot (stable flow) + // or is cuttable as-is — SKIP the cascade entirely. A genuine selector-down state + // still falls through to the full repair. Manual repair (no `auto`) always rebuilds + // (an operator asked for it). Publish/grey-screen recovery is handled separately by + // mxl-selfheal's heal_publish, so skipping here never leaves PGM grey. + if (req.body.auto) { + const sel = await mxlApi(9604, '/pipeline/status').catch(() => null); + const healthy = sel && sel.running === true && !sel.error + && Array.isArray(sel.input_flow_uuids) && sel.input_flow_uuids.length > 0; + if (healthy) { + mxlBusy = false; + return res.json({ ok: true, skipped: 'selector healthy — auto announce needs no rebuild (never-interrupt)' }); + } + } await mxlApi(9604, '/pipeline/stop', {}).catch(() => {}); await mxlApi(9604, '/pipeline/start', mxlSelectorBody); await new Promise(r => setTimeout(r, 1000)); diff --git a/config/facility.json b/config/facility.json index 0e88f4e..5ae9db5 100644 --- a/config/facility.json +++ b/config/facility.json @@ -13,73 +13,252 @@ "for readability (prefix encodes the role: ca1=cam1, ca2=cam2, 9e1/9e2=", "guests, a0d=pgm-audio, a1/a2=guest-audio, 1a9=layout, 57a=stabilizer).", "The random-looking ones (2f34c189.., 5c73394e..) are MXL-assigned/legacy", - "and must match what the writers actually created — read them off the", + "and must match what the writers actually created \u2014 read them off the", "easy-mxl flows API once the writers are up if you change the topology." ], - "facility": { "name": "OHG MXL Lab", "domain_path": "/mxl-domain", "resolution": "1920x1080", "framerate": "30" }, - "network": { "mxl_vm": "", "mxl_vm_internal": "10.0.0.5", "docker_gateway": "172.17.0.1", "host_from_container": "host.docker.internal", - "_note": "mxl_vm is a PLACEHOLDER in the public repo: set MXL_VM_IP (or edit this file locally) to the real address. mxl_vm is the address the control-plane (backend) uses to reach the VM's easy-mxl APIs — the public/static IP. mxl_vm_internal is the VNet address used VM-to-VM (e.g. guest_audio RTSP). docker_gateway is how containers reach the host-side mediamtx RTSP." + "_note": "mxl_vm is a PLACEHOLDER in the public repo: set MXL_VM_IP (or edit this file locally) to the real address. mxl_vm is the address the control-plane (backend) uses to reach the VM's easy-mxl APIs \u2014 the public/static IP. mxl_vm_internal is the VNet address used VM-to-VM (e.g. guest_audio RTSP). docker_gateway is how containers reach the host-side mediamtx RTSP." }, - "control_api": { "_note": "Per-process easy-mxl control endpoints. Bound to 127.0.0.1 only (see quickstart.sh); reach them from the host, not the public internet.", "bind": "127.0.0.1", "ports": { - "test_generator": { "port": 9600, "process": "easy-mxl TG", "role": "playout / test pattern source", "parked": false }, - "encoder": { "port": 9601, "process": "easy-mxl encoder", "role": "reads keyer PGM video + PGM audio -> mediamtx egress" }, - "file_player": { "port": 9602, "process": "easy-mxl file-player", "role": "looping clip source (pattern slot)" }, - "cam_ingest": { "port": 9603, "process": "cam_ingest.py", "role": "camera 1 ingest control" }, - "selector": { "port": 9604, "process": "easy-mxl selector", "role": "program input-selector (the cut)" }, - "keyer": { "port": 9605, "process": "easy-mxl HTML5 keyer", "role": "cam + lower-third graphics -> keyer PGM" }, - "test_generator_2": { "port": 9606, "process": "easy-mxl TG2", "role": "second test generator", "parked": true }, - "mxl_info_gui": { "port": 9608, "process": "mxl-info-gui", "role": "hands-on inspector (optional)", "parked": true }, - "whip_commentary": { "port": 9609, "process": "webrtc2mxl", "role": "remote commentator mic via WHIP (voice)", "parked": true }, - "whip_commentary2": { "port": 9610, "process": "webrtc2mxl", "role": "remote commentator 2 mic via WHIP (voice2)", "parked": true } + "test_generator": { + "port": 9600, + "process": "easy-mxl TG", + "role": "playout / test pattern source", + "parked": false + }, + "encoder": { + "port": 9601, + "process": "easy-mxl encoder", + "role": "reads keyer PGM video + PGM audio -> mediamtx egress" + }, + "file_player": { + "port": 9602, + "process": "easy-mxl file-player", + "role": "looping clip source (pattern slot)" + }, + "cam_ingest": { + "port": 9603, + "process": "cam_ingest.py", + "role": "camera 1 ingest control" + }, + "selector": { + "port": 9604, + "process": "easy-mxl selector", + "role": "program input-selector (the cut)" + }, + "keyer": { + "port": 9605, + "process": "easy-mxl HTML5 keyer", + "role": "cam + lower-third graphics -> keyer PGM" + }, + "test_generator_2": { + "port": 9606, + "process": "easy-mxl TG2", + "role": "second test generator", + "parked": true + }, + "mxl_info_gui": { + "port": 9608, + "process": "mxl-info-gui", + "role": "hands-on inspector (optional)", + "parked": true + }, + "whip_commentary": { + "port": 9609, + "process": "webrtc2mxl", + "role": "remote commentator mic via WHIP (voice)", + "parked": true + }, + "whip_commentary2": { + "port": 9610, + "process": "webrtc2mxl", + "role": "remote commentator 2 mic via WHIP (voice2)", + "parked": true + } } }, - "video_flows": { "_note": "Authoritative label->UUID map for video flows (was duplicated in grain_probe.py FLOWS + backend + layout_pgm + bring-up).", - "cam": { "uuid": "ca111e00-aaaa-4bbb-8ccc-000000000001", "label": "CAM 1 Live", "grouphint": "Camera", "writer": "cam_ingest.py" }, - "cam2": { "uuid": "ca222e00-aaaa-4bbb-8ccc-000000000001", "label": "CAM 2 Live", "grouphint": "Camera", "writer": "cam2_ingest.py" }, - "playout": { "uuid": "2f34c189-64bf-5971-993a-332a28a7a6ee", "label": "TG Video", "grouphint": "Test-Generator", "writer": "easy-mxl TG (:9600)" }, - "pattern": { "uuid": "6b5d8d68-64ce-56f8-bea2-e79b6c282a86", "label": "Clip Video", "grouphint": "File-Player", "writer": "easy-mxl file-player (:9602)" }, - "guest1": { "uuid": "9e111e00-aaaa-4bbb-8ccc-000000000001", "label": "Guest 1 Video", "grouphint": "Guest", "writer": "guest_ingest.py" }, - "guest2": { "uuid": "9e222e00-aaaa-4bbb-8ccc-000000000001", "label": "Guest 2 Video", "grouphint": "Guest", "writer": "guest_ingest.py" }, - "layout": { "uuid": "1a900700-aaaa-4bbb-8ccc-000000000001", "label": "Layout PGM", "grouphint": "Layout", "writer": "layout_pgm.py" }, - "selector": { "uuid": "9437652d-20d9-565e-be6e-b98c36067930", "label": "Selector PGM", "grouphint": "Input-Selector", "writer": "easy-mxl selector (:9604)" }, - "keyer": { "uuid": "5c73394e-85df-50a3-8988-5edde5b5522a", "label": "Keyer PGM", "grouphint": "HTML5-Keyer", "writer": "easy-mxl keyer (:9605)" }, - "stabilized_guest1": { "uuid": "57ab1e00-aaaa-4bbb-8ccc-000000000001", "label": "Guest 1 Stable", "grouphint": "Stabilizer", "role": "freewheeling stable copy of a volatile guest flow — slots point here so a guest reconnect never wedges a reader", "writer": "flow_stabilizer.py (default DST)" }, - "stabilized_guest2": { "uuid": "57ab2e00-aaaa-4bbb-8ccc-000000000001", "label": "Guest 2 Stable", "grouphint": "Stabilizer", "role": "freewheeling stable copy of the volatile Guest 2 flow", "writer": "flow_stabilizer.py guest2 " } + "cam": { + "uuid": "ca111e00-aaaa-4bbb-8ccc-000000000001", + "label": "CAM 1 Live", + "grouphint": "Camera", + "writer": "cam_ingest.py" + }, + "cam2": { + "uuid": "ca222e00-aaaa-4bbb-8ccc-000000000001", + "label": "CAM 2 Live", + "grouphint": "Camera", + "writer": "cam2_ingest.py" + }, + "playout": { + "uuid": "2f34c189-64bf-5971-993a-332a28a7a6ee", + "label": "TG Video", + "grouphint": "Test-Generator", + "writer": "easy-mxl TG (:9600)" + }, + "pattern": { + "uuid": "6b5d8d68-64ce-56f8-bea2-e79b6c282a86", + "label": "Clip Video", + "grouphint": "File-Player", + "writer": "easy-mxl file-player (:9602)" + }, + "guest1": { + "uuid": "9e111e00-aaaa-4bbb-8ccc-000000000001", + "label": "Guest 1 Video", + "grouphint": "Guest", + "writer": "guest_ingest.py" + }, + "guest2": { + "uuid": "9e222e00-aaaa-4bbb-8ccc-000000000001", + "label": "Guest 2 Video", + "grouphint": "Guest", + "writer": "guest_ingest.py" + }, + "layout": { + "uuid": "1a900700-aaaa-4bbb-8ccc-000000000001", + "label": "Layout PGM", + "grouphint": "Layout", + "writer": "layout_pgm.py" + }, + "selector": { + "uuid": "9437652d-20d9-565e-be6e-b98c36067930", + "label": "Selector PGM", + "grouphint": "Input-Selector", + "writer": "easy-mxl selector (:9604)" + }, + "keyer": { + "uuid": "5c73394e-85df-50a3-8988-5edde5b5522a", + "label": "Keyer PGM", + "grouphint": "HTML5-Keyer", + "writer": "easy-mxl keyer (:9605)" + }, + "stabilized_guest1": { + "uuid": "57ab1e00-aaaa-4bbb-8ccc-000000000001", + "label": "Guest 1 Stable", + "grouphint": "Stabilizer", + "role": "freewheeling stable copy of a volatile guest flow \u2014 slots point here so a guest reconnect never wedges a reader", + "writer": "flow_stabilizer.py (default DST)" + }, + "stabilized_guest2": { + "uuid": "57ab2e00-aaaa-4bbb-8ccc-000000000001", + "label": "Guest 2 Stable", + "grouphint": "Stabilizer", + "role": "freewheeling stable copy of the volatile Guest 2 flow", + "writer": "flow_stabilizer.py guest2 " + }, + "stabilized_guest3": { + "uuid": "57ab3e00-aaaa-4bbb-8ccc-000000000001", + "label": "Guest 3 Stable", + "grouphint": "Stabilizer", + "role": "freewheeling stable copy of the volatile Guest 3 flow \u2014 the selector wires HERE so a guest join/drop never re-wires PGM", + "writer": "flow_stabilizer.py guest3 " + }, + "stabilized_guest4": { + "uuid": "57ab4e00-aaaa-4bbb-8ccc-000000000001", + "label": "Guest 4 Stable", + "grouphint": "Stabilizer", + "role": "freewheeling stable copy of the volatile Guest 4 flow \u2014 the selector wires HERE so a guest join/drop never re-wires PGM", + "writer": "flow_stabilizer.py guest4 " + }, + "stabilized_guest5": { + "uuid": "57ab5e00-aaaa-4bbb-8ccc-000000000001", + "label": "Guest 5 Stable", + "grouphint": "Stabilizer", + "role": "freewheeling stable copy of the volatile Guest 5 flow \u2014 the selector wires HERE so a guest join/drop never re-wires PGM", + "writer": "flow_stabilizer.py guest5 " + }, + "stabilized_guest6": { + "uuid": "57ab6e00-aaaa-4bbb-8ccc-000000000001", + "label": "Guest 6 Stable", + "grouphint": "Stabilizer", + "role": "freewheeling stable copy of the volatile Guest 6 flow \u2014 the selector wires HERE so a guest join/drop never re-wires PGM", + "writer": "flow_stabilizer.py guest6 " + } }, - "audio_flows": { "_note": "Authoritative label->UUID map for audio flows (was duplicated in audio_pgm.py SOURCES/DST + bring-up guest-audio loop).", - "pgm": { "uuid": "a0d10000-aaaa-4bbb-8ccc-000000000001", "label": "PGM Audio", "grouphint": "Audio-PGM", "role": "program audio mix (encoder reads this)", "writer": "audio_pgm.py" }, - "playout": { "uuid": "4a37a1ae-e0e1-59de-8354-c6884b25e551", "label": "TG Audio 1", "grouphint": "Test-Generator", "role": "test-generator tone", "writer": "easy-mxl TG (:9600)" }, - "guest1": { "uuid": "a1111e00-aaaa-4bbb-8ccc-000000000001", "label": "Guest 1 Audio", "grouphint": "Guest", "writer": "guest_audio.py" }, - "guest2": { "uuid": "a2222e00-aaaa-4bbb-8ccc-000000000001", "label": "Guest 2 Audio", "grouphint": "Guest", "writer": "guest_audio.py" }, - "voice": { "uuid": "101fcb5f-8b42-5e92-a2c3-67542abedf6c", "label": "Commentary Audio", "grouphint": "Commentary", "role": "remote commentator via WHIP", "writer": "webrtc2mxl (:9609)" }, - "voice2": { "uuid": "e8eb9046-c6fa-53ed-97e8-8799c68fa55b", "label": "Commentary 2 Audio", "grouphint": "Commentary2", "role": "remote commentator 2 via WHIP", "writer": "webrtc2mxl (:9610)" } + "pgm": { + "uuid": "a0d10000-aaaa-4bbb-8ccc-000000000001", + "label": "PGM Audio", + "grouphint": "Audio-PGM", + "role": "program audio mix (encoder reads this)", + "writer": "audio_pgm.py" + }, + "playout": { + "uuid": "4a37a1ae-e0e1-59de-8354-c6884b25e551", + "label": "TG Audio 1", + "grouphint": "Test-Generator", + "role": "test-generator tone", + "writer": "easy-mxl TG (:9600)" + }, + "guest1": { + "uuid": "a1111e00-aaaa-4bbb-8ccc-000000000001", + "label": "Guest 1 Audio", + "grouphint": "Guest", + "writer": "guest_audio.py" + }, + "guest2": { + "uuid": "a2222e00-aaaa-4bbb-8ccc-000000000001", + "label": "Guest 2 Audio", + "grouphint": "Guest", + "writer": "guest_audio.py" + }, + "voice": { + "uuid": "101fcb5f-8b42-5e92-a2c3-67542abedf6c", + "label": "Commentary Audio", + "grouphint": "Commentary", + "role": "remote commentator via WHIP", + "writer": "webrtc2mxl (:9609)" + }, + "voice2": { + "uuid": "e8eb9046-c6fa-53ed-97e8-8799c68fa55b", + "label": "Commentary 2 Audio", + "grouphint": "Commentary2", + "role": "remote commentator 2 via WHIP", + "writer": "webrtc2mxl (:9610)" + } }, - "program": { "_note": "The live program path: selector picks an input, keyer adds graphics, encoder egresses. These reference video_flows keys.", - "selector_inputs": ["cam", "playout", "pattern", "cam2"], + "selector_inputs": [ + "cam", + "playout", + "pattern", + "cam2" + ], "_selector_inputs_note": "The portable 4-input demo order (slots 0=cam,1=playout,2=pattern,3=cam2). Used by mxl-routes.js + bring-up. The live prodbots facility exposes the full 7 below.", - "layout_inputs": ["cam", "playout", "pattern", "cam2", "guest1", "guest2", "layout"], - "layout_input_labels": ["Studio Cam 1", "Playout", "Pattern", "Studio Cam 2", "Guest 1", "Guest 2", "Layout"], + "layout_inputs": [ + "cam", + "playout", + "pattern", + "cam2", + "guest1", + "guest2", + "layout" + ], + "layout_input_labels": [ + "Studio Cam 1", + "Playout", + "Pattern", + "Studio Cam 2", + "Guest 1", + "Guest 2", + "Layout" + ], "selector_out": "selector", "keyer_in": "selector", "keyer_out": "keyer", @@ -87,4 +266,4 @@ "encoder_audio_in": "pgm", "legacy_cam_flow": "991e65d8-4fc4-58de-b22a-2d02f5952252" } -} +} \ No newline at end of file diff --git a/docs/HEALTH-SKIN.md b/docs/HEALTH-SKIN.md new file mode 100644 index 0000000..bc8bfe0 --- /dev/null +++ b/docs/HEALTH-SKIN.md @@ -0,0 +1,71 @@ +# Health Skin — read-only facility monitor + +A second **Skin** over the same Core: a read-only page that shows what the switcher +is doing under the hood — grain flow, system load, and pipeline-writer state. It is +deliberately **separate from the control Skin** (`web/local.html`). A TD cutting a +show should see sources and a cut button, not grain internals; an engineer watching +health should not be able to touch the program bus. Core vs Skin, applied twice. + +## What it shows + +- **System** — CPU %, memory, 1/5/15-min load, CPU count, and the live WebRTC + viewer count (reported for free by the encoder). +- **Grain flow** — per flow: grains/s **and unique-frames/s**. The unique-fps column + is the one that matters: a flow can tick 30 grains/s while the *picture is frozen* + (a repeat-wedge — head advances, frame doesn't). When that happens unique-fps + collapses and the row goes amber (`frozen`). `dead` = probe read nothing; `slow` = + rate below half; `ok` = both near rate. +- **Pipeline writers** — selector / keyer / encoder: running state, errors, active + input, format, key-on, viewers, bitrate. + +## Where the data comes from (and why it's cheap) + +The grain/frozen measurement is **already being produced** on a running facility: +`tools/mxl_thumbs.py` (the multiview thumbnail generator) also writes `health.json` +beside the thumbnails, carrying per-slot `{age, frozen}` — `age` = seconds since the +last frame, `frozen` = seconds since the content last *changed* (its own repeat-wedge +detector). Plus system load/mem and viewers. Because the thumbnail probe is already +running, the Health Skin adds **no new process and no second set of MXL readers**. + +`GET /api/mxl/health`: + +1. reads `health.json` (already on disk; falls back to `grains.json` if you run + `tools/grain_probe.py` instead — that probe adds unique-fps and has hard load + guards: pauses above load 26, exits above 29), +2. uses the probe's host-true load/mem/viewers (so it's correct even when the Core + runs off-box) and local `/proc` for CPU %, +3. fetches the three `:/pipeline/status` endpoints (same VM the cuts use), + +then caches the aggregate for ~2 s (`MXL_HEALTH_TTL_MS`) so N open Health tabs +collapse to a single scrape. The Core never spawns a probe or `docker exec`s anything, +and the live PGM/encode path is never touched. + +> Note: `mxl_thumbs.py`'s `health.json` is rewritten on a ~15 s cadence (the per-slot +> age/frozen values inside it are sub-2 s fresh). For a tighter live refresh, run +> `tools/grain_probe.py` (3 s, adds unique-fps) — the endpoint picks up `grains.json` +> automatically. + +## Running it + +- Serve: the Core (`backend/local-server.js`) serves the page at **`/health`** and the + data at **`/api/mxl/health`** — no extra process. +- The Grain-flow section populates from whichever probe snapshot exists: + `health.json` (written by the already-running `tools/mxl_thumbs.py`) is used first; + `grains.json` (from `tools/grain_probe.py`) is the fallback. If neither is present the + page still shows System + Pipeline writers and marks grains stale. + +## Config + +| env | default | meaning | +|-----|---------|---------| +| `MXL_GRAINS_PATH` | `/grains.json` | where grain_probe writes its snapshot | +| `MXL_GRAIN_RATE` | `30` | expected grain rate (1080p30) — classifies ok/slow/frozen | +| `MXL_HEALTH_TTL_MS` | `2000` | server-side cache TTL so viewers share one scrape | + +## Files + +- `backend/health-info.js` — pure aggregation/classification (unit-tested, no network) +- `backend/local-server.js` — `GET /api/mxl/health` + `GET /health` (read-only) +- `web/health.html` — the page (air-gapped: self-hosted fonts, no external hosts) +- `tests/js/health-info.test.js` — 14 unit tests (frozen/dead/slow, system, pipelines) +- `tests/test_local_ui.py` — air-gap + read-only + separation-from-Core checks diff --git a/docs/NEVER-INTERRUPT-PGM.md b/docs/NEVER-INTERRUPT-PGM.md new file mode 100644 index 0000000..a65aeb7 --- /dev/null +++ b/docs/NEVER-INTERRUPT-PGM.md @@ -0,0 +1,191 @@ +# Never-interrupt-PGM: the live correspondent architecture + +**The requirement (Guy, Oct 9, trade-show use case):** correspondents roam a show floor, +each scans a QR → publishes from their phone → the director cuts them LIVE. **Cutting to a +correspondent — or a correspondent joining/dropping — must NEVER interrupt the program.** + +## Why it breaks today + +Adding a guest to the live switcher re-wires the input-selector (`POST /pipeline/start` +with a new input list). That restarts the selector's router thread; the encoder reading the +selector output sees a grain gap and stalls → **PGM freezes ~15s** until `mxl-pgm-heal` +republishes. Worse, the selector **drops** any slot whose flow isn't currently advancing, so +an idle (not-yet-connected) correspondent's slot can't just sit pre-wired — it falls out. + +Two root causes, each with an existing-but-unassembled fix in `tools/`: + +1. **Re-wire on join** → restart → PGM gap. +2. **Idle slots collapse** → can't pre-wire → forced to re-wire when a correspondent appears. + +## The architecture (assemble + extend what exists) + +### 1. A persistent STABLE flow per correspondent slot — `tools/flow_stabilizer.py` +The stabilizer reads a VOLATILE guest flow and writes a STABLE flow that is **created once at +boot and never recreated**. When the volatile side goes silent (no correspondent, or a +reconnect recreates the volatile flow), it swaps ONLY its reader in-place — the stable flow +and its grains keep flowing (freewheel). Downstream consumers (the selector slot) point at the +STABLE flow and therefore **never wedge and never disappear.** + +→ Run one stabilizer per correspondent slot: guest1..guestN → +`flow_stabilizer.py guestK "Guest K Stable"`. +Today only `stabilized_guest1/2` exist in `config/facility.json`; **add stabilized_guest3..6** +(uuids `57ab3e00`..`57ab6e00`), one per slot we want live-joinable. + +### 2. Pre-wire ALL slots to their STABLE flows at boot — fixed-length slot map +Because the stable flows always exist, the selector can be wired ONCE at startup with the full +fixed slot list (cam, playout, pattern, cam2, stabilized_guest1..6, layout) and **never +re-wired again.** `tools/guest_slot_watcher.py` already implements the fixed-length slot map +with a SAFE placeholder — but with stabilizers present, the placeholder is rarely needed +because the stable flow is the permanent wiring. A correspondent joining just makes their +stable flow carry real frames; the slot was already wired → **cutting to them is instant and +touches nothing upstream of their own slot.** + +### 3. Contribution path per correspondent — Azure SRT relay (IP-hidden) +Each correspondent's phone publishes to the Azure SRT relay (see +[[mxl-azure-srt-relay-2026-10-09]] / `MXL_GUEST_HOSTS`), the box pulls outbound → feeds that +guest's volatile ingest → stabilizer → pre-wired slot. Home IP never exposed. One relay +port-pair per simultaneous correspondent (guest5 = 8897/18897 today; add 8896/18896, +8898/18898, … for more). + +## Result +``` +Correspondent phone ─SRT→ Azure relay ─(box pulls)→ guestK volatile ─→ stabilizer ─→ stabilized_guestK + │ (pre-wired at boot) + input-selector (NEVER re-wired) ──→ encoder ──→ PGM +``` +- Correspondent joins/drops/reconnects → only their volatile flow changes; stabilizer absorbs + it; selector + encoder + PGM untouched. +- Director cuts to any slot instantly (`active-input` only — no `/pipeline/start`). +- **PGM is never interrupted.** + +## Build plan (do on a CLEAN facility bring-up, not patched onto today's hand-surgered box) +1. Add `stabilized_guest3..6` to `config/facility.json` (box copy; uuids 57ab3e00..57ab6e00). +2. `quickstart.sh`/`bring-up`: launch a `flow_stabilizer.py` per guest slot (systemd: + `mxl-stabilizer@guestK` template unit). +3. Wire the selector ONCE at boot to the fixed list of STABLE guest flows (+ cam/playout/ + pattern/layout). Remove the per-join `/pipeline/start` re-wire from the Core/pgm-heal cut + path — cuts become `active-input`-only. +4. Per correspondent: an Azure relay port-pair + a box puller (template: + `mxl-guestK-azure-pull`), advertised via `MXL_GUEST_HOSTS=guestK=`. +5. Verify: with PGM live on cam, connect/disconnect a phone on guest5 repeatedly → PGM bytes + never drop to 0; cutting guest5↔cam is instant. + +## Known-good invariants to preserve (learned the hard way Oct 9) +- Restarting guest CORES or re-wiring the selector ripples into PGM — the whole point of this + architecture is to make both unnecessary during a live show. +- The gst-keyer (lower-third) must read a STABLE program flow too, or it stalls on re-wire. +- `mxl-pgm-heal` stays as the BOOT bring-up + recovery tool, not a per-join mechanism. + +## ⚠️ HW REALITY (Oct 9, first build attempt) — stabilizer-per-slot is too heavy + +Validated the CORE idea on HW: a stabilizer creates its STABLE flow at boot (even with no +source — "boot frame pushed"), and the selector KEEPS a wired slot pointing at an existing- +but-stale stable flow (wire → 200, slot holds while idle). So **pre-wiring to stable flows +genuinely prevents re-wire on join.** ✅ The architecture is sound. + +BUT: **running a flow_stabilizer per slot ×6 overloaded the box to load 54 (on 24 cores).** +Each stabilizer is a full mxlsrc→appsink + appsrc→mxlsink pipeline (~1 core). Six of them, on +top of the switcher + encoder + thumbs, starved the box — SSH stalled, the selector wedged +(API 000), and under the starvation the stabilizers' restamp couldn't stay monotonic so the +stable flows' head FROZE (mxlsink stopped advancing) despite the push-loop reporting pushes. + +**Revised approach needed (pick one):** +- **(a) Stabilize ONLY the slots that need live-join** (the roving correspondents), not all 6. + e.g. 2–3 stabilizers, not 6. Cheapest; fits the box. +- **(b) A lighter stabilizer** — the current one fully decodes+re-encodes v210. A passthrough/ + copy stabilizer (no decode) would be far cheaper, but your notes say a plain passthrough + stable flow stalls+timeline-jumps on respawn (the 9/12 kill). Needs the freewheel conform + but cheaper (e.g. operate on compressed/copy, or lower the conform cost). +- **(c) GPU-assist the stabilizer conform** (the GTX 970 is now installed) — offload the + v210↔raw there. +- **(d) Fewer, bigger correspondent slots** + accept that cutting to a NEW correspondent + (first join) does one re-wire, but cutting BETWEEN already-joined ones is instant. + +DECISION PENDING (Guy). Do NOT start 6 stabilizers again — it takes the box down. + +## ✅ HW REALITY (Oct 10, option (a) build) — 2 stabilizers fit; three more findings + +Guy chose **option (a)**. Built + measured on HW: + +1. **Two stabilizers fit with huge headroom.** Started `mxl-stabilizer@guest5` + + `@guest6` only. Load went **11.8 → 12.5** (vs 54 with six). Both stable flows + (`57ab5e00`, `57ab6e00`) created cleanly, "boot frame pushed — stable flow exists + from t0", and sat in freewheel standby (`have_input=False pushed=0 clock=True`) — + exactly correct with no correspondent connected. PGM was undisturbed throughout + (1.56 MB/2s). **Option (a) is viable; 2–3 stabilizers is the right ceiling for this box.** + +2. **Pre-wiring the stable flows into the selector requires `/pipeline/start`, which + RESTARTS the selector output and DROPS PGM to 0 for the republish.** This is the + crucial nuance the architecture hinges on: the never-interrupt payoff (a correspondent + joins → their pre-wired stable slot just starts carrying frames → NO re-wire) only holds + *after* a one-time pre-wire done **at setup/boot**. You still pay ONE program blip to + establish the wiring — so **do the pre-wire during bring-up, never mid-show.** Once wired, + joins/drops are free. → The build is: teach `mxl-pgm-heal` (the boot/bring-up tool) to + wire the **stable** correspondent UUIDs (`57abNe00`) instead of the volatile ones + (`9eNNNe00`), so the one-time cost is folded into the boot it already does. + +3. **UUIDs MUST be discovered, not taken from config (reviewer issue #2, reproduced).** + The selector resolves flows by FULL uuid. `pattern`/`playout` have random suffixes + (`d3e15194-6d1f-5955-8d52-52e7076b7a99`), NOT the config placeholder + (`…-4c2a-9b3e-000000000001`). Sending the placeholder → `flow_def.json not found`. + The pre-wire body must be built from `/pipeline/status`'s live `input_flow_uuids` + + the domain dir listing, never from `config/facility.json` literals. + +4. **`mxl-pgm-heal` has a real bug: it cuts to the first slot whose flow FILE EXISTS, + not one that's ADVANCING.** After a re-wire it cut to slot 2 (Makito), whose flow + existed but was stale (delta=0) → selector output froze → PGM dark. Pattern(0)/ + playout(1)/PTZ(3) were all LIVE (delta≈36). Fix: `have()` is not enough — gate the + default cut on a 1-second head-advance check and fall back to pattern(0). (This bug + bit us independently of the stabilizers and is the actual cause of the Oct-10 PGM-dark.) + +**NEXT (option (a) build, concrete):** +- Patch `mxl-pgm-heal.sh`: (i) build inputs as pattern, playout, G1(Makito vol), G2(PTZ vol), + then **stabilized** 57ab5e00/57ab6e00 for the two correspondent slots; (ii) replace the + file-exists default-cut with an **advancing** check (1 s head delta > 0, else pattern). +- Make the Core cut path `active-input`-ONLY for already-wired slots (it mostly is; remove any + residual `/pipeline/start` on join). +- Boot order: facility → stabilizers (guest5/6) → pgm-heal (wires stable, cuts to a live slot, + republishes). Then a phone join on guest5 makes 57ab5e00 carry frames with ZERO re-wire. +- VERIFY: connect/disconnect a phone on guest5 repeatedly → PGM bytes never hit 0; cut + guest5↔PTZ instant. + +## ✅ BUILT + LIVE (Oct 10, option (a)) — never-interrupt wiring is now the boot default + +`mxl-pgm-heal` was patched (on the box; see "repo note" below) and is the live boot/recovery +path. Two changes: + +1. **Correspondent slots wired to STABLE flows.** Fixed sources (pattern, playout, Makito, + PTZ, guest4) stay on volatile flows; the roving-correspondent slots guest5/guest6 wire to + their STABLE flows `57ab5e00`/`57ab6e00` **when the stabilizer is up** (stable flow + present), else fall back to the volatile flow so the slot is still cuttable. A new + `add_slot()` tracks the uuid each wired slot carries in `SLOTS` so the cut logic tests the + right flow. +2. **Default cut gated on ADVANCING, not file-exists** (the Oct-10 bug fix). New `advancing()` + helper samples head over 1 s; the boot cut walks the slots and picks the first one actually + moving, falling back to pattern(0). A freewheeled-dark stable flow reads "not advancing", so + we never blank PGM by cutting to an idle correspondent slot. + +**Verified on HW (Oct 10):** +- After pgm-heal: selector = `[pattern, playout, PTZ, 57ab5e00(STABLE), 57ab6e00(STABLE)]`, + cut to playout (verified advancing), PGM live 1.5 MB/2s. ✅ +- **The never-interrupt property, demonstrated:** stopped the guest5 Azure pull, pushed/killed + test callers at the ingest, restarted the pull — through ALL that source churn the selector + wiring stayed byte-identical and PGM never dropped (would have re-wired + blipped under the + old design). ✅ +- **Boot-durable:** re-running pgm-heal cold re-establishes stable slots [3,4] + live PGM every + time — i.e. survives the reboot it's designed for. ✅ +- NOT yet closed: a *positive* end-to-end live-video test (test pattern all the way through to + the stable flow advancing) — blocked only by test-harness SRT encoder availability, not the + architecture. Real proof = a phone joining via the Azure relay (previously proven), which now + rides the stable-flow wiring. Still TODO: strip any residual `/pipeline/start` on join from + the Core cut path (cuts should be `active-input`-only; mostly already true). + +**Repo note:** the patched `mxl-pgm-heal.sh` lives on the box, NOT committed here, because it +hardcodes this box's DISCOVERED (random-suffix) pattern/playout UUIDs — box-specific config +that must not ship in the shared repo (reviewer R6e + issue #2). The proper repo version should +DISCOVER pattern/playout UUIDs at runtime (from `/pipeline/status` + the domain dir) rather than +hardcode them; that generalization is the follow-up that makes this committable. + +**Box state (Oct-10, build complete):** PGM LIVE (~1.5 MB/2s, cut to a verified-advancing +source). Stabilizers guest5/guest6 ACTIVE+enabled. Selector pre-wired to stable correspondent +slots. Load ~11. guest-leg-watchdog CPU-spin guard active. All reboot-durable. diff --git a/docs/REVIEW-RESPONSE.md b/docs/REVIEW-RESPONSE.md new file mode 100644 index 0000000..8ebc04d --- /dev/null +++ b/docs/REVIEW-RESPONSE.md @@ -0,0 +1,61 @@ +# Review response thread + +Shared async thread between the author (Claude, on Guy's behalf) and the independent +reviewer. Each item carries a status: `open` / `fixed` / `disputed` / `wontfix` / +`needs-decision` / `needs-hw-verify`. + +## Oct 9 — Guy's decisions + actions on the Oct 9 review + +Thanks for the thorough pass. Guy made the five calls; here they are, with what's done. + +### Decisions from Guy +1. **Merge order** — `fixed(plan)`: proceed in your recommended order **#38 → #40 → #37 → #39 → #41**. + #37 is held until its open notes below are addressed; the other four can go once conflicts + are resolved (you offered — yes please, go ahead on the two small ones). +2. **Deploying unmerged branches to the live box** — `accepted`: we'll keep doing it *only* to + close the gap faster, and the merge above is the plan to end the drift. The box-specific + bits (facility.json, UUIDs, hostnames) stay uncommitted/placeholdered, never on master. +3. **/join policy** — `decided`: **passphrase (merge #40)**. Public contributors must present the + SRT passphrase; the /join QR embeds it so legit scanners stay one-tap. Approval-first lobby + is a possible later addition, not now. +4. **Apache copyright holder (#39)** — `decided`: **Guy Cochran** (individual). +5. **Naming the hostname + Jonas** — `decided`: remove the live hostname from the repo; drop the + **employer tag** on the EFA changelog line; keep the open-source **mxl-fabrics-proxy** credit + to Jonas Ohland (crediting public OSS is fair). Public IBC-vendor mentions (Riedel/Matrox/etc. + in the awards recap, sourced to trade press) stay — they're public record, not private info. + +### Actions already taken (Oct 9) +- **Hostname leak (#41)** — `fixed`: `docs/index.html` and `docs/what-are-grains.html` no longer + hardcode the live server hostname (placeholder `YOUR-SWITCHER-HOST`; the health page honors a + `window.MXL_HEALTH_API` override). The live mxlswitcher.com `public/` copies keep the real URL + (box-specific, served from the deploy target, not committed to master). +- **Control API hardening** — `fixed`: mediamtx's API now binds `127.0.0.1:9997` + (`MTX_APIADDRESS`) instead of `*:9997`. With `--network host` the default exposed an + unauthenticated control API on the LAN; the Core reads it over localhost, so nothing breaks. + Applied to quickstart.sh **and** the live box. (Other control ports — Core :3100, test-gen, + encoder, selector, keyer — were already bound to 127.0.0.1.) +- **Employer tag (#37 changelog)** — `fixed`: reworded the EFA entry to state the technical cause + (Nitro v3 vs v4) without naming an individual/employer. OSS proxy credit retained. + +### Still open — your #37 notes (`needs-work`, holding the merge) +- NMOS routing still ignores the flow ID; domain ID isn't a proper ID; the ADR over-claims both + as done; the soak check reports a stale result as a failure; PR is ~1,600 lines. + → These are real. We'll address them on #37 before it merges (after #38/#40). Leaving #37 + `open` until then. + +### Open question back to you +- On **#40**, do you want the passphrase to also gate the **multiview visibility** (right now + the public multiview shows every guest tile), or is passphrase-to-publish enough and + everyone-can-see is fine for the demo? Not decided yet — flagging it per your note. + +## Oct 9 (later) — two more decisions from Guy + +- **Multiview visibility (#40 sub-question)** — `decided`: **passphrase-to-publish is enough; + everyone-can-see is fine.** The public multiview showing every guest tile is acceptable for + the demo. No visibility gating needed. +- **Core-vs-Skin leak sweep** — `fixed` (new PR #42): Guy's principle is that the Core is for + everyone and OUR deployment specifics belong in a Skin or stay private. The runnable Core + (backend/scripts/tools) was defaulting env vars to our hosts (prodbots.com, *.cochran.cloud). + PR #42 neutralizes those defaults (localhost / YOUR-*-HOST placeholders), still overridable + via MXL_* env. `docs/` (the mxlswitcher.com site) keeps its OHG bylines — that's our own + publication, correctly ours. Suggest merging #42 early (small, master-based, no conflicts). diff --git a/docs/index.html b/docs/index.html index 975a3ce..e3b79b3 100644 --- a/docs/index.html +++ b/docs/index.html @@ -87,7 +87,7 @@

What if the switcher
was shared memory?

A working exploration of a live production control room built around the Media eXchange Layer: real cameras, remote guests, playout, layouts, graphics, audio, multiview, recording and standard control — assembled as software on general-purpose cloud VMs.

- See it in action + ▶ Try it LIVE — put your phone on air Explore the source Run it in ~10 minutes
@@ -289,6 +289,7 @@

From fresh Ubuntu VM to a cuttable MXL production in about ten minutes.

The point of this lab isn't the demo — it's the knowledge. If you want to understand MXL and where production is heading, start with these. All measured, all in the open.

+ PLAIN ENGLISH + LIVE DATAWhat's a grain?The one idea the whole system is built on — with live numbers from the $400 box → EXPLAINERWhat is MXL?The shared-memory media plane, in plain English + a NDI/2110/SRT comparison → THE WHOLE STACKThe smallest complete DMFThe entire open-standard pipeline in one diagram → DEEP REFERENCEMXL architectureGrains, flows, domains, the ring buffer + TAI timing — sourced from the SDK docs → diff --git a/docs/what-are-grains.html b/docs/what-are-grains.html new file mode 100644 index 0000000..194cf2a --- /dev/null +++ b/docs/what-are-grains.html @@ -0,0 +1,125 @@ + + + + + + +What's a grain? MXL in plain English — with live data · Switcher Lab for MXL™ + + + + + + + + + + + + + + + +
+ Switcher Lab FOR MXL™ + +
+ +
+
+

MXL in plain English

+

What's a "grain"? And what is a $400 computer actually doing?

+

The whole system is built out of one tiny idea. Once you get it, everything else — the cut, the multiview, the health numbers below — follows. No broadcast background needed.

+

Office Hours GlobalLive data from a running facility

+ +

A grain is one frame

+

Think of an old film reel: a long strip of still pictures. Play them fast enough — 30 a second — and your eye sees smooth motion. A "grain" is one of those pictures. One frame of video. (Audio works the same way: a grain is a short block of sound.)

+

So a normal 1080p camera running at 30 frames a second is producing 30 video grains every second, forever, as long as it's on.

+ +

A flow is one source

+

String all the grains from one camera together in order and you have a flow — the continuous stream of frames coming from that camera. Camera 1 is a flow. The playback clip is a flow. The guest on their phone is a flow. Your switcher is really just a room full of flows.

+ +
+ The clever part (MXL's whole reason to exist): normally, to send one camera to five different places you'd make five copies of the video — expensive. MXL instead puts each flow in shared memory — one copy that everything on the computer can read at once, like a whiteboard everyone in the room can see. The fifth reader costs almost nothing. That's why a cheap computer can do so much: it isn't copying video over and over. +
+ +

Why we count grains — and unique grains

+

If you want to know a camera is healthy, you count its grains per second. Seeing a steady 30/s means frames are arriving on time. Simple.

+

But there's a sneakier failure. Sometimes a feed keeps delivering 30 grains a second while showing the same frozen picture — the counter ticks, but nothing moves. So we also measure how many of those frames are actually different. When "frames arriving" stays high but "frames actually changing" drops to near zero, we know the picture is frozen even though the plumbing looks fine. That single trick catches a whole class of live-TV disasters before anyone on air notices.

+ +

Live right now: one $400 computer, running a show

+

This isn't a diagram. Below is the actual health readout from the reference switcher — a second-hand dual-Xeon machine bought for about $400 — pulled live as you loaded this page. It's running several real sources at once and serving the program to browsers.

+ +
+ + + +
Source (flow)StatusFrames arriving
reading the live facility…
+
+

Reading system load from the box…

+

Source: the switcher's own /api/mxl/health endpoint, same data an engineer watching the rack sees. "Dead" rows are simply sources that aren't connected at the moment (e.g. no phone guest joined).

+ +

The point, for the skeptics

+

Broadcast infrastructure has always meant specialized, expensive hardware. The numbers above are the argument against that: an entire live production control room — ingest, switching, graphics, encoding and remote control — fits on a computer that costs less than a single pro SDI cable run, with CPU to spare. The grains flow through shared memory, nothing is needlessly copied, and commodity silicon keeps up.

+

That's the Dynamic Media Facility idea, made concrete. Read the field findings → · See the full glossary →

+ +
MXL is a trademark of LF Projects, LLC. This is an independent explainer. Live figures come from a real reference facility and vary moment to moment. Not affiliated with or endorsed by LF Projects, EBU, AMWA or any vendor.
+
+
+ + + + + + diff --git a/scripts/quickstart.sh b/scripts/quickstart.sh index 378536f..91bdbde 100755 --- a/scripts/quickstart.sh +++ b/scripts/quickstart.sh @@ -295,8 +295,15 @@ fi # mediamtx keeps serving WebRTC (8889/8189), which the program monitor still needs. MTX_SRT_ARGS=() [ "$GUEST_TRANSPORT" = srt-listen ] && MTX_SRT_ARGS=(-e MTX_SRT=no) +# Enable mediamtx's control API so the Core can read the REAL viewer count — the WHIP +# encoder only publishes, it never sees the WebRTC clients, which connect to mediamtx +# directly. GET /v3/webrtcsessions/list is the truth. BIND IT TO LOCALHOST +# (MTX_APIADDRESS=127.0.0.1:9997): with --network host the default *:9997 exposes an +# UNAUTHENTICATED control API on the LAN. The Core reads it via 127.0.0.1, so localhost +# is all it needs. (Security review Oct 9.) docker run -d --name mediamtx --network host --restart unless-stopped "${MTX_CONF_ARGS[@]}" \ - "${MTX_SRT_ARGS[@]}" -e MTX_WEBRTCADDITIONALHOSTS="$PUBLIC_IP" "$IMG_MEDIAMTX" >/dev/null + "${MTX_SRT_ARGS[@]}" -e MTX_API=yes -e MTX_APIADDRESS=127.0.0.1:9997 \ + -e MTX_WEBRTCADDITIONALHOSTS="$PUBLIC_IP" "$IMG_MEDIAMTX" >/dev/null run_mf(){ # name hostport image extra... local name=$1 port=$2 image=$3; shift 3 # Control APIs bind 127.0.0.1 ONLY (least privilege): these are unauthenticated diff --git a/tests/js/health-info.test.js b/tests/js/health-info.test.js new file mode 100644 index 0000000..58c671a --- /dev/null +++ b/tests/js/health-info.test.js @@ -0,0 +1,191 @@ +// Unit tests for backend/health-info.js — the pure (no-network, no-express) health +// aggregation used by the read-only Health Skin. Run with `node --test`. +// +// The marquee case: a flow can tick grains at full rate while the picture is +// FROZEN (a repeat-wedge). classifyGrains must flag that as 'frozen' from the +// unique-fps collapse — the exact failure we hit live with the keyer. +const test = require('node:test'); +const assert = require('node:assert'); +const path = require('node:path'); + +const h = require(path.join(__dirname, '..', '..', 'backend', 'health-info.js')); + +// ── classifyGrains ──────────────────────────────────────────────────────────── +test('grains: healthy flow -> ok', () => { + const now = 1000; + const snap = { ts: 999.5, flows: { cam: { bps: 30, ufps: 29.7 } } }; + const out = h.classifyGrains(snap, { expectedRate: 30, nowS: now }); + assert.equal(out.stale, false); + assert.equal(out.flows.cam.state, 'ok'); +}); + +test('grains: FROZEN flow (full rate, unique collapsed) -> frozen', () => { + // 30 grains/s but only ~3 unique frames/s = repeat-wedge. + const snap = { ts: 1000, flows: { guest1: { bps: 30, ufps: 3 } } }; + const out = h.classifyGrains(snap, { expectedRate: 30, nowS: 1000 }); + assert.equal(out.flows.guest1.state, 'frozen', + 'a flow ticking at rate with collapsed unique-fps must read as frozen'); +}); + +test('grains: dead flow (null) -> dead', () => { + const snap = { ts: 1000, flows: { playout: null } }; + const out = h.classifyGrains(snap, { expectedRate: 30, nowS: 1000 }); + assert.equal(out.flows.playout.state, 'dead'); + assert.equal(out.flows.playout.bps, 0); +}); + +test('grains: slow flow (bps below half rate) -> slow', () => { + const snap = { ts: 1000, flows: { cam2: { bps: 10, ufps: 10 } } }; + const out = h.classifyGrains(snap, { expectedRate: 30, nowS: 1000 }); + assert.equal(out.flows.cam2.state, 'slow'); +}); + +test('grains: stale snapshot flagged by age', () => { + const snap = { ts: 900, flows: { cam: { bps: 30, ufps: 30 } } }; + const out = h.classifyGrains(snap, { expectedRate: 30, nowS: 1000, staleAfterS: 12 }); + assert.equal(out.stale, true, 'a 100s-old snapshot is stale'); + assert.equal(out.age_s, 100); +}); + +test('grains: missing/garbage snapshot -> stale, empty', () => { + const out = h.classifyGrains(null, { nowS: 1000 }); + assert.equal(out.stale, true); + assert.deepEqual(out.flows, {}); +}); + +// ── classifyThumbsHealth (the probe already running on the box) ───────────────── +test('thumbs: live slot (fresh frames, content changing) -> ok', () => { + const h2 = { ts: 1000, slots: { pattern: { age: 1.2, frozen: 1.2 } }, load1: 2 }; + const out = h.classifyThumbsHealth(h2, { nowS: 1000 }); + assert.equal(out.flows.pattern.state, 'ok'); + assert.equal(out.stale, false); +}); + +test('thumbs: FROZEN slot (frames arrive but content stuck) -> frozen', () => { + // age low (frames keep arriving) but frozen high (picture hasn't changed). + const h2 = { ts: 1000, slots: { guest1: { age: 1.0, frozen: 11.0 } } }; + const out = h.classifyThumbsHealth(h2, { nowS: 1000, frozenAfterS: 5 }); + assert.equal(out.flows.guest1.state, 'frozen', + 'frames arriving but content unchanged for >5s must read as frozen'); +}); + +test('thumbs: no reader data (null) -> unknown; real stale age -> dead', () => { + const h2 = { ts: 1000, slots: { guest3: null, playout: { age: 9.0, frozen: 9.0 } } }; + const out = h.classifyThumbsHealth(h2, { nowS: 1000, deadAfterS: 5 }); + // null = probe has no reader (not confirmed dead) -> unknown, so a live-but-unprobed + // source isn't painted red on the health page. + assert.equal(out.flows.guest3.state, 'unknown'); + assert.equal(out.flows.playout.state, 'dead', 'real age data >5s -> dead'); +}); + +test('thumbs: pulls system (load/mem/cpus/viewers) from the same file', () => { + const h2 = { + ts: 1000, slots: {}, + load1: 12.5, load5: 12, load15: 11, + cores: 24, mem_total_mb: 16000, mem_avail_mb: 4000, viewers: 3, + }; + const out = h.classifyThumbsHealth(h2, { nowS: 1000 }); + assert.equal(out.system.load.load1, 12.5); + assert.equal(out.system.cpus, 24); + assert.equal(out.system.mem.used_pct, 75); + assert.equal(out.system.viewers, 3); +}); + +test('thumbs: missing/garbage -> stale, empty', () => { + const out = h.classifyThumbsHealth(null, { nowS: 1000 }); + assert.equal(out.stale, true); + assert.deepEqual(out.flows, {}); +}); + +// ── system parsing ──────────────────────────────────────────────────────────── +test('cpuPercent: computes busy fraction from two /proc/stat samples', () => { + const prev = { total: 1000, idle: 800 }; + const cur = { total: 1100, idle: 850 }; // +100 total, +50 idle -> 50% busy + assert.equal(h.cpuPercent(prev, cur), 50); +}); + +test('cpuPercent: null when no delta / missing sample', () => { + assert.equal(h.cpuPercent(null, { total: 1, idle: 0 }), null); + assert.equal(h.cpuPercent({ total: 10, idle: 5 }, { total: 10, idle: 5 }), null); +}); + +test('readMem: parses MemTotal/MemAvailable into used %', () => { + const fake = () => 'MemTotal: 8000000 kB\nMemAvailable: 2000000 kB\nMemFree: 1000000 kB\n'; + const m = h.readMem(fake); + assert.equal(m.total_mb, Math.round(8000000 / 1024)); + assert.equal(m.used_pct, 75); // (8000000-2000000)/8000000 +}); + +test('readProcStat: aggregate cpu line -> total/idle', () => { + const fake = () => 'cpu 100 0 100 700 100 0 0 0\ncpu0 1 2 3 4\n'; + const s = h.readProcStat(fake); + // fields: user100 nice0 system100 idle700 iowait100 irq0 softirq0 steal0 + // idle = idle(700)+iowait(100) = 800; total = sum of all = 1000 + assert.equal(s.idle, 800); + assert.equal(s.total, 1000); +}); + +test('cpuCount: counts per-core lines', () => { + const fake = () => 'cpu 1 2\ncpu0 1\ncpu1 1\ncpu2 1\n'; + assert.equal(h.cpuCount(fake), 3); +}); + +// ── pipeline shaping ────────────────────────────────────────────────────────── +test('shapePipelines: normalizes selector/keyer/encoder; marks unreachable', () => { + const out = h.shapePipelines({ + selector: { running: true, active_input: 2, input_flow_uuids: ['a', 'b', 'c'], + format: { frame_width: 1920, frame_height: 1080, grain_rate: { numerator: 30, denominator: 1 } } }, + keyer: { running: true, key_on: false, mode: 'key' }, + encoder: null, // unreachable + }); + assert.equal(out.selector.active_input, 2); + assert.equal(out.selector.inputs, 3); + assert.deepEqual(out.selector.format, { w: 1920, h: 1080, rate: 30 }); + assert.equal(out.keyer.key_on, false); + assert.equal(out.encoder.unreachable, true); + assert.equal(out.encoder.running, false); +}); + +test('shapePipelines: encoder viewers + bitrate surfaced', () => { + const out = h.shapePipelines({ + encoder: { running: true, viewers: 4, encoder: { bitrate: 6000, tune: 4 } }, + }); + assert.equal(out.encoder.viewers, 4); + assert.equal(out.encoder.bitrate_kbps, 6000); +}); + +// ── compose ─────────────────────────────────────────────────────────────────── +test('composeHealth: assembles a stable top-level shape', () => { + const out = h.composeHealth({ ts: 123, system: { cpu_pct: 10 }, grains: { stale: false, flows: {} }, pipelines: {} }); + assert.equal(out.ts, 123); + assert.equal(out.system.cpu_pct, 10); + assert.ok('grains' in out && 'pipelines' in out); +}); + +// ── buildSources (the Sources panel join) ─────────────────────────────────────── +test('buildSources: joins wiring + liveness + roles, flags PGM', () => { + const out = h.buildSources({ + selector: { input_flow_uuids: ['d3e15194-x', '9e333e00-x', '57ab5e00-x'], active_input: 1 }, + flows: { guest3: { state: 'ok' }, guest5: { state: 'dead' } }, + roleMap: { 'd3e15194-x': { name: 'Pattern', role: 'generator' }, + '9e333e00-x': { name: 'PTZ', role: 'camera' } }, + flowSlot: { '9e333e00-x': 'guest3', '57ab5e00-x': 'guest5' }, + }); + assert.equal(out.length, 3); + assert.equal(out[1].name, 'PTZ'); + assert.equal(out[1].state, 'ok'); + assert.equal(out[1].is_pgm, true); // active_input=1 + assert.equal(out[0].is_pgm, false); + // stable slot with no live correspondent -> 'standby', flagged stable + assert.equal(out[2].stable, true); + assert.equal(out[2].state, 'standby'); + assert.equal(out[2].name, 'Correspondent 2'); // default name for unmapped stable +}); + +test('buildSources: unknown uuid still renders; empty wiring -> []', () => { + assert.deepEqual(h.buildSources({ selector: { input_flow_uuids: [], active_input: null } }), []); + const out = h.buildSources({ selector: { input_flow_uuids: ['abcd1234-x'], active_input: 0 } }); + assert.equal(out[0].name, 'Input 0'); + assert.equal(out[0].state, 'unknown'); + assert.equal(out[0].is_pgm, true); +}); diff --git a/tests/js/mxl-ingest-transport.test.js b/tests/js/mxl-ingest-transport.test.js index 2078f88..0d86a0e 100644 --- a/tests/js/mxl-ingest-transport.test.js +++ b/tests/js/mxl-ingest-transport.test.js @@ -62,3 +62,63 @@ test('no public IP: srt_url + qr are null, no crash', () => { assert.equal(j.guests[0].srt_url, null); assert.equal(j.guests[0].qr, null); }); + +// ── Configurable guest list + explicit ports (public phone-inject flow) ───────── +// A real deployment may run a 3rd "phone" slot on a NON-contiguous port (the box's +// guest3 listens on :8895, not the computed :8892). The advertised QR/SRT URL MUST +// match the real listener or the scan sends the phone to a dead port. +const { parseGuests, parsePorts } = require(path.join(__dirname, '..', '..', 'backend', 'ingest-info.js')); + +test('MXL_GUESTS adds a third slot (guest3) to the advertised list', () => { + const cfg = ingestConfig({ publicIp: PUBLIC_IP, srtPort: BASE_PORT, guests: 'guest1,guest2,guest3' }); + const j = ingestInfo(cfg); + assert.equal(j.guests.length, 3); + assert.equal(j.guests[2].slot, 'guest3'); + assert.equal(j.guests[2].label, 'Guest 3'); +}); + +test('MXL_GUEST_PORTS pins non-contiguous real ports (guest3 on :8895, not :8892)', () => { + const cfg = ingestConfig({ + publicIp: PUBLIC_IP, srtPort: BASE_PORT, transport: 'srt-listen', + guests: 'guest1,guest2,guest3', guestPorts: '8890,8891,8895', + }); + const j = ingestInfo(cfg); + assert.equal(j.guests[0].port, 8890); + assert.equal(j.guests[1].port, 8891); + assert.equal(j.guests[2].port, 8895, 'guest3 must advertise its REAL port 8895'); + assert.match(j.guests[2].srt_url, /:8895\?/); + // the Larix deep-link for guest3 must carry :8895 too + const lx = larixUrl(cfg, 'guest3'); + assert.match(lx, /%3A8895/, 'guest3 deep-link must point at the real :8895'); +}); + +test('parseGuests: accepts "Label:slot" and bare slots; bad entries dropped', () => { + assert.deepEqual(parseGuests('guest1,guest2'), [{ slot: 'guest1', label: 'Guest 1' }, { slot: 'guest2', label: 'Guest 2' }]); + assert.deepEqual(parseGuests('Phone:guest3'), [{ slot: 'guest3', label: 'Phone' }]); + assert.deepEqual(parseGuests('nonsense,guest4'), [{ slot: 'guest4', label: 'Guest 4' }]); + assert.equal(parseGuests('').length, 2, 'empty falls back to the 2 defaults'); +}); + +test('parsePorts: parses a CSV port list, null on empty/garbage', () => { + assert.deepEqual(parsePorts('8890,8891,8895'), [8890, 8891, 8895]); + assert.equal(parsePorts(''), null); + assert.equal(parsePorts('abc'), null); +}); + +// ── Per-guest relay host (IP-hiding: a guest publishes to an SRT relay, not the box) ── +test('MXL_GUEST_HOSTS routes a guest slot to a relay host (home IP stays private)', () => { + const cfg = ingestConfig({ + publicIp: '192.168.1.254', srtPort: 8890, transport: 'srt-listen', + guests: 'guest3,guest5', guestPorts: '8895,8897', + guestHosts: 'guest5=20.230.141.204', // guest5 -> Azure SRT relay + }); + const j = ingestInfo(cfg); + const g3 = j.guests.find((g) => g.slot === 'guest3'); + const g5 = j.guests.find((g) => g.slot === 'guest5'); + // guest3 stays on the LAN/public IP; guest5 advertises the relay host + assert.match(g3.srt_url, /192\.168\.1\.254:8895/); + assert.match(g5.srt_url, /20\.230\.141\.204:8897/, 'guest5 must advertise the relay host, not the box IP'); + // the Larix deep-link must ALSO carry the relay host (the QR a phone scans) + assert.match(larixUrl(cfg, 'guest5'), /20\.230\.141\.204/, 'guest5 QR must point at the relay, not the box'); + assert.ok(!larixUrl(cfg, 'guest5').includes('192.168.1.254'), 'guest5 QR must NOT leak the box IP'); +}); diff --git a/tests/js/mxl-routes.test.js b/tests/js/mxl-routes.test.js index 7fd22ce..0205f27 100644 --- a/tests/js/mxl-routes.test.js +++ b/tests/js/mxl-routes.test.js @@ -77,6 +77,60 @@ test('startup warning is logged when no token is set', () => { assert.ok(warns.some((m) => /MXL_CONTROL_TOKEN is not set/.test(m))); }); +// ── Never-interrupt: auto announce must NOT re-wire a healthy selector ─────────── +test('/repair {auto:1}: healthy selector -> SKIP cascade (no selector rebuild, PGM untouched)', async () => { + const app = mkApp(); registerMxlRoutes(app, { env: {}, log: silent }); + const realST = global.setTimeout; global.setTimeout = (f) => realST(f, 0); + // selector reports running with inputs wired; everything else default-ok + const healthySel = JSON.stringify({ running: true, error: null, input_flow_uuids: ['a', 'b', 'c', 'd'], active_input: 1 }); + global.fetch = async (url) => ({ ok: true, text: async () => (String(url).includes(':9604/pipeline/status') ? healthySel : '{}') }); + const calls = []; + const orig = global.fetch; + global.fetch = async (url, opts) => { calls.push(String(url)); return orig(url, opts); }; + try { + const res = await call(app, 'POST /api/mxl/repair', { headers: {}, body: { auto: 1 }, ip: '3.3.3.3' }); + assert.strictEqual(res.code, 200); + assert.ok(res.body.skipped, 'should report skipped'); + // the whole point: NO selector stop/start, NO active-input, NO keyer/encoder rebuild + assert.ok(!calls.some((u) => /\/pipeline\/(stop|start|active-input)/.test(u)), + 'auto announce on a healthy selector must not stop/start/re-cut anything'); + } finally { global.setTimeout = realST; } +}); + +test('/repair {auto:1}: selector DOWN -> full cascade still runs (recovery intact)', async () => { + const app = mkApp(); registerMxlRoutes(app, { env: {}, log: silent }); + const realST = global.setTimeout; global.setTimeout = (f) => realST(f, 0); + const downSel = JSON.stringify({ running: false, error: 'stopped', input_flow_uuids: [] }); + const calls = []; + global.fetch = async (url) => { + calls.push(String(url)); + return { ok: true, text: async () => (String(url).includes(':9604/pipeline/status') ? downSel : '{}') }; + }; + try { + const res = await call(app, 'POST /api/mxl/repair', { headers: {}, body: { auto: 1 }, ip: '4.4.4.4' }); + assert.strictEqual(res.code, 200); + assert.ok(!res.body.skipped, 'a down selector must NOT be skipped'); + assert.ok(calls.some((u) => /:9604\/pipeline\/start/.test(u)), 'full cascade restarts the selector'); + } finally { global.setTimeout = realST; } +}); + +test('/repair manual (no auto): always rebuilds even if selector is healthy', async () => { + const app = mkApp(); registerMxlRoutes(app, { env: {}, log: silent }); + const realST = global.setTimeout; global.setTimeout = (f) => realST(f, 0); + const healthySel = JSON.stringify({ running: true, error: null, input_flow_uuids: ['a', 'b'] }); + const calls = []; + global.fetch = async (url) => { + calls.push(String(url)); + return { ok: true, text: async () => (String(url).includes(':9604/pipeline/status') ? healthySel : '{}') }; + }; + try { + const res = await call(app, 'POST /api/mxl/repair', { headers: {}, body: { slot: 0 }, ip: '5.5.5.5' }); + assert.strictEqual(res.code, 200); + assert.ok(!res.body.skipped, 'manual repair is never skipped'); + assert.ok(calls.some((u) => /:9604\/pipeline\/start/.test(u)), 'manual repair runs the cascade'); + } finally { global.setTimeout = realST; } +}); + // ── Review-fix regression tests (R3 pattern slot, R4 input validation, R5 timeout) ── // a fetch mock that records {url, body} and can return per-URL responses @@ -204,3 +258,47 @@ test('Take flip-flops PVW↔PGM: the old program drops back to preview', async ( assert.strictEqual(took.body.input, 1, 'program is now the armed source (slot 1)'); assert.strictEqual(took.body.pvw, 0, 'preview is now the OLD program (slot 0) — the flip-flop'); }); + +// ── Keyer-tolerant cut (public-demo robustness) ──────────────────────────────── +// The CUT is the selector's active-input move. A down/bypassed keyer must never +// fail it. Before this, the cut path did `await mxlApi(9605,...)` unconditionally, +// so a stopped keyer (:9605 unreachable or 400) sank an otherwise-good cut. + +test('cut succeeds even when the keyer (:9605) is DOWN', async () => { + const CAM = 'ca111e00-aaaa-4bbb-8ccc-000000000001'; + const app = mkApp(); registerMxlRoutes(app, { env: { MXL_PREWARM: '0' }, log: silent }); + const calls = mkFetch((url) => { + if (url.includes(':9605/')) throw new Error('connect ECONNREFUSED'); // keyer down + if (url.includes(':9604/pipeline/status')) return { ok: true, text: async () => JSON.stringify({ active_input: 1, input_flow_uuids: [CAM] }) }; + return { ok: true, text: async () => '{}' }; + }); + const res = await call(app, 'POST /api/mxl/input', { headers: {}, body: { input: 'cam' }, ip: '1.1.1.1' }); + assert.strictEqual(res.code, 200, 'cut must succeed despite the keyer being down'); + // the real cut (active-input) still went out + assert.ok(calls.some((c) => c.url.includes(':9604/pipeline/active-input')), 'the selector cut was issued'); +}); + +test('MXL_KEYER_BYPASS=1: a cut never pokes the keyer (:9605)', async () => { + const CAM = 'ca111e00-aaaa-4bbb-8ccc-000000000001'; + const app = mkApp(); registerMxlRoutes(app, { env: { MXL_PREWARM: '0', MXL_KEYER_BYPASS: '1' }, log: silent }); + const calls = mkFetch((url) => { + if (url.includes(':9604/pipeline/status')) return { ok: true, text: async () => JSON.stringify({ active_input: 1, input_flow_uuids: [CAM] }) }; + return { ok: true, text: async () => '{}' }; + }); + const res = await call(app, 'POST /api/mxl/input', { headers: {}, body: { input: 'cam' }, ip: '1.1.1.1' }); + assert.strictEqual(res.code, 200, 'cut succeeds in bypass mode'); + assert.ok(!calls.some((c) => c.url.includes(':9605/')), 'bypass mode must not touch the keyer at all'); +}); + +test('status survives a down keyer (:9605) — the Skin keeps polling', async () => { + const CAM = 'ca111e00-aaaa-4bbb-8ccc-000000000001'; + const app = mkApp(); registerMxlRoutes(app, { env: {}, log: silent }); + mkFetch((url) => { + if (url.includes(':9605/')) throw new Error('ECONNREFUSED'); // keyer down + if (url.includes(':9600/')) throw new Error('ECONNREFUSED'); // test-gen down too + if (url.includes(':9604/pipeline/status')) return { ok: true, text: async () => JSON.stringify({ active_input: 0, input_flow_uuids: [CAM] }) }; + return { ok: true, text: async () => '{}' }; + }); + const res = await call(app, 'GET /api/mxl/status', { headers: {}, ip: '1.1.1.1' }); + assert.strictEqual(res.code, 200, 'status must still return 200 with keyer+testgen down'); +}); diff --git a/tests/test_guest_leg_watchdog.py b/tests/test_guest_leg_watchdog.py new file mode 100644 index 0000000..4ef2970 --- /dev/null +++ b/tests/test_guest_leg_watchdog.py @@ -0,0 +1,71 @@ +"""guest-leg-watchdog.sh invariants (text-scanned — it's bash, no deps). + +HARD LESSON baked into these tests (HW Oct 9): restarting a guest chain recreates its MXL +flows → the input-selector re-wires → if that happens repeatedly the selector churns and +the whole facility destabilizes (PGM sticks, cuts fail, thumbnails flap). The first +watchdog restarted whole chains on any leg blip and broke the facility. So the watchdog is +now deliberately MINIMAL: it only watches the PUBLIC listener, never touches a chain with +live video, restarts ONLY the fanout (not the cores), and is OFF by default. +""" +import re +from pathlib import Path + +WD = Path(__file__).resolve().parent.parent / "tools" / "guest-leg-watchdog.sh" + + +def _src(): + assert WD.is_file(), "tools/guest-leg-watchdog.sh missing" + return WD.read_text() + + +def test_exists_and_bash_shebang(): + s = _src() + assert s.startswith("#!/usr/bin/env bash") or s.startswith("#!/bin/bash") + + +def test_off_by_default(): + """It's a sharp tool — must do nothing unless deliberately enabled (ENABLE=1).""" + s = _src() + assert re.search(r'ENABLE=\$\{ENABLE:-0\}', s), "watchdog must default ENABLE=0 (observe-only)" + assert 'ENABLE" = "1"' in s or "ENABLE\" == \"1\"" in s, "heal must be gated on ENABLE=1" + + +def test_never_touches_a_chain_with_live_video(): + """The bug that broke the facility: restarting a chain whose video is live churns the + selector. The watchdog MUST skip a chain whose video flow is advancing.""" + s = _src() + assert "video_advancing" in s, "no live-video guard" + # the guard must cause a skip/continue, not a heal + assert re.search(r'video_advancing "\$vuuid".*;\s*then\s*strike\[\$name\]=0;\s*continue', s, re.S), \ + "a chain with advancing video must be skipped, never healed" + + +def test_restarts_only_the_fanout_not_the_cores(): + """Restarting the cores recreates the MXL flows → selector churn. Heal must restart ONLY + the public-listener fanout, leaving the cores (and their flows) untouched.""" + s = _src() + assert re.search(r'docker restart "\$\{name\}-fanout"', s), "heal must restart the fanout" + assert not re.search(r'docker restart "\$\{name\}-audio"\s+"\$\{name\}"', s), \ + "heal must NOT restart the cores (that recreates flows and churns the selector)" + + +def test_only_acts_on_a_dead_public_listener(): + """The only thing that silently blocks a join is the PUBLIC listener being down on an + idle slot. Internal leg flaps are explicitly NOT the watchdog's concern anymore.""" + s = _src() + assert re.search(r'if bound "\$pub"; then', s), "must key off the public listener being up" + + +def test_has_strikes_cooldown_and_giveup(): + s = _src() + assert "STRIKES" in s and "COOLDOWN_SECONDS" in s and "GIVE_UP" in s, \ + "must have strikes + cooldown + give-up guards against thrash" + + +def test_watches_all_six_guest_chains_with_real_ports(): + """Default map covers guest1-6 on their REAL (non-contiguous) public ports + video UUIDs.""" + s = _src() + for spec in ("guest1:8890", "guest2:8891", "guest3:8895", + "guest4:8896", "guest5:8897", "guest6:8898"): + assert spec in s, f"default GUEST_PORTS missing {spec}" + assert "9e333e00" in s, "guest3 video UUID must be in the map (for the live-video check)" diff --git a/tests/test_local_ui.py b/tests/test_local_ui.py index c674164..c17b7b9 100644 --- a/tests/test_local_ui.py +++ b/tests/test_local_ui.py @@ -8,8 +8,12 @@ REPO = Path(__file__).resolve().parent.parent HTML = REPO / "web" / "local.html" +HEALTH_HTML = REPO / "web" / "health.html" +JOIN_HTML = REPO / "web" / "join.html" +WELCOME_HTML = REPO / "web" / "welcome.html" SERVER = REPO / "backend" / "local-server.js" ROUTES = REPO / "backend" / "mxl-routes.js" +HEALTH_INFO = REPO / "backend" / "health-info.js" PKG = REPO / "backend" / "package.json" FONTS = REPO / "web" / "fonts" @@ -110,3 +114,94 @@ def test_package_pins_express(): import json pkg = json.loads(PKG.read_text()) assert "express" in pkg.get("dependencies", {}), "express not pinned in package.json" + + +# ── Health Skin (read-only monitor, separate page from the control Skin) ────── + +def test_health_skin_exists_and_is_separate(): + """The grain/system monitor is its own page + its own read-only endpoint — it + must NOT leak control into the Core Skin (local.html drives only control + routes). Keeping health separate is the whole point of 'Core vs Skin'.""" + assert HEALTH_HTML.is_file(), "web/health.html missing" + assert HEALTH_INFO.is_file(), "backend/health-info.js missing" + # the control Skin must not start calling /api/mxl/health — health is its own page + assert "/api/mxl/health" not in HTML.read_text(), \ + "the control Skin (local.html) should not poll health — keep it separate" + + +def test_health_skin_is_self_contained(): + """Same air-gap promise as the control Skin: no external hosts, self-hosted + fonts only, no CDN.""" + html = HEALTH_HTML.read_text() + urls = re.findall(r"https?://[a-z0-9.\-]+", html, re.I) + external = [u for u in urls if not re.search(r"(127\.0\.0\.1|localhost|w3\.org)", u)] + assert not external, f"health.html references external hosts: {external}" + assert "cdn" not in html.lower(), "health.html pulls from a CDN" + woff2 = list(FONTS.glob("*.woff2")) + for f in woff2: + assert f.name in html, f"health.html doesn't @font-face {f.name}" + + +def test_join_page_exists_and_is_self_contained(): + """The public phone-inject page: QR + Larix steps + live program preview. Must be + air-gapped (self-hosted fonts, no external hosts) EXCEPT the two app-store links, + which are legitimate outbound links a visitor taps to install Larix.""" + assert JOIN_HTML.is_file(), "web/join.html missing" + html = JOIN_HTML.read_text() + urls = re.findall(r"https?://[a-z0-9.\-]+", html, re.I) + # allow the app-store install links (apple + google) — everything else must be local + allowed_ext = ("apps.apple.com", "play.google.com") + external = [u for u in urls + if not re.search(r"(127\.0\.0\.1|localhost|w3\.org)", u) + and not any(a in u for a in allowed_ext)] + assert not external, f"join.html references unexpected external hosts: {external}" + woff2 = list(FONTS.glob("*.woff2")) + for f in woff2: + assert f.name in html, f"join.html doesn't @font-face {f.name}" + + +def test_join_page_served_and_qr_route_not_hardcoded_to_guest12(): + """The Core serves /join, and the QR route validates against the configured guest + slots (so a 3rd 'phone' slot works) rather than a hardcoded guest1/guest2 regex.""" + src = SERVER.read_text() + assert "app.get('/join'" in src, "local-server doesn't serve the /join page" + assert "INGEST_SLOTS" in src, "QR route still hardcodes guest slots (should validate against config)" + assert "guest[12]" not in src, "QR route still has the old guest[12]-only gate" + + +def test_welcome_landing_served_and_self_contained(): + """The public landing (/welcome) routes visitors to Join/Watch/Monitor. Air-gapped + (self-hosted fonts, no external hosts), served by the Core.""" + assert WELCOME_HTML.is_file(), "web/welcome.html missing" + assert "app.get('/welcome'" in SERVER.read_text(), "local-server doesn't serve /welcome" + html = WELCOME_HTML.read_text() + urls = re.findall(r"https?://[a-z0-9.\-]+", html, re.I) + external = [u for u in urls if not re.search(r"(127\.0\.0\.1|localhost|w3\.org)", u)] + assert not external, f"welcome.html references external hosts: {external}" + # the three public entry points must be linked, and NOT the token-gated control Skin + for href in ("/join", "/program/", "/health"): + assert href in html, f"welcome.html doesn't link {href}" + + +def test_health_endpoint_is_read_only(): + """The health endpoint + page must be GET-only — a monitor never mutates the + facility. No POST/PUT/DELETE handlers touch the health surface.""" + src = SERVER.read_text() + assert "app.get('/api/mxl/health'" in src, "local-server missing GET /api/mxl/health" + assert "app.get('/health'" in src, "local-server doesn't serve /health page" + # no mutating verb on the health paths + for verb in ("post", "put", "delete", "patch"): + assert f"app.{verb}('/api/mxl/health'" not in src, \ + f"health endpoint must not expose {verb.upper()}" + + +def test_health_reuses_existing_probe_snapshot(): + """Cost control: the endpoint must READ an existing probe's snapshot, never spawn + its own MXL readers. It prefers the already-running thumbs health.json and falls + back to grain_probe's grains.json — either way, no new process.""" + src = SERVER.read_text() + assert "health.json" in src, "health endpoint doesn't read the thumbs health snapshot" + assert "grains.json" in src, "health endpoint doesn't fall back to the grain-probe snapshot" + # must cache so N viewers collapse to one scrape + assert "HEALTH_TTL" in src or "_healthCache" in src, \ + "health endpoint has no server-side cache (N viewers would each scrape)" diff --git a/tests/test_no_hardcoded_hosts.py b/tests/test_no_hardcoded_hosts.py index c054f51..e5424ab 100644 --- a/tests/test_no_hardcoded_hosts.py +++ b/tests/test_no_hardcoded_hosts.py @@ -41,6 +41,7 @@ "tools/guest-leg-doctor.sh": {"10.0.0.4", "10.0.0.5", "prodbots.com"}, # MXL_VM1_IP/VM2_IP/BACKEND defaults "tools/selector-doctor.sh": {"prodbots.com"}, # MXL_BACKEND_URL default "scripts/quickstart.sh": {"mxlswitcher.com"}, # comment only + "backend/local-server.js": {"mxlswitcher.com"}, # comment only (CORS rationale; override via MXL_HEALTH_CORS) "tools/start-jonas-leg.sh": {"10.0.0.4"}, # MXL_VM1_IP default "tools/tams_shipper.py": {"203.0.113.140"}, # TAMS_HOST default (RFC 5737 placeholder) "tools/backfill-mini.py": {"203.0.113.140"}, # TAMS_HOST default (RFC 5737 placeholder) diff --git a/tools/guest-leg-watchdog.sh b/tools/guest-leg-watchdog.sh new file mode 100644 index 0000000..0b56eca --- /dev/null +++ b/tools/guest-leg-watchdog.sh @@ -0,0 +1,136 @@ +#!/usr/bin/env bash +# guest-leg-watchdog — conservatively keeps a guest's PUBLIC SRT listener available so a +# contributor (phone/Larix) can connect to a FREE slot. +# +# ⚠️ HARD LESSON (HW Oct 9): restarting a guest chain RECREATES its MXL flows, which forces +# the input-selector to RE-WIRE (router thread restarts). Do that every few seconds and the +# selector never stabilizes → PGM sticks, cuts fail, multiview thumbnails flap. The first +# version of this watchdog restarted a chain whenever any internal leg blipped (which is +# NORMAL for an idle slot, and ALSO for a video-only hardware encoder whose audio core +# cycles 'not-linked' forever) — so it churned the selector and broke the whole facility. +# +# So this version is deliberately MINIMAL and SAFE: +# • It only cares about the PUBLIC listener (:PUB) — the thing a contributor connects to. +# Internal legs flapping is NOT our business (that's the idle cycle / source quirks). +# • It NEVER restarts a chain whose VIDEO flow is advancing (a live source → hands off). +# • It only restarts a chain that is BOTH: public port missing AND video flow not +# advancing AND stayed that way across STRIKES consecutive polls — i.e. a genuinely +# dead listener on an idle slot, the only case that silently blocks a new join. +# • It gives up after GIVE_UP restarts that don't help, and has a long cooldown. +# • OFF by default. Enable deliberately: ENABLE=1 (or the systemd unit sets it). +# +# Honestly: on this box the better fix is usually to leave guests alone and let a CONNECTING +# contributor's SRT handshake wake the listener. Use this only if idle listeners are +# observed to die outright. Config (env): +# ENABLE=1 required to do anything (default 0 = observe-only, just logs) +# GUEST_PORTS "name:pub:vleg:aleg" specs, comma-separated +# POLL_SECONDS=30 STRIKES=4 GIVE_UP=3 COOLDOWN_SECONDS=300 +set -u + +ENABLE=${ENABLE:-0} +POLL_SECONDS=${POLL_SECONDS:-30} +STRIKES=${STRIKES:-4} +GIVE_UP=${GIVE_UP:-3} +COOLDOWN_SECONDS=${COOLDOWN_SECONDS:-300} +SELECTOR_CONTAINER=${SELECTOR_CONTAINER:-input-selector} +DOMAIN=${MXL_DOMAIN:-/mxl-domain} +MXL_INFO=${MXL_INFO:-/opt/mxl/tools/mxl-info/mxl-info} +# CPU-spin guard (HW Oct 10): a source that CONNECTS then goes silent (phone stalls, +# Makito unplugged mid-stream) leaves the conform pipeline freewheeling the last frame +# through videorate!videoconvert!v210 at full rate — ~1 full core burned while the flow +# does NOT advance (head frozen). Idle slots that never had a caller sit near 0% (srtsrc +# blocks), so "high CPU + flow not advancing" uniquely identifies the stuck-hot case. We +# restart ONLY the fanout (same safe action as a dead listener) to drop the leg back to +# idle. SPIN_CPU=0 disables this guard; the dead-listener logic below is unaffected. +SPIN_CPU=${SPIN_CPU:-60} # %CPU above which an ingest counts as "spinning" +SPIN_STRIKES=${SPIN_STRIKES:-3} # consecutive polls spinning+not-advancing before heal + +# name:public:video-leg:audio-leg : video-flow-uuid +GUEST_PORTS=${GUEST_PORTS:-"guest1:8890:8990:9090:9e111e00-aaaa-4bbb-8ccc-000000000001,guest2:8891:8991:9091:9e222e00-aaaa-4bbb-8ccc-000000000001,guest3:8895:8995:9095:9e333e00-aaaa-4bbb-8ccc-000000000001,guest4:8896:8996:9096:9e444e00-aaaa-4bbb-8ccc-000000000001,guest5:8897:8997:9097:9e555e00-aaaa-4bbb-8ccc-000000000001,guest6:8898:8998:9098:9e666e00-aaaa-4bbb-8ccc-000000000001"} + +declare -A last_heal heal_count strike spin_strike + +bound() { ss -ulnp 2>/dev/null | grep -q ":$1 "; } + +# %CPU of the guest's ingest WORKER process (the actual `python3 guest_ingest.py `, +# not the `while :; do ...` supervisor shell that wraps it). Integer; empty if not found. +ingest_cpu() { + ps -eo %cpu,args 2>/dev/null \ + | grep -E "[p]ython3 guest_ingest\.py $1 " \ + | awk '{c=$1} END{ if (c!="") printf "%d", c }' +} + +head_idx() { # current head index of a flow, or empty + docker exec "$SELECTOR_CONTAINER" sh -c "$MXL_INFO -d $DOMAIN -f $1 2>/dev/null" 2>/dev/null \ + | grep -i 'Head index' | grep -oE '[0-9]+' | head -1 +} +video_advancing() { # is the guest's VIDEO flow moving? (a live source — never touch it) + local uuid=$1 a b + a=$(head_idx "$uuid"); sleep 1; b=$(head_idx "$uuid") + [ -n "$a" ] && [ -n "$b" ] && [ "$b" -gt "$a" ] 2>/dev/null +} + +heal_chain() { + local name=$1 now; now=$(date +%s) + local last=${last_heal[$name]:-0} + [ $((now - last)) -lt "$COOLDOWN_SECONDS" ] && { logger -t guest-leg-watchdog "skip $name — cooldown"; return; } + heal_count[$name]=$(( ${heal_count[$name]:-0} + 1 )) + if [ "${heal_count[$name]}" -gt "$GIVE_UP" ]; then + logger -t guest-leg-watchdog "GIVE UP on $name (${heal_count[$name]}x) — restart won't fix; leaving alone" + last_heal[$name]=$now; return + fi + last_heal[$name]=$now + logger -t guest-leg-watchdog "HEAL $name public listener (heal #${heal_count[$name]}) — restart fanout only" + # Restart ONLY the fanout (the public listener). Leave the cores alone so we don't + # recreate the MXL flows and churn the selector. + docker restart "${name}-fanout" >/dev/null 2>&1 +} + +logger -t guest-leg-watchdog "started (ENABLE=$ENABLE, poll ${POLL_SECONDS}s, SPIN_CPU=${SPIN_CPU}%) — watches PUBLIC listeners + CPU-spin; never touches a chain with live (advancing) video" +while :; do + IFS=',' read -ra specs <<< "$GUEST_PORTS" + for spec in "${specs[@]}"; do + IFS=':' read -r name pub vleg aleg vuuid <<< "$spec" + docker ps --format '{{.Names}}' 2>/dev/null | grep -qx "${name}-fanout" || continue + + # --- CPU-spin guard (runs for EVERY slot, whatever the listener topology) ---------- + # A source can be present yet silent — spinning the conform hot (~1 core) while the + # flow stays FROZEN. This happens on a bound public listener (Makito unplugged + # mid-stream) AND on an outbound-pull slot (guest5 phone stalls behind the Azure + # relay, no local public port), so the check can't live inside the bound-listener + # branch. "high CPU AND flow not advancing" is the signature; a live source conforms + # at ~1 core too, so advancing is the discriminator (checked last — it costs a 1s sleep). + if [ "$SPIN_CPU" -gt 0 ] 2>/dev/null; then + cpu=$(ingest_cpu "$name") + if [ -n "$cpu" ] && [ "$cpu" -ge "$SPIN_CPU" ] 2>/dev/null && ! video_advancing "$vuuid"; then + spin_strike[$name]=$(( ${spin_strike[$name]:-0} + 1 )) + logger -t guest-leg-watchdog "$name SPINNING (${cpu}% CPU, flow not advancing) (${spin_strike[$name]}/${SPIN_STRIKES})" + if [ "${spin_strike[$name]}" -ge "$SPIN_STRIKES" ]; then + if [ "$ENABLE" = "1" ]; then heal_chain "$name"; else + logger -t guest-leg-watchdog "$name would heal spin, but ENABLE!=1 (observe-only)"; fi + spin_strike[$name]=0 + fi + continue # addressed this slot's health for this poll + fi + spin_strike[$name]=0 + fi + + # --- dead-listener guard (idle slot whose PUBLIC listener died = a real join block) -- + # Only concern: is the PUBLIC listener up? If yes, a contributor can connect → done. + if bound "$pub"; then strike[$name]=0; heal_count[$name]=0; continue; fi + + # Public listener is DOWN. If the video flow is advancing, a source is live on this + # slot anyway — do NOT touch it (restarting would churn the selector). + if video_advancing "$vuuid"; then strike[$name]=0; continue; fi + + # Public down AND no live video = an idle slot whose listener died = a real join block. + strike[$name]=$(( ${strike[$name]:-0} + 1 )) + logger -t guest-leg-watchdog "$name public listener :$pub DOWN, no live video (${strike[$name]}/${STRIKES})" + if [ "${strike[$name]}" -ge "$STRIKES" ]; then + if [ "$ENABLE" = "1" ]; then heal_chain "$name"; else + logger -t guest-leg-watchdog "$name would heal, but ENABLE!=1 (observe-only)"; fi + strike[$name]=0 + fi + done + sleep "$POLL_SECONDS" +done diff --git a/tools/guest_av_listen.sh b/tools/guest_av_listen.sh index c4e7c5c..bdad587 100755 --- a/tools/guest_av_listen.sh +++ b/tools/guest_av_listen.sh @@ -35,13 +35,22 @@ LATENCY_MS="${4:-300}" # our srtsink callers reconnect to them. wait-for-connection=false so the pipeline comes # up and listens even before a contributor or a leg is present. while :; do + # tee branches MUST be decoupled with leaky queues (HW Oct-8): both legs carry the full TS + # (~13 Mbit/s each), so if either srtsink's SRT send buffer backs up momentarily, a PLAIN + # bounded queue fills → backpressures the tee → the OTHER leg's srtsink SRT-errors ("Error on + # SRT socket (0)"). Observed: the audio leg (:${AUDIO_PORT}) errored + its core -5-looped while + # the video leg looked fine — classic unbalanced-tee stall. leaky=downstream makes each branch + # DROP its own oldest TS packets rather than stall the tee, so one slow leg can't kill the other. + # tee allow-not-linked so a torn-down leg (core restart) doesn't error the whole fan-out. gst-launch-1.0 -e \ srtsrc uri="srt://0.0.0.0:${PUBLIC_PORT}?mode=listener&latency=${LATENCY_MS}" \ - ! tsparse set-timestamps=true ! tee name=t \ - t. ! queue ! srtsink uri="srt://127.0.0.1:${VIDEO_PORT}?mode=caller&latency=${LATENCY_MS}" \ - wait-for-connection=false \ - t. ! queue ! srtsink uri="srt://127.0.0.1:${AUDIO_PORT}?mode=caller&latency=${LATENCY_MS}" \ - wait-for-connection=false \ + ! tsparse ! tee name=t allow-not-linked=true \ + t. ! queue leaky=downstream max-size-time=400000000 max-size-buffers=0 max-size-bytes=0 \ + ! srtsink uri="srt://127.0.0.1:${VIDEO_PORT}?mode=caller&latency=${LATENCY_MS}" \ + wait-for-connection=false \ + t. ! queue leaky=downstream max-size-time=400000000 max-size-buffers=0 max-size-bytes=0 \ + ! srtsink uri="srt://127.0.0.1:${AUDIO_PORT}?mode=caller&latency=${LATENCY_MS}" \ + wait-for-connection=false \ || true echo "guest_av_listen: fan-out exited (contributor gone or leg churn) — rebuilding" >&2 sleep 2 diff --git a/tools/mxl-selfheal.sh b/tools/mxl-selfheal.sh index 3fbb156..9125e33 100755 --- a/tools/mxl-selfheal.sh +++ b/tools/mxl-selfheal.sh @@ -34,6 +34,18 @@ SEL_CTR="${SELECTOR_CONTAINER:-input-selector}" RELAY_CTR="${RELAY_CONTAINER:-mxl2webrtc}" DOMAIN="${MXL_DOMAIN:-/mxl-domain}" KEYER_FLOW="" # discovered lazily +# Thumbs-probe heal (HW Oct 10): the thumbs probe (mxl_thumbs.py) readers can wedge on a +# specific slot — a live Makito/PTZ then shows 'unknown' on the Health Skin even though its +# flow is advancing (mxl-info sees it; the probe's reader doesn't). It needs a restart to +# re-attach. heal_thumbs detects that exact mismatch (Core reports a wired source NOT 'ok' +# while its flow IS advancing) and restarts the thumbs unit. STRIKES avoids restarting during +# the probe's own ~30s startup. CORE_URL = the Skin's health API; THUMBS_UNIT = systemd unit. +THUMBS_UNIT="${MXL_THUMBS_UNIT:-mxl-thumbs.service}" +CORE_URL="${MXL_CORE_URL:-http://127.0.0.1:3100}" +THUMBS_HEAL="${MXL_THUMBS_HEAL:-1}" # 0 disables the thumbs-probe heal +THUMBS_STRIKES="${MXL_THUMBS_STRIKES:-3}" # consecutive mismatched polls before restart +_thumbs_strike=0; _thumbs_last_heal=0 +_sel_api_last_heal=0 # cooldown for the "selector HTTP API died" container restart # Use `docker` directly if we can (quickstart runs as root), else `sudo docker`. if docker ps >/dev/null 2>&1; then DOCKER="docker"; else DOCKER="sudo docker"; fi @@ -94,7 +106,29 @@ keyer_out_flow(){ # ── heal 1: selector down → restart with its inputs + re-cut to slot 0 ────────── heal_selector(){ local st running - st=$(_get 9604 /pipeline/status) || return 0 + st=$(_get 9604 /pipeline/status) + # API UNREACHABLE (empty body / http 000) while the container is up = the selector's HTTP + # control server died but the router thread keeps running (HW Oct 10: container "Up 7h", + # nothing listening on 9604, PGM frozen, every cut/status 000 — selfheal's own probes blind). + # A /pipeline/start can't fix a dead API; the container must be restarted to respawn it. + # Guard: only restart if the container is actually running (don't fight a stopped container) + # and respect the publish cooldown pattern via a dedicated timestamp to avoid restart storms. + if [ -z "$st" ]; then + local now; now=$(date +%s) + if [ $((now - _sel_api_last_heal)) -ge 120 ] && $DOCKER ps --format '{{.Names}}' 2>/dev/null | grep -qx "$SEL_CTR"; then + log "SELECTOR API UNREACHABLE (container up, :9604 dead) — restarting $SEL_CTR" + $DOCKER restart "$SEL_CTR" >/dev/null 2>&1 + _sel_api_last_heal=$now + # wait for the API to answer, then let the next pass (or pgm-heal) re-wire + republish + local i=0; while [ "$i" -lt 20 ]; do curl -s -m3 "http://127.0.0.1:9604/pipeline/status" >/dev/null 2>&1 && break; i=$((i+1)); sleep 2; done + # after an API restart the router comes up but the encoder must be re-pointed at the + # selector output (its flow object was recreated) — republish via the proven path. + local out; out=$(sel_out_flow) + [ -n "$out" ] && _enc_restart "$out" + log "selector API restarted + encoder republished" + fi + return 0 + fi running=$(printf '%s' "$st" | _json running) [ "$running" = "True" ] && return 0 # healthy local inputs; inputs=$(selector_inputs_json) @@ -124,10 +158,120 @@ heal_relay(){ log "relay restarted video-only" } -pass(){ heal_selector; heal_relay; } +# ── heal 3: PGM publish dropped → republish encoder (the grey-screen fix) ─────── +# The recurring grey screen: the selector output keeps ADVANCING (source + cut fine) but +# mxl2webrtc's publish to mediamtx silently dies downstream (ICE/renegotiation/mediamtx +# hiccup) — the relay's /pipeline/status still says running:true, so heal_relay above +# doesn't catch it. The only reliable signal is mediamtx itself: 0 program bytes while +# the selector output is moving = publish dropped. Fix = restart the encoder read of the +# current program flow (same body pgm-heal uses). We require the selector to be ADVANCING +# first so we never "heal" a genuinely-stopped program into a busy-loop. +SEL_OUT_FLOW="" +sel_out_flow(){ + [ -n "$SEL_OUT_FLOW" ] && { printf '%s' "$SEL_OUT_FLOW"; return; } + SEL_OUT_FLOW=$(_get 9604 /pipeline/status | _json output_flow_uuid) + printf '%s' "$SEL_OUT_FLOW" +} +headidx(){ $DOCKER exec "$SEL_CTR" sh -c "/opt/mxl/tools/mxl-info/mxl-info -d $DOMAIN -f $1 2>/dev/null" 2>/dev/null | grep -i 'Head index' | grep -oE '[0-9]+' | head -1; } +mtx_pgm_bytes(){ _get 9997 /v3/paths/list | python3 -c "import sys,json;print(sum(p.get('bytesReceived',0) for p in json.load(sys.stdin).get('items',[])))" 2>/dev/null; } +heal_publish(){ + local out; out=$(sel_out_flow) + [ -n "$out" ] || return 0 + # is the program source actually advancing? (don't republish a legitimately-idle PGM) + local a b; a=$(headidx "$out"); sleep 1; b=$(headidx "$out") + [ -n "$a" ] && [ -n "$b" ] && [ "$b" -gt "$a" ] 2>/dev/null || return 0 # not advancing — nothing to publish + # selector is moving. is mediamtx actually receiving program bytes? + local p0 p1; p0=$(mtx_pgm_bytes); sleep 2; p1=$(mtx_pgm_bytes) + [ -n "$p0" ] && [ -n "$p1" ] || return 0 # can't read mediamtx — leave alone + [ "$p1" -gt "$p0" ] 2>/dev/null && return 0 # publishing fine + # ADVANCING but NO bytes to mediamtx = the grey screen. Republish the encoder on the + # SELECTOR output (keyer bypassed for stability — HW Oct 10: the gst-keyer throttled the + # whole chain to ~300KB/s and its flow 86efffa4 keeps a stale advancing head even when the + # keyer is stopped, so an earlier version kept republishing onto that DEAD keyer flow every + # 30s = permanent 0 bytes. Always use the selector output here.). To re-enable keyed PGM, + # fix the keyer's throughput first, then revisit. src is '' only transiently. + local src="$out" + # The encoder sticks on its current flow if `start` is sent while running, and a container + # restart auto-starts it on its DEFAULT (keyer) flow — both leave PGM dead. So STOP until + # confirmed stopped, then START, then VERIFY it took (one retry). No container restart. + log "PGM PUBLISH DROPPED (selector advancing, mediamtx 0 bytes) — republishing encoder on selector ${src}" + _enc_restart "$src" + log "encoder republished" +} + +# Re-point the encoder at $1 reliably: stop-until-stopped -> start -> verify (one retry). +_enc_restart(){ + local want="$1" i r now + curl -s -m3 "http://127.0.0.1:9601/pipeline/status" >/dev/null 2>&1 || { $DOCKER restart "$RELAY_CTR" >/dev/null 2>&1; sleep 8; } + i=0; while [ "$i" -lt 6 ]; do + _post 9601 /pipeline/stop '{}' >/dev/null + sleep 2 + r=$(_get 9601 /pipeline/status | _json running) + [ "$r" = "False" ] && break; i=$((i+1)) + done + _post 9601 /pipeline/start "{\"domain_path\":\"$DOMAIN\",\"video_flow_uuid\":\"$want\",\"use_mediamtx\":true,\"encoder\":{\"tune\":4,\"speed_preset\":2,\"bitrate\":6000,\"key_int_max\":30}}" >/dev/null + sleep 3 + now=$(_get 9601 /pipeline/status | _json video_flow_uuid) + if [ "$now" != "$want" ]; then + i=0; while [ "$i" -lt 6 ]; do _post 9601 /pipeline/stop '{}' >/dev/null; sleep 2 + [ "$(_get 9601 /pipeline/status | _json running)" = "False" ] && break; i=$((i+1)); done + _post 9601 /pipeline/start "{\"domain_path\":\"$DOMAIN\",\"video_flow_uuid\":\"$want\",\"use_mediamtx\":true,\"encoder\":{\"tune\":4,\"speed_preset\":2,\"bitrate\":6000,\"key_int_max\":30}}" >/dev/null + fi +} + +# ── heal 4: thumbs probe wedged on a live source → restart it (keeps Makito/PTZ visible) ─ +# Symptom: a source wired into the selector is ADVANCING, but the Core's health reports it +# NOT 'ok' (unknown/dead/frozen) — the thumbs reader for that slot wedged and the camera +# vanishes from the Health Skin. Restart the thumbs unit so its readers re-attach. Guards: +# only counts a strike when a flow is genuinely advancing (never restarts for a truly idle +# source), needs THUMBS_STRIKES in a row (rides out the probe's own startup), and a 120s +# cooldown so a restart that takes a while to settle doesn't trigger a restart storm. +heal_thumbs(){ + [ "$THUMBS_HEAL" = "1" ] || return 0 + local now; now=$(date +%s) + [ $((now - _thumbs_last_heal)) -lt 120 ] && return 0 # cooling down after a restart + # Pull the Core's per-source view once (it already joins thumbs state + wiring, cheaply). + local health; health=$(_get_url "$CORE_URL/api/mxl/health") || return 0 + [ -n "$health" ] || return 0 + # Any wired, non-stable source reported not-ok whose flow IS advancing = a wedged reader. + local wedged; wedged=$(printf '%s' "$health" | python3 -c ' +import sys, json +try: d = json.load(sys.stdin) +except Exception: sys.exit(0) +for s in d.get("sources", []): + # stable correspondent slots legitimately sit "standby" with no source — skip them + if s.get("stable"): continue + if s.get("state") not in ("ok",): + print(s.get("name","?")); break +' 2>/dev/null) + if [ -z "$wedged" ]; then _thumbs_strike=0; return 0; fi + # The Core said a source is not-ok; confirm its FLOW is actually advancing before blaming + # the probe (if the flow is genuinely dead, that is a source problem, not a thumbs wedge). + # We only have names here; the selector output advancing + >2 sources ok is a good proxy + # that the facility is live and it is the probe (not the domain) that is broken. + local oks; oks=$(printf '%s' "$health" | python3 -c ' +import sys, json +try: d = json.load(sys.stdin) +except Exception: print(0); sys.exit(0) +print(sum(1 for s in d.get("sources", []) if s.get("state")=="ok")) +' 2>/dev/null) + [ -n "$oks" ] && [ "$oks" -ge 1 ] 2>/dev/null || { _thumbs_strike=0; return 0; } # whole probe cold -> let it start + _thumbs_strike=$((_thumbs_strike + 1)) + log "thumbs probe looks wedged ($wedged not-ok while others ok) (${_thumbs_strike}/${THUMBS_STRIKES})" + if [ "$_thumbs_strike" -ge "$THUMBS_STRIKES" ]; then + log "restarting $THUMBS_UNIT so its readers re-attach" + systemctl restart "$THUMBS_UNIT" >/dev/null 2>&1 || sudo systemctl restart "$THUMBS_UNIT" >/dev/null 2>&1 + _thumbs_strike=0; _thumbs_last_heal=$now + fi +} + +# curl to a full URL (the thumbs heal talks to the Core's health API, not a bare port). +_get_url(){ curl -s -m 5 "$1" 2>/dev/null; } + +pass(){ heal_selector; heal_relay; heal_publish; heal_thumbs; } if [ "${1:-}" = "--watch" ]; then - log "watching (every ${INTERVAL}s) — selector + relay" + log "watching (every ${INTERVAL}s) — selector + relay + publish + thumbs" while :; do pass; sleep "$INTERVAL"; done else pass diff --git a/tools/run-stabilizer.sh b/tools/run-stabilizer.sh new file mode 100644 index 0000000..3134916 --- /dev/null +++ b/tools/run-stabilizer.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +# run-stabilizer.sh — supervise a flow_stabilizer for one guest slot. +# +# The stabilizer reads the VOLATILE guest flow (9eNNNe00) and writes a STABLE flow +# (57abNe00) that exists from boot and never dies — so the input-selector can be wired +# ONCE to the stable flows and NEVER re-wired when a correspondent joins/drops. That's +# what keeps PGM from ever being interrupted on a live join (docs/NEVER-INTERRUPT-PGM.md). +# +# Runs flow_stabilizer.py inside a container that has mxlsrc/mxlsink + python-gi (the +# html5-keyer image). systemd supervises this wrapper; the stabilizer exits on a deep +# wedge and gets relaunched. +# +# Flow UUID convention: guestN video = 9e{N}{N}{N}e00-...; stable = 57ab{N}e00-... +set -eu + +N="${1:?usage: run-stabilizer.sh }" +N="${N#guest}" # accept "guest3" or "3" +CONTAINER="${STAB_CONTAINER:-html5-keyer}" # image with mxlsrc/mxlsink + gi +DOMAIN="${MXL_DOMAIN:-/mxl-domain}" +VOL="9e${N}${N}${N}e00-aaaa-4bbb-8ccc-000000000001" +STABLE="57ab${N}e00-aaaa-4bbb-8ccc-000000000001" +SCRIPT_HOST="${STAB_SCRIPT:-/home/guycochran/mxl-switcher/tools/flow_stabilizer.py}" + +for i in $(seq 1 30); do + docker ps --format '{{.Names}}' | grep -qx "$CONTAINER" && break + sleep 2 +done +docker ps --format '{{.Names}}' | grep -qx "$CONTAINER" || { echo "stabilizer guest$N: $CONTAINER not up" >&2; exit 1; } + +# refresh the script into the container, reap any stale stabilizer for THIS flow only +docker cp "$SCRIPT_HOST" "$CONTAINER":/tmp/flow_stabilizer.py 2>/dev/null || true +docker exec "$CONTAINER" pkill -9 -f "flow_stabilizer.py guest$N " 2>/dev/null || true +sleep 1 +echo "stabilizer guest$N: $VOL -> $STABLE" +# foreground so systemd owns the lifecycle; exits on deep wedge -> unit restarts it +exec docker exec "$CONTAINER" python3 -u /tmp/flow_stabilizer.py "guest$N" "$VOL" "$STABLE" "Guest $N Stable" diff --git a/web/health.html b/web/health.html new file mode 100644 index 0000000..efc6205 --- /dev/null +++ b/web/health.html @@ -0,0 +1,272 @@ + + + + + +MXL Health + + + +
+
+
MXL Healthread-only facility monitor
+
+ connecting… + ← Switcher +
+ +

System

+
+
CPU
—
+
Memory
—
+
Load (1m)
—
—
+
Viewers
—
WebRTC program
+
+ +

Sources

+
+
waiting for selector…
+
+ +

Grain flow

+
+ + + +
FlowStateGrains/sUnique fps
waiting for probe…
+
+ +

Pipeline writers

+
+
Selector
state—
+
Keyer
state—
+
Encoder
state—
+
+ +
+ Grain rate and unique-frames/s come from the long-lived grain probe: a flow can tick 30 grains/s while the + picture is frozen (a repeat-wedge) — when that happens unique fps collapses + and the row turns amber. This page never controls the facility; it only reads. +
+
+ + + + diff --git a/web/join.html b/web/join.html new file mode 100644 index 0000000..58dc2a3 --- /dev/null +++ b/web/join.html @@ -0,0 +1,272 @@ + + + + + +Join the show — MXL + + + +
+
+
MXLJoin the show
+
+ Watch program → +
+ +

Be on air in 30 seconds

+

Put your phone's camera
into a live broadcast.

+

No app store account, no login. Scan the code with the free Larix Broadcaster app and your camera becomes a switchable source in a real production control room — running on a single $400 computer.

+ +
+
+
+ Scan to join +
Scan to go live
+
loading…
+
+ +
+ +
+
    +
  1. + Install Larix Broadcaster +
    Free, on iPhone & Android. It's the app that sends your camera over SRT.
    +
    + iPhone — App Store ↗ + Android — Google Play ↗ +
    +
    2. +
  2. + Scan the code +
    Open your phone camera (or Larix's built-in scanner) and point it at the QR. It loads all the settings for you — tap to import.
    +
    3. +
  3. + Tap the red button +
    In Larix, hit record/stream. You're now publishing to the switcher. Watch yourself appear in the program below.
    +
    4. +
+ +
+
Manual settings (if the QR won't scan)
+
ProtocolSRT (Caller)
+
URL—
+
Latency1000 ms
+
ModeAudio + Video
+ +
+ +
+

Can we see you? — your tile lights up when you're live

+
+
Waiting for a source… go live in Larix and watch your tile appear below.
+
+ +
+

Program output — what's on air

+
+ ● LIVE + +
+
+
+
+ +
+ An open-source MXL (Media eXchange Layer) switcher. Your phone's video travels as SRT into a shared-memory media domain and is composited live. How it works → +
+
+ + + + diff --git a/web/local.html b/web/local.html index 7b958dc..16f49e4 100644 --- a/web/local.html +++ b/web/local.html @@ -41,6 +41,8 @@ font-weight:500;letter-spacing:.5px;color:var(--dim);border:1px solid var(--line); border-radius:6px;padding:5px 9px;text-transform:uppercase} .chip .dot{width:7px;height:7px;border-radius:50%;background:var(--faint)} + a.navlink{text-decoration:none;color:var(--dim);transition:color .15s,border-color .15s} + a.navlink:hover{color:var(--acc);border-color:var(--acc)} .chip.ok{color:var(--ink)} .chip.ok .dot{background:var(--grn);box-shadow:0 0 7px var(--grn-glow)} .chip.bad .dot{background:var(--red);box-shadow:0 0 7px var(--red-glow)} .chip b{color:var(--ink);font-weight:600} @@ -156,6 +158,8 @@
MXL Switcher
+ Health ↗ + Join ↗
connecting…
PGM —
@@ -223,6 +227,9 @@

Multiview

free Larix Broadcaster app, or paste the URL into OBS / ffmpeg.

+

Running a public demo? Send visitors the self-serve page: + /welcome ↗ + · /join ↗

diff --git a/web/welcome.html b/web/welcome.html new file mode 100644 index 0000000..eaa2a04 --- /dev/null +++ b/web/welcome.html @@ -0,0 +1,104 @@ + + + + + +MXL — a live switcher on one $400 box + + + +
+
Open-source · MXL / Dynamic Media Facility
+

A broadcast control room, as software

+

One $400 computer
is running a live show.

+

Real cameras, phone guests, playout, switching and streaming — all in shared memory on commodity hardware. Jump in: put your own camera on air, watch the program, or watch the machine work.

+

checking the facility…

+ +
+ +
📱
+

Put your phone on air

+

Scan a QR with the free Larix app and your camera becomes a live switchable source. ~30 seconds, no account.

+
Join the show →
+ + +
📺
+

Watch the program

+

The live program output, straight from the switcher over WebRTC — sub-second, encoded once for everyone.

+
Watch live →
+ + +
📊
+

Watch the machine

+

Live health: grain flow per source, CPU/memory, and viewer count — see exactly how little a $400 box breaks a sweat.

+
Open the monitor →
+ +
+ +

+ New to this? What's a "grain"? → +  ·  Built on the EBU/AMWA Media eXchange Layer. +

+
+ + + +