From e8a0f0e7bc5dbe2c7afcb097bb0d746d4a25284e Mon Sep 17 00:00:00 2001
From: Cursor Agent <cursoragent@cursor.com>
Date: Tue, 23 Jun 2026 04:28:21 +0000
Subject: [PATCH 01/28] feat(agent-runtime): autonomous OpenMontage agent
 runtime for gateway mode=agent (Gap B Phase 1)

OpenMontage has no Python pipeline runner (host LLM is the orchestrator), so mode=agent needs a new
unattended runtime. Adds services/agent_runtime/:
- llm.py: urllib-only cloud LLM client (anthropic|openai|kakeya|stub, auto-detect from keys).
- run.py: LLM -> brief/script/scene_plan (schema-validated + checkpointed) -> assets via
  video_selector (auto best provider) + tts_selector + music_gen -> edit -> audio_mixer +
  video_compose (ffmpeg) -> final mp4. AGENT_RUNTIME_FAKE_ASSETS=1 for offline CI.
- tests: stub-LLM + ffmpeg end-to-end (valid h264 + checkpoints) + JSON extraction. 23 pass.
ADR 0016 + loop-log iteration 41. Needs a cloud LLM key to run live; quality is provider-bounded.

Co-authored-by: FluffyAIcode <FluffyAIcode@users.noreply.github.com>
---
 docs/adr/0001-kakeya-integration-loop-log.md |  26 ++
 docs/adr/0016-autonomous-agent-runtime.md    |  71 +++++
 services/agent_runtime/__init__.py           |   1 +
 services/agent_runtime/llm.py                | 117 ++++++++
 services/agent_runtime/run.py                | 277 +++++++++++++++++++
 tests/tools/test_agent_runtime.py            |  78 ++++++
 6 files changed, 570 insertions(+)
 create mode 100644 docs/adr/0016-autonomous-agent-runtime.md
 create mode 100644 services/agent_runtime/__init__.py
 create mode 100644 services/agent_runtime/llm.py
 create mode 100644 services/agent_runtime/run.py
 create mode 100644 tests/tools/test_agent_runtime.py

diff --git a/docs/adr/0001-kakeya-integration-loop-log.md b/docs/adr/0001-kakeya-integration-loop-log.md
index 9b5128f76..6c2631e78 100644
--- a/docs/adr/0001-kakeya-integration-loop-log.md
+++ b/docs/adr/0001-kakeya-integration-loop-log.md
@@ -1122,6 +1122,32 @@ auto-recovery. (Caveat: the ~84 G I2V model re-downloads unless `HF_HOME` is on
 
 ---
 
+## Iteration 41 — Gap B Phase 1: autonomous agent runtime (mode=agent) — ADR 0016
+
+**Finding:** OpenMontage has **no Python pipeline runner** — the host LLM is normally the
+orchestrator. So `mode=agent` integration = building a new **autonomous agent runtime** (a cloud-LLM
+reasoner + tool-calling loop). In-repo LLM is only `kakeya_llm` (mechanical).
+
+**Built:** `services/agent_runtime/` (`AGENT_RUNTIME_CMD`):
+- `llm.py` — urllib-only cloud LLM client (`AGENT_LLM=anthropic|openai|kakeya|stub`, auto-detect).
+- `run.py` — unattended orchestrator: LLM → brief→script→scene_plan (schema-validated + checkpointed
+  via `lib/checkpoint`), assets via `video_selector` (auto-routes to best provider; premium if
+  configured, else WAN draft) + `tts_selector` + `music_gen`, deterministic edit → `audio_mixer` +
+  `video_compose` (ffmpeg) → final mp4. `AGENT_RUNTIME_FAKE_ASSETS=1` for CI.
+
+**Topology:** agent runtime on the head Mac CPU (cloud LLM, doesn't steal video GPU); headless = video
+worker; vast = heavy gen — the cost/stability recommendation.
+
+**Validated offline (no keys/GPU):** stub LLM + ffmpeg fixtures → full plan→validate→checkpoint→
+assets→mix→compose → valid h264 mp4 + checkpoints. 23 tests pass (runtime+gateway+pipeline).
+
+**Honest scope:** MVP "auto" path (no research/proposal/approval/reviewer governance). **Needs a
+cloud LLM key** to actually run; quality is provider-bounded (WAN draft unless a premium video
+provider key is set — then `video_selector` auto-upgrades). Not yet "all skills". Live `mode=agent`
+test on agent.kakeya.ai pending the key.
+
+---
+
 ## Open follow-ups (next iterations)
 - **Phase 2b — native gRPC transport.** Add an optional `kakeya` Python SDK transport
   for the bounded-memory long-context path (W3), behind the same tool, once the proto
diff --git a/docs/adr/0016-autonomous-agent-runtime.md b/docs/adr/0016-autonomous-agent-runtime.md
new file mode 100644
index 000000000..df65f7bff
--- /dev/null
+++ b/docs/adr/0016-autonomous-agent-runtime.md
@@ -0,0 +1,71 @@
+# ADR 0016 — Autonomous agent runtime for the public gateway (Gap B, Phase 1)
+
+**Status:** Accepted; MVP implemented + offline-validated. Live run pending a cloud LLM key.
+**Date:** 2026-06-23
+**Related:** ADR 0011/0013 (agent gateway, re-anchor), AGENT_GUIDE.md (instruction-driven model).
+
+## 1. Problem (Gap B)
+
+`agent.kakeya.ai` only did **bare text→video** through the WAN cluster. The gateway's `mode=agent`
+requires an `AGENT_RUNTIME_CMD`, which was unattached (`agent_runtime=false`) — so none of
+OpenMontage's pipeline/skills (script → scene_plan → assets via selectors → compose) ran. The user
+asked to **integrate the OpenMontage agent** so the public service runs a real multi-stage pipeline.
+
+## 2. Key finding
+
+OpenMontage is **instruction-driven**: there is **no Python pipeline runner** — the *host LLM*
+(Cursor/Claude) is normally the orchestrator, reading manifests + director skills and driving tools.
+To run **unattended** behind the gateway, we must supply our own LLM reasoner. In-repo LLM access is
+only `kakeya_llm`/`llm_selector` (mechanical text). So this ADR introduces a new **autonomous agent
+runtime**.
+
+## 3. Decision
+
+New module `services/agent_runtime/` exposed as `AGENT_RUNTIME_CMD`
+(`python -m services.agent_runtime.run --prompt … --out …`):
+
+- **`llm.py`** — dependency-free (urllib) cloud-LLM client: `AGENT_LLM = anthropic|openai|kakeya|stub`
+  (auto-detected from `ANTHROPIC_API_KEY`/`OPENAI_API_KEY`/`KAKEYA_ENDPOINT`). `complete_json` parses
+  + retries. Creative reasoning runs here; the host agent is no longer required at runtime.
+- **`run.py`** — minimal unattended orchestrator that reuses the real machinery:
+  - LLM produces **brief → script → scene_plan** (each schema-validated via `schemas.artifacts`
+    and checkpointed via `lib/checkpoint`, exactly like the governed pipeline; director-skill text
+    is loaded as LLM context — "intelligence in the skills").
+  - assets: per-scene video via **`video_selector`** (auto-routes to the BEST available provider —
+    premium Seedance/Kling/Veo if configured, else the local WAN draft cluster), narration via
+    **`tts_selector`**, one music track via **`music_gen`** (skipped when unconfigured).
+  - deterministic **edit_decisions** → **`audio_mixer`** + **`video_compose`** (ffmpeg) → final mp4.
+- `AGENT_RUNTIME_FAKE_ASSETS=1` swaps providers for ffmpeg fixtures so the harness is CI-testable
+  with no keys/GPU.
+
+**Topology** (per the cost/stability recommendation): agent runtime runs on the **head Mac CPU**
+(cloud LLM) next to the gateway; headless Mac stays a video worker; vast does heavy generation. The
+agent uses a cloud LLM (light CPU/network) — it does **not** steal GPU from video.
+
+## 4. Honest scope / limits
+
+- This is an **MVP "auto" path**, not the full governed `animated-explainer` pipeline (no
+  research/proposal, human-approval gates, reviewer governance, or per-stage director nuance).
+- **Needs a cloud LLM key** to reason (none configured). Without it, `mode=agent` stays honestly
+  "not attached".
+- **Output quality is provider-bounded.** With only the WAN cluster, clips are draft quality.
+  Configure a **premium video provider** (e.g. fal.ai/Seedance) and `video_selector` auto-upgrades —
+  that is the real path to production-grade. TTS/music/images need their own keys.
+- Not "all skills integrated": skills light up as their providers are configured + as more pipelines
+  are wired. This is the first integration, not the finish line.
+
+## 5. Validation
+
+`tests/tools/test_agent_runtime.py` (offline, no keys/GPU): stub LLM emits valid brief/script/
+scene_plan; `AGENT_RUNTIME_FAKE_ASSETS=1` → full plumbing (plan → validate → checkpoint → per-scene
+assets → audio mix → ffmpeg compose) → **valid h264 mp4** + checkpoints for idea/script/scene_plan/
+assets/edit. JSON-extraction unit test. 2 pass (23 with gateway+pipeline suites).
+
+## 6. Next
+
+- Deploy: set `AGENT_RUNTIME_CMD="<venv> -m services.agent_runtime.run"` + a cloud LLM key on the
+  head; live-test `mode=agent` via `agent.kakeya.ai`.
+- Configure premium video provider for production-grade clips (the quality answer from the
+  draft-vs-premium discussion).
+- Grow toward the governed pipeline (proposal/approval/reviewer, more director skills) and more
+  skills (avatar, images, music) as keys are added.
diff --git a/services/agent_runtime/__init__.py b/services/agent_runtime/__init__.py
new file mode 100644
index 000000000..6101e6540
--- /dev/null
+++ b/services/agent_runtime/__init__.py
@@ -0,0 +1 @@
+"""OpenMontage autonomous agent runtime (gateway mode=agent)."""
diff --git a/services/agent_runtime/llm.py b/services/agent_runtime/llm.py
new file mode 100644
index 000000000..cc86642bf
--- /dev/null
+++ b/services/agent_runtime/llm.py
@@ -0,0 +1,117 @@
+"""LLM client for the OpenMontage agent runtime.
+
+The OpenMontage "agent" is normally the host LLM (Cursor/Claude). To run the pipeline
+UNATTENDED behind the gateway's mode=agent, we need our own cloud-LLM reasoner. This is a
+thin, dependency-free client (urllib only) over the providers' HTTP APIs, selected by env:
+
+  AGENT_LLM = anthropic | openai | kakeya | stub   (auto-detected from keys if unset)
+  ANTHROPIC_API_KEY / OPENAI_API_KEY / KAKEYA_ENDPOINT
+  AGENT_LLM_MODEL  (optional override)
+
+Creative reasoning (brief/script/scene_plan/edit) runs here; mechanical text can still use
+the in-repo kakeya_llm tool. `complete_json` returns a parsed dict, retrying once on bad JSON.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import re
+import urllib.request
+from typing import Any, Callable, Optional
+
+_DEFAULT_MODELS = {
+    "anthropic": "claude-3-5-sonnet-latest",
+    "openai": "gpt-4o-mini",
+    "kakeya": "kakeya-local",
+}
+
+
+class LLM:
+    def __init__(self, provider: Optional[str] = None, model: Optional[str] = None,
+                 stub: Optional[Callable[[str, str], str]] = None):
+        self.stub = stub
+        self.provider = (provider or os.environ.get("AGENT_LLM", "")).strip().lower()
+        if not self.provider:
+            if stub is not None:
+                self.provider = "stub"
+            elif os.environ.get("ANTHROPIC_API_KEY"):
+                self.provider = "anthropic"
+            elif os.environ.get("OPENAI_API_KEY"):
+                self.provider = "openai"
+            elif os.environ.get("KAKEYA_ENDPOINT"):
+                self.provider = "kakeya"
+            else:
+                self.provider = "stub"
+        self.model = model or os.environ.get("AGENT_LLM_MODEL", "") or _DEFAULT_MODELS.get(self.provider, "")
+
+    # --- transports ---------------------------------------------------------
+    def complete(self, system: str, user: str, max_tokens: int = 3000) -> str:
+        if self.provider == "stub":
+            if self.stub is None:
+                raise RuntimeError("AGENT_LLM=stub but no stub callable provided")
+            return self.stub(system, user)
+        if self.provider == "anthropic":
+            return self._anthropic(system, user, max_tokens)
+        if self.provider == "openai":
+            return self._openai(system, user, max_tokens)
+        if self.provider == "kakeya":
+            return self._openai(system, user, max_tokens, kakeya=True)
+        raise RuntimeError(f"unknown AGENT_LLM provider: {self.provider}")
+
+    def _post(self, url: str, headers: dict, body: dict, timeout: int = 180) -> dict:
+        req = urllib.request.Request(url, data=json.dumps(body).encode(), headers=headers)
+        with urllib.request.urlopen(req, timeout=timeout) as r:
+            return json.loads(r.read())
+
+    def _anthropic(self, system: str, user: str, max_tokens: int) -> str:
+        out = self._post(
+            "https://api.anthropic.com/v1/messages",
+            {"Content-Type": "application/json", "x-api-key": os.environ["ANTHROPIC_API_KEY"],
+             "anthropic-version": "2023-06-01"},
+            {"model": self.model, "max_tokens": max_tokens, "system": system,
+             "messages": [{"role": "user", "content": user}]})
+        return "".join(b.get("text", "") for b in out.get("content", []))
+
+    def _openai(self, system: str, user: str, max_tokens: int, kakeya: bool = False) -> str:
+        if kakeya:
+            base = os.environ["KAKEYA_ENDPOINT"].rstrip("/")
+            headers = {"Content-Type": "application/json"}
+        else:
+            base = os.environ.get("OPENAI_BASE_URL", "https://api.openai.com").rstrip("/")
+            headers = {"Content-Type": "application/json",
+                       "Authorization": f"Bearer {os.environ['OPENAI_API_KEY']}"}
+        out = self._post(base + "/v1/chat/completions", headers,
+                         {"model": self.model, "max_tokens": max_tokens,
+                          "messages": [{"role": "system", "content": system},
+                                       {"role": "user", "content": user}]})
+        return out["choices"][0]["message"]["content"]
+
+    # --- JSON helper --------------------------------------------------------
+    @staticmethod
+    def _extract_json(text: str) -> dict:
+        # strip ```json fences, then take the outermost {...}
+        text = re.sub(r"```(?:json)?", "", text).strip()
+        start, depth = text.find("{"), 0
+        if start < 0:
+            raise ValueError("no JSON object in LLM output")
+        for i in range(start, len(text)):
+            if text[i] == "{":
+                depth += 1
+            elif text[i] == "}":
+                depth -= 1
+                if depth == 0:
+                    return json.loads(text[start:i + 1])
+        raise ValueError("unbalanced JSON in LLM output")
+
+    def complete_json(self, system: str, user: str, retries: int = 1) -> dict:
+        last = None
+        for attempt in range(retries + 1):
+            txt = self.complete(system, user + ("\n\nReturn ONLY valid JSON." if attempt == 0
+                                                else f"\n\nYour previous output failed to parse ({last}). "
+                                                     "Return ONLY a single valid JSON object, no prose."))
+            try:
+                return self._extract_json(txt)
+            except Exception as exc:  # noqa: BLE001
+                last = exc
+        raise RuntimeError(f"LLM did not return valid JSON after {retries + 1} attempts: {last}")
diff --git a/services/agent_runtime/run.py b/services/agent_runtime/run.py
new file mode 100644
index 000000000..668de16dd
--- /dev/null
+++ b/services/agent_runtime/run.py
@@ -0,0 +1,277 @@
+"""OpenMontage autonomous agent runtime (MVP) — `AGENT_RUNTIME_CMD` for the gateway's mode=agent.
+
+OpenMontage is instruction-driven (the host LLM is normally the orchestrator). This module is a
+minimal UNATTENDED orchestrator so the public gateway can run a real multi-stage pipeline from a
+brief, instead of a bare text->video call:
+
+    brief (LLM) -> script (LLM) -> scene_plan (LLM)
+      -> assets: per-scene video via video_selector (auto-routes to the BEST available provider:
+                 premium Seedance/Kling/Veo if configured, else the local WAN draft cluster),
+                 + narration via tts_selector, + one music track via music_gen (when available)
+      -> edit_decisions (deterministic cuts) -> compose via audio_mixer + video_compose (ffmpeg)
+      -> final mp4 at --out.
+
+Each canonical artifact is schema-validated and checkpointed (lib/checkpoint), exactly like the
+governed pipeline. Creative decisions come from the LLM; only mechanical wiring is in code.
+
+Honest scope: this is an MVP "auto" path, not the full governed animated-explainer pipeline
+(no research/proposal/human-approval/reviewer governance). Output quality is bounded by the
+configured providers — configure a premium video provider key to get production-grade clips;
+with only the WAN cluster it is draft quality.
+
+CLI:  python -m services.agent_runtime.run --prompt "..." --out /path/final.mp4
+Env:  AGENT_LLM + key (see llm.py). AGENT_RUNTIME_FAKE_ASSETS=1 -> ffmpeg fixtures (offline tests).
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import subprocess
+import sys
+import uuid
+from pathlib import Path
+
+REPO = Path(__file__).resolve().parents[2]
+sys.path.insert(0, str(REPO))
+
+from services.agent_runtime.llm import LLM  # noqa: E402
+
+FAKE = os.environ.get("AGENT_RUNTIME_FAKE_ASSETS", "0").lower() in ("1", "true", "yes")
+MAX_SCENES = int(os.environ.get("AGENT_MAX_SCENES", "5"))
+CLIP_W = int(os.environ.get("AGENT_CLIP_WIDTH", "1280"))
+CLIP_H = int(os.environ.get("AGENT_CLIP_HEIGHT", "720"))
+
+
+def log(msg: str):
+    print(f"[agent] {msg}", flush=True)
+
+
+def _skill(path: str) -> str:
+    """Best-effort director-skill text for LLM context (truncated). Intelligence lives in skills."""
+    for cand in (REPO / "skills" / f"{path}.md", REPO / f"{path}.md"):
+        try:
+            return cand.read_text()[:6000]
+        except OSError:
+            continue
+    return ""
+
+
+# --------------------------------------------------------------------------- #
+# asset helpers (real providers via the registry, or ffmpeg fixtures for tests)
+# --------------------------------------------------------------------------- #
+def _ffmpeg_clip(out: str, seconds: float, color: str = "darkblue"):
+    subprocess.run(["ffmpeg", "-y", "-f", "lavfi", "-i",
+                    f"color=c={color}:s={CLIP_W}x{CLIP_H}:d={max(1, int(seconds))}:r=24",
+                    "-c:v", "libx264", "-pix_fmt", "yuv420p", out], capture_output=True, check=True)
+
+
+def _ffmpeg_sine(out: str, seconds: float):
+    subprocess.run(["ffmpeg", "-y", "-f", "lavfi", "-i",
+                    f"sine=frequency=320:duration={max(1, int(seconds))}", "-ar", "44100", "-ac", "1", out],
+                   capture_output=True, check=True)
+
+
+def generate_clip(registry, prompt: str, out: str, seconds: float, idx: int) -> str:
+    if FAKE:
+        _ffmpeg_clip(out, seconds, ["darkblue", "darkgreen", "darkred", "purple", "teal"][idx % 5])
+        return out
+    sel = registry.get("video_selector")
+    if sel is None:
+        raise RuntimeError("no video_selector tool available")
+    res = sel.execute({"prompt": prompt, "aspect_ratio": "16:9", "duration": str(int(seconds)),
+                       "output_path": out})
+    if not res.success:
+        raise RuntimeError(f"video generation failed: {res.error}")
+    return res.data.get("output") or res.data.get("output_path") or out
+
+
+def generate_tts(registry, text: str, out: str) -> str | None:
+    if FAKE:
+        _ffmpeg_sine(out, max(2, len(text) // 15)); return out
+    avail = [t for t in registry.get_by_capability("tts") if getattr(t, "name", "") != "tts_selector"
+             and _tool_available(t)]
+    if not avail:
+        return None
+    res = registry.get("tts_selector").execute({"text": text, "output_path": out})
+    return (res.data.get("output") or out) if res.success else None
+
+
+def generate_music(registry, prompt: str, out: str, seconds: float) -> str | None:
+    if FAKE:
+        _ffmpeg_sine(out, seconds); return out
+    mg = registry.get("music_gen")
+    if mg is None or not _tool_available(mg):
+        return None
+    res = mg.execute({"prompt": prompt, "duration_seconds": int(seconds), "output_path": out})
+    return (res.data.get("output") or out) if res.success else None
+
+
+def _tool_available(tool) -> bool:
+    try:
+        info = tool.get_info()
+        return str(info.get("status", "")).lower() in ("available", "ready", "ok", "")
+    except Exception:  # noqa: BLE001
+        return True
+
+
+# --------------------------------------------------------------------------- #
+# LLM planning stages
+# --------------------------------------------------------------------------- #
+def plan_brief(llm: LLM, prompt: str) -> dict:
+    sys_p = ("You are OpenMontage's idea director. Output a `brief` JSON artifact. "
+             "Required keys: version='1.0', title, hook, key_points (array of 3-6 strings), "
+             "tone, style, target_platform, target_duration_seconds (integer). " + _skill("pipelines/explainer/idea-director"))
+    return llm.complete_json(sys_p, f"User brief: {prompt}")
+
+
+def plan_script(llm: LLM, brief: dict) -> dict:
+    dur = int(brief.get("target_duration_seconds", 20))
+    sys_p = ("You are OpenMontage's script director. Output a `script` JSON artifact. Required: "
+             "version='1.0', title, total_duration_seconds (integer), sections (array). Each section: "
+             "id, text, start_seconds, end_seconds (contiguous, covering 0..total). Keep it concise. "
+             + _skill("pipelines/explainer/script-director"))
+    return llm.complete_json(sys_p, f"Brief:\n{json.dumps(brief)}\nTotal duration ~{dur}s.")
+
+
+def plan_scenes(llm: LLM, script: dict) -> dict:
+    sys_p = ("You are OpenMontage's scene director. Output a `scene_plan` JSON artifact. Required: "
+             "version='1.0', scenes (array). Each scene: id, type (one of generated|text_card|"
+             "animation|diagram), description (a vivid visual prompt for a video generator), "
+             "start_seconds, end_seconds, script_section_id. One scene per script section, contiguous. "
+             + _skill("pipelines/explainer/scene-director"))
+    return llm.complete_json(sys_p, f"Script:\n{json.dumps(script)}")
+
+
+# --------------------------------------------------------------------------- #
+# orchestration
+# --------------------------------------------------------------------------- #
+def run(prompt: str, out: str, llm: LLM | None = None, project_dir: Path | None = None) -> str:
+    from lib.checkpoint import write_checkpoint
+    from schemas.artifacts import validate_artifact
+    from tools.tool_registry import registry
+
+    registry.discover()
+    llm = llm or LLM()
+    log(f"LLM provider={llm.provider} model={llm.model or '(default)'}  fake_assets={FAKE}")
+    pid = "agent_" + uuid.uuid4().hex[:8]
+    proj = project_dir or (REPO / "projects" / "_agent" / pid)
+    pdir = proj / "pipeline"
+    assets = proj / "assets"
+    assets.mkdir(parents=True, exist_ok=True)
+    # pipeline_type=None -> validate against ALL known stages (this MVP uses the canonical
+    # idea->script->scene_plan->assets->edit->compose path, not a specific manifest's stage set).
+    ck = lambda stage, art: write_checkpoint(pdir, pid, stage, "completed", artifacts=art,
+                                              pipeline_type=None)  # noqa: E731
+
+    # 1) brief -> 2) script -> 3) scene_plan (LLM, validated, checkpointed)
+    log("planning brief...")
+    brief = plan_brief(llm, prompt); brief.setdefault("version", "1.0")
+    validate_artifact("brief", brief); ck("idea", {"brief": brief})
+    log(f"brief: {brief.get('title')}")
+
+    log("planning script...")
+    script = plan_script(llm, brief); script.setdefault("version", "1.0")
+    validate_artifact("script", script); ck("script", {"script": script})
+
+    log("planning scenes...")
+    scene_plan = plan_scenes(llm, script); scene_plan.setdefault("version", "1.0")
+    scenes = scene_plan.get("scenes", [])[:MAX_SCENES]
+    scene_plan["scenes"] = scenes
+    validate_artifact("scene_plan", scene_plan); ck("scene_plan", {"scene_plan": scene_plan})
+    log(f"{len(scenes)} scenes")
+
+    # 4) assets: per-scene video (video_selector -> best provider) + narration; one music track
+    sect_by_id = {s["id"]: s for s in script.get("sections", [])}
+    asset_list, cuts = [], []
+    for i, sc in enumerate(scenes):
+        dur = max(2.0, float(sc.get("end_seconds", 0)) - float(sc.get("start_seconds", 0)) or 5.0)
+        vid = str(assets / f"scene_{i}.mp4")
+        log(f"scene {i}: generating clip ({dur:.0f}s) — {sc.get('description','')[:60]}")
+        vid = generate_clip(registry, sc.get("description", prompt), vid, dur, i)
+        asset_list.append({"id": f"v{i}", "type": "video", "path": vid, "source_tool": "video_selector",
+                           "scene_id": sc.get("id", f"sc{i}")})
+        cuts.append({"id": f"cut{i}", "source": vid, "in_seconds": 0.0, "out_seconds": dur, "speed": 1.0})
+        # narration from the linked script section
+        sec = sect_by_id.get(sc.get("script_section_id"))
+        if sec and sec.get("text"):
+            nar = generate_tts(registry, sec["text"], str(assets / f"nar_{i}.mp3"))
+            if nar:
+                asset_list.append({"id": f"n{i}", "type": "narration", "path": nar,
+                                   "source_tool": "tts_selector", "scene_id": sc.get("id", f"sc{i}")})
+
+    total = sum(c["out_seconds"] for c in cuts)
+    music = generate_music(registry, f"background music, {brief.get('tone','')}",
+                           str(assets / "music.mp3"), total)
+    if music:
+        asset_list.append({"id": "music", "type": "music", "path": music, "source_tool": "music_gen",
+                           "scene_id": scenes[0].get("id", "sc0") if scenes else "sc0"})
+    asset_manifest = {"version": "1.0", "assets": asset_list}
+    validate_artifact("asset_manifest", asset_manifest); ck("assets", {"asset_manifest": asset_manifest})
+
+    # 5) edit_decisions (deterministic: clips in order, ffmpeg runtime)
+    narrations = [a["path"] for a in asset_list if a["type"] == "narration"]
+    edit = {"version": "1.0", "cuts": cuts, "render_runtime": "ffmpeg"}
+    if music:
+        edit["music"] = {"asset_id": "music", "volume": 0.2, "ducking": True}
+    validate_artifact("edit_decisions", edit); ck("edit", {"edit_decisions": edit})
+
+    # 6) compose: mix narration+music, then concat clips with audio
+    from tools.video.video_compose import VideoCompose
+    audio_path = None
+    if narrations or music:
+        audio_path = str(assets / "mix.wav")
+        _build_audio(narrations, music, audio_path)
+    log("composing final video (ffmpeg)...")
+    res = VideoCompose().execute({
+        "operation": "compose",
+        "edit_decisions": {"cuts": [{"source": c["source"], "in_seconds": c["in_seconds"],
+                                     "out_seconds": c["out_seconds"], "speed": 1.0} for c in cuts]},
+        **({"audio_path": audio_path} if audio_path else {}),
+        "output_path": out, "codec": "libx264", "crf": 23, "preset": "fast"})
+    if not res.success or not Path(out).exists():
+        raise RuntimeError(f"compose failed: {res.error}")
+    render_report = {"version": "1.0", "outputs": [{"path": out, "format": "mp4", "codec": "h264",
+                     "resolution": f"{CLIP_W}x{CLIP_H}", "duration_seconds": round(total, 2)}]}
+    try:
+        validate_artifact("render_report", render_report); ck("compose", {"render_report": render_report})
+    except Exception as exc:  # noqa: BLE001
+        log(f"render_report checkpoint skipped: {exc}")
+    log(f"done -> {out}  ({brief.get('title')}, {len(scenes)} scenes, ~{total:.0f}s)")
+    return out
+
+
+def _build_audio(narrations: list[str], music: str | None, out: str):
+    """Concat narration tracks, then duck music under them via the audio_mixer tool (ffmpeg)."""
+    from tools.tool_registry import registry
+    tmp = Path(out).with_suffix(".nar.wav")
+    if narrations:
+        lst = Path(out).with_suffix(".list.txt")
+        lst.write_text("".join(f"file '{p}'\n" for p in narrations))
+        subprocess.run(["ffmpeg", "-y", "-f", "concat", "-safe", "0", "-i", str(lst), str(tmp)],
+                       capture_output=True, check=True)
+    mixer = registry.get("audio_mixer")
+    if music and narrations and mixer is not None:
+        r = mixer.execute({"operation": "duck",
+                           "tracks": [{"path": str(tmp), "role": "speech"}, {"path": music, "role": "music"}],
+                           "ducking": {"enabled": True, "music_volume_during_speech": 0.15},
+                           "output_path": out})
+        if r.success:
+            return
+    # fallback: narration-only (or music-only)
+    src = str(tmp) if narrations else music
+    subprocess.run(["ffmpeg", "-y", "-i", src, out], capture_output=True, check=True)
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--prompt", required=True)
+    ap.add_argument("--out", required=True)
+    args = ap.parse_args()
+    Path(args.out).parent.mkdir(parents=True, exist_ok=True)
+    run(args.prompt, args.out)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/tests/tools/test_agent_runtime.py b/tests/tools/test_agent_runtime.py
new file mode 100644
index 000000000..a832e486d
--- /dev/null
+++ b/tests/tools/test_agent_runtime.py
@@ -0,0 +1,78 @@
+"""Offline test for the autonomous agent runtime (no LLM key, no GPU, no provider keys).
+
+Uses a STUB LLM (emits valid brief/script/scene_plan artifacts) + AGENT_RUNTIME_FAKE_ASSETS=1
+(ffmpeg color/sine fixtures) so the full pipeline plumbing — LLM planning -> schema validation ->
+checkpoints -> per-scene assets -> audio mix -> ffmpeg compose -> final mp4 — is exercised
+deterministically. Proves the harness works end-to-end before any cloud LLM / provider is attached.
+
+Run:  pytest tests/tools/test_agent_runtime.py -q   (needs ffmpeg on PATH)
+"""
+
+from __future__ import annotations
+
+import json
+import shutil
+import subprocess
+from pathlib import Path
+
+import pytest
+
+REPO = Path(__file__).resolve().parents[2]
+
+pytest.importorskip("jsonschema")
+if shutil.which("ffmpeg") is None or shutil.which("ffprobe") is None:
+    pytest.skip("ffmpeg/ffprobe not available", allow_module_level=True)
+
+
+def _stub(system: str, user: str) -> str:
+    s = system[:120].lower()  # match the prompt PREFIX, before appended skill text
+    if "scene director" in s:
+        return json.dumps({"version": "1.0", "scenes": [
+            {"id": "sc1", "type": "generated", "description": "a red fox walking into a snowy forest, cinematic",
+             "start_seconds": 0, "end_seconds": 5, "script_section_id": "s1"},
+            {"id": "sc2", "type": "generated", "description": "fox pauses, soft winter light, slow motion",
+             "start_seconds": 5, "end_seconds": 10, "script_section_id": "s2"}]})
+    if "script director" in s:
+        return json.dumps({"version": "1.0", "title": "Snowy Fox", "total_duration_seconds": 10,
+                           "sections": [
+                               {"id": "s1", "text": "A red fox steps into a snowy forest.",
+                                "start_seconds": 0, "end_seconds": 5},
+                               {"id": "s2", "text": "It pauses as soft winter light filters through.",
+                                "start_seconds": 5, "end_seconds": 10}]})
+    if "idea director" in s:
+        return json.dumps({"version": "1.0", "title": "Snowy Fox", "hook": "A fox in the snow.",
+                           "key_points": ["winter", "forest", "cinematic"], "tone": "calm",
+                           "style": "cinematic", "target_platform": "youtube",
+                           "target_duration_seconds": 10})
+    return "{}"
+
+
+def test_agent_runtime_end_to_end(tmp_path, monkeypatch):
+    monkeypatch.setenv("AGENT_RUNTIME_FAKE_ASSETS", "1")
+    monkeypatch.setenv("AGENT_CLIP_WIDTH", "320")   # tiny for speed
+    monkeypatch.setenv("AGENT_CLIP_HEIGHT", "240")
+    import importlib
+    import services.agent_runtime.run as run_mod
+    importlib.reload(run_mod)
+    from services.agent_runtime.llm import LLM
+
+    out = tmp_path / "final.mp4"
+    run_mod.run("a red fox in a snowy forest", str(out), llm=LLM(stub=_stub),
+                project_dir=tmp_path / "proj")
+
+    assert out.exists() and out.stat().st_size > 0
+    probe = subprocess.run(["ffprobe", "-v", "error", "-show_entries",
+                            "stream=codec_name,width,height", "-of", "default=noprint_wrappers=1", str(out)],
+                           capture_output=True, text=True)
+    assert "codec_name=h264" in probe.stdout, probe.stdout + probe.stderr
+    # checkpoints written for the planning + asset + edit stages
+    cks = list((tmp_path / "proj" / "pipeline").rglob("checkpoint_*.json"))
+    names = {c.stem for c in cks}
+    assert {"checkpoint_idea", "checkpoint_script", "checkpoint_scene_plan",
+            "checkpoint_assets", "checkpoint_edit"} <= names, names
+
+
+def test_llm_json_extraction():
+    from services.agent_runtime.llm import LLM
+    llm = LLM(stub=lambda s, u: 'prefix ```json\n{"a": 1, "b": {"c": 2}}\n``` suffix')
+    assert llm.complete_json("sys", "user") == {"a": 1, "b": {"c": 2}}

From 0bb32f40255189b364ceab4108c9baf9730fd59a Mon Sep 17 00:00:00 2001
From: Cursor Agent <cursoragent@cursor.com>
Date: Tue, 23 Jun 2026 04:40:55 +0000
Subject: [PATCH 02/28] fix(agent-runtime): resolve ffmpeg via imageio-ffmpeg
 on PATH (Macs have no system ffmpeg)

_prepare_ffmpeg symlinks the bundled imageio-ffmpeg binary as 'ffmpeg' on PATH so video_compose/
audio_mixer subprocesses work without a brew install. Verified end-to-end on the head Mac (stub LLM
+ fake assets -> real mp4). Offline tests still pass.

Co-authored-by: FluffyAIcode <FluffyAIcode@users.noreply.github.com>
---
 services/agent_runtime/run.py | 28 ++++++++++++++++++++++++++++
 1 file changed, 28 insertions(+)

diff --git a/services/agent_runtime/run.py b/services/agent_runtime/run.py
index 668de16dd..b3aa2dcdb 100644
--- a/services/agent_runtime/run.py
+++ b/services/agent_runtime/run.py
@@ -48,6 +48,33 @@ def log(msg: str):
     print(f"[agent] {msg}", flush=True)
 
 
+def _prepare_ffmpeg():
+    """Ensure `ffmpeg`/`ffprobe` resolve on PATH for all tool subprocesses.
+
+    Macs in this deployment have no system ffmpeg — the cluster uses imageio-ffmpeg's bundled
+    binary. Symlink it as `ffmpeg` (and `ffprobe` if imageio-ffmpeg ships one) into a dir on PATH
+    so video_compose/audio_mixer (which call bare `ffmpeg`) work without a brew install."""
+    import shutil
+    if shutil.which("ffmpeg") and shutil.which("ffprobe"):
+        return
+    try:
+        import imageio_ffmpeg
+        exe = imageio_ffmpeg.get_ffmpeg_exe()
+    except Exception:  # noqa: BLE001
+        return
+    bind = Path(os.environ.get("TMPDIR", "/tmp")) / "agent_ffmpeg_bin"
+    bind.mkdir(parents=True, exist_ok=True)
+    link = bind / "ffmpeg"  # imageio-ffmpeg bundles ffmpeg only (not ffprobe)
+    if not link.exists():
+        try:
+            os.symlink(exe, link)
+        except OSError:
+            pass
+    os.environ["PATH"] = f"{bind}{os.pathsep}{os.environ.get('PATH', '')}"
+    os.environ.setdefault("IMAGEIO_FFMPEG_EXE", exe)
+    os.environ.setdefault("FFMPEG_BINARY", exe)
+
+
 def _skill(path: str) -> str:
     """Best-effort director-skill text for LLM context (truncated). Intelligence lives in skills."""
     for cand in (REPO / "skills" / f"{path}.md", REPO / f"{path}.md"):
@@ -152,6 +179,7 @@ def run(prompt: str, out: str, llm: LLM | None = None, project_dir: Path | None
     from schemas.artifacts import validate_artifact
     from tools.tool_registry import registry
 
+    _prepare_ffmpeg()
     registry.discover()
     llm = llm or LLM()
     log(f"LLM provider={llm.provider} model={llm.model or '(default)'}  fake_assets={FAKE}")

From 083ba8a51f352e1945ec50dd0efc7ccd79466c44 Mon Sep 17 00:00:00 2001
From: Cursor Agent <cursoragent@cursor.com>
Date: Tue, 23 Jun 2026 10:53:07 +0000
Subject: [PATCH 03/28] feat(worker): add HunyuanVideo backend (T2V + I2V) as a
 higher-quality provider option

diffusers HunyuanVideoPipeline / HunyuanVideoImageToVideoPipeline; 13B, native 720p ~5s clips.
--backend hunyuan, ops framework,i2v; HUNYUAN_OFFLOAD (default on) for VRAM. For an A/B vs WAN
to decide whether Hunyuan becomes the default quality layer.

Co-authored-by: FluffyAIcode <FluffyAIcode@users.noreply.github.com>
---
 services/distributed_wan/grpc_worker.py | 97 ++++++++++++++++++++++++-
 1 file changed, 96 insertions(+), 1 deletion(-)

diff --git a/services/distributed_wan/grpc_worker.py b/services/distributed_wan/grpc_worker.py
index b76f41797..64877c6d0 100644
--- a/services/distributed_wan/grpc_worker.py
+++ b/services/distributed_wan/grpc_worker.py
@@ -187,6 +187,99 @@ def refine(self, req, q):
             return _to_uint8(out.frames[0])
 
 
+# --------------------------------------------------------------------------- #
+# HunyuanVideo backend (diffusers) — a higher-quality open-source provider (ADR 0017).
+# T2V via HunyuanVideoPipeline, I2V via HunyuanVideoImageToVideoPipeline. 13B; native 720p,
+# ~5s clips. Heavy: CPU offload on by default (HUNYUAN_OFFLOAD=1). CUDA/vast only (no MLX).
+# --------------------------------------------------------------------------- #
+class HunyuanBackend:
+    MODEL = os.environ.get("HUNYUAN_MODEL", "hunyuanvideo-community/HunyuanVideo")
+
+    def __init__(self, ops):
+        self.ops = ops or ["framework"]
+        self._lock = threading.Lock()
+        self.pipe = None
+        self._i2v = None
+        self.offload = os.environ.get("HUNYUAN_OFFLOAD", "1").lower() in ("1", "true", "yes")
+
+    def _load_t2v(self):
+        if self.pipe is not None:
+            return
+        import torch
+        from diffusers import HunyuanVideoPipeline, HunyuanVideoTransformer3DModel
+        torch.set_grad_enabled(False)
+        tr = HunyuanVideoTransformer3DModel.from_pretrained(self.MODEL, subfolder="transformer",
+                                                            torch_dtype=torch.bfloat16)
+        p = HunyuanVideoPipeline.from_pretrained(self.MODEL, transformer=tr, torch_dtype=torch.float16)
+        p.vae.enable_tiling()
+        p.enable_model_cpu_offload() if self.offload else p.to("cuda")
+        self.pipe = p
+
+    def _load_i2v(self):
+        if self._i2v is not None:
+            return
+        import torch
+        from diffusers import HunyuanVideoImageToVideoPipeline
+        p = HunyuanVideoImageToVideoPipeline.from_pretrained(self.MODEL, torch_dtype=torch.float16)
+        p.vae.enable_tiling()
+        p.enable_model_cpu_offload() if self.offload else p.to("cuda")
+        self._i2v = p
+
+    def health(self):
+        import torch
+        return pb.HealthReply(device="cuda" if torch.cuda.is_available() else "cpu",
+                              backend="hunyuan-diffusers", model=self.MODEL, ops=self.ops, ready=False,
+                              note=torch.cuda.get_device_name(0) if torch.cuda.is_available() else "cpu",
+                              relative_speed=float(os.environ.get("HUNYUAN_RELATIVE_SPEED", "0.8")))
+
+    def _cb(self, q, total):
+        def cb(pipe, i, t, kw):
+            try:
+                q.put_nowait(("p", (i + 1) / max(total, 1)))
+            except queue.Full:
+                pass
+            return kw
+        return cb
+
+    @staticmethod
+    def _snap(req):
+        w = max(16, (req.width or 1280) // 16 * 16)
+        h = max(16, (req.height or 720) // 16 * 16)
+        nf = req.num_frames or 61
+        if nf % 4 != 1:                       # Hunyuan needs num_frames == 4k+1
+            nf = (nf // 4) * 4 + 1
+        return w, h, nf
+
+    def framework(self, req, q):
+        import torch
+        if getattr(req, "init_image", b"") and "i2v" in self.ops:
+            return self.i2v(req, q)
+        self._load_t2v()
+        w, h, nf = self._snap(req)
+        with self._lock:
+            g = torch.Generator("cpu").manual_seed(req.seed or 42)
+            out = self.pipe(prompt=req.prompt, height=h, width=w, num_frames=nf,
+                            num_inference_steps=req.steps or 30, generator=g,
+                            callback_on_step_end=self._cb(q, req.steps or 30))
+            return _to_uint8(out.frames[0])
+
+    def i2v(self, req, q):
+        import torch
+        from PIL import Image
+        self._load_i2v()
+        w, h, nf = self._snap(req)
+        img = Image.open(io.BytesIO(req.init_image)).convert("RGB").resize((w, h))
+        with self._lock:
+            g = torch.Generator("cpu").manual_seed(req.seed or 42)
+            out = self._i2v(image=img, prompt=req.prompt, height=h, width=w, num_frames=nf,
+                            num_inference_steps=req.steps or 30, generator=g,
+                            callback_on_step_end=self._cb(q, req.steps or 30))
+            return _to_uint8(out.frames[0])
+
+    def refine(self, req, q):
+        raise RuntimeError("hunyuan backend has no refine op; use framework/i2v")
+
+
 # --------------------------------------------------------------------------- #
 # MLX backend (wraps mlx-video) — Mac mini; OWNER-RUN (untested in CI; no Mac here)
 # --------------------------------------------------------------------------- #
@@ -425,7 +518,7 @@ def RefineTile(self, request, context):
 
 def main():
     ap = argparse.ArgumentParser()
-    ap.add_argument("--backend", choices=["cuda", "mlx", "test"], default="cuda")
+    ap.add_argument("--backend", choices=["cuda", "mlx", "test", "hunyuan"], default="cuda")
     ap.add_argument("--host", default="0.0.0.0")
     ap.add_argument("--port", type=int, default=50051)
     ap.add_argument("--preload", action="store_true")
@@ -441,6 +534,8 @@ def main():
             backend.load()
     elif args.backend == "test":
         backend = TestBackend()
+    elif args.backend == "hunyuan":
+        backend = HunyuanBackend([o.strip() for o in (args.ops or "framework,i2v").split(",") if o.strip()])
     else:
         backend = MlxBackend(args.mlx_model_dir, [o.strip() for o in args.mlx_ops.split(",") if o.strip()])
 

From 9a539c1ef4215583310465d31b18c7a74d732c56 Mon Sep 17 00:00:00 2001
From: Cursor Agent <cursoragent@cursor.com>
Date: Tue, 23 Jun 2026 11:21:56 +0000
Subject: [PATCH 04/28] =?UTF-8?q?docs:=20ADR=200017=20+=20loop-log=2042=20?=
 =?UTF-8?q?=E2=80=94=20A/B=20HunyuanVideo=20vs=20WAN;=20adopt=20Hunyuan=20?=
 =?UTF-8?q?as=20quality=20layer?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Real same-prompt 720p A/B on vast: WAN quality=high = unusable abstract blobs (bad 1.3B seed);
HunyuanVideo = coherent photorealistic cinematic clip. Verdict: Hunyuan = default quality provider.
Caveats: ~8min/2s@720p, VRAM-heavy, prompt-adherence/long-form/license noted.

Co-authored-by: FluffyAIcode <FluffyAIcode@users.noreply.github.com>
---
 docs/adr/0001-kakeya-integration-loop-log.md  | 17 +++++++
 .../adr/0017-hunyuanvideo-quality-provider.md | 48 +++++++++++++++++++
 2 files changed, 65 insertions(+)
 create mode 100644 docs/adr/0017-hunyuanvideo-quality-provider.md

diff --git a/docs/adr/0001-kakeya-integration-loop-log.md b/docs/adr/0001-kakeya-integration-loop-log.md
index 6c2631e78..9892ec50d 100644
--- a/docs/adr/0001-kakeya-integration-loop-log.md
+++ b/docs/adr/0001-kakeya-integration-loop-log.md
@@ -1148,6 +1148,23 @@ test on agent.kakeya.ai pending the key.
 
 ---
 
+## Iteration 42 — A/B: HunyuanVideo vs WAN (real frames) → adopt Hunyuan as quality layer (ADR 0017)
+
+Same prompt (border collie, snowy forest, cinematic), both 1280×720 ~2s on vast Blackwell.
+- **WAN** (`quality=high`: 1.3B T2V seed → I2V-14B): frame = chaotic abstract blobs, **no recognizable
+  dog/forest** — unusable. Root cause: garbage low-res 1.3B seed amplified by I2V.
+- **Hunyuan** (`hunyuanvideo-community/HunyuanVideo`, 45f/30steps, no offload): frame = **coherent,
+  photorealistic** dog in a snowy forest, cinematic DoF — production-looking.
+
+**Verdict (frame-level visual judgment): adopt HunyuanVideo as the default quality layer.** Night-and-
+day. Caveats: ~8 min / 2s clip @720p (~13–16 s/step), VRAM-heavy (evicted the 85 G WAN worker to avoid
+OOM), prompt adherence imperfect (rendered a Shiba-type dog), long-form still needs Hunyuan-I2V,
+license stricter (Tencent community). Added `HunyuanBackend` to `grpc_worker.py` (`--backend hunyuan`,
+ops framework/i2v). Next: register `hunyuan_video` as a provider for `video_selector`/agent; resolve
+WAN-14B vs Hunyuan VRAM (load-on-demand); wire Hunyuan-I2V for continuity.
+
+---
+
 ## Open follow-ups (next iterations)
 - **Phase 2b — native gRPC transport.** Add an optional `kakeya` Python SDK transport
   for the bounded-memory long-context path (W3), behind the same tool, once the proto
diff --git a/docs/adr/0017-hunyuanvideo-quality-provider.md b/docs/adr/0017-hunyuanvideo-quality-provider.md
new file mode 100644
index 000000000..0555a40a5
--- /dev/null
+++ b/docs/adr/0017-hunyuanvideo-quality-provider.md
@@ -0,0 +1,48 @@
+# ADR 0017 — HunyuanVideo as the default quality video provider (A/B vs WAN)
+
+**Status:** Accepted (decision: adopt Hunyuan as the quality layer). Productization pending.
+**Date:** 2026-06-23
+**Related:** ADR 0015/0016 (quality/duration, agent runtime), the "four open-source models as
+providers" design.
+
+## 1. Question
+
+Does replacing/augmenting WAN 2.1 with **HunyuanVideo** (13B) improve video quality enough to make it
+the default quality layer? Decided by a **real A/B** on the same prompt, judged on actual frames.
+
+## 2. Setup
+
+- vast RTX PRO 6000 Blackwell (97 G). Same prompt: *"a border collie dog running through a snowy
+  forest, cinematic, soft winter light."* Both 1280×720, ~2 s.
+- **WAN** = current `quality=high` path (1.3B T2V seed → I2V-14B @720p), via the public gateway.
+- **Hunyuan** = `hunyuanvideo-community/HunyuanVideo` (diffusers `HunyuanVideoPipeline`), 45 frames,
+  30 steps, no offload. New `HunyuanBackend` added to `grpc_worker.py` (`--backend hunyuan`, ops
+  framework/i2v).
+
+## 3. Result (frame-level visual judgment)
+
+- **WAN**: a chaotic, abstract blob field — **no recognizable dog or forest**, heavy SR artifacts.
+  Effectively unusable. (Root cause: the 1.3B low-res T2V *seed* is garbage; I2V amplifies it.)
+- **Hunyuan**: a **coherent, photorealistic** dog walking through a snowy forest, cinematic shallow
+  depth of field, realistic snow + blurred trunks. **Clearly production-looking** for a single clip.
+
+**Verdict: adopt HunyuanVideo as the default quality layer.** The gap is night-and-day.
+
+## 4. Honest caveats
+
+- **Speed/VRAM:** Hunyuan ~13–16 s/step × 30 + VAE decode ≈ **~8 min for a ~2 s 720p clip**, and it
+  needs the GPU largely to itself (~49 GB active; the resident 85 GB WAN worker had to be evicted to
+  avoid OOM). Slower + heavier than WAN.
+- **Prompt adherence:** it rendered a Shiba/Akita-type dog, not specifically a *border collie* — scene
+  quality is excellent but fine-grained subject control needs prompt work (Layer-3 skill).
+- **Long-form:** multi-shot >5 s still needs `HunyuanVideo-I2V` chunking (drift risk remains).
+- **License:** Tencent community license (region/MAU limits) — stricter than WAN's Apache-2.0.
+
+## 5. Decision / next
+
+1. **Hunyuan = default quality provider** (single-clip T2V hero path); WAN demoted to fast/draft +
+   fallback. Both stay (the four-model provider design).
+2. **Productize:** register `hunyuan_video` as a `video_generation` provider so the agent/
+   `video_selector` routes to it for quality; resolve the VRAM tradeoff (can't keep WAN-14B (85 G) +
+   Hunyuan resident — load the quality model on demand or dedicate the box to Hunyuan).
+3. Wire HunyuanVideo-I2V for long-form continuity; improve prompting for subject fidelity.

From 6f0990897a53381cee8d1d4f86b539738ce7dc0a Mon Sep 17 00:00:00 2001
From: Cursor Agent <cursoragent@cursor.com>
Date: Tue, 23 Jun 2026 13:08:44 +0000
Subject: [PATCH 05/28] agent runtime: add LLM prompt-director step (activate
 Layer-3 video skills)

Per ADR 0016, enrich each scene into a rich native-T2V prompt before generation,
using the provider's advertised agent_skills (ai-video-gen/ltx2) as guidance, and
persist directed prompts to assets/director_prompts.json. Extends offline test.

Co-authored-by: FluffyAIcode <FluffyAIcode@users.noreply.github.com>
---
 docs/adr/0001-kakeya-integration-loop-log.md | 26 ++++++
 docs/adr/0016-autonomous-agent-runtime.md    | 20 +++--
 services/agent_runtime/run.py                | 84 +++++++++++++++++++-
 tests/tools/test_agent_runtime.py            | 10 +++
 4 files changed, 132 insertions(+), 8 deletions(-)

diff --git a/docs/adr/0001-kakeya-integration-loop-log.md b/docs/adr/0001-kakeya-integration-loop-log.md
index 9892ec50d..15aced870 100644
--- a/docs/adr/0001-kakeya-integration-loop-log.md
+++ b/docs/adr/0001-kakeya-integration-loop-log.md
@@ -1165,6 +1165,32 @@ WAN-14B vs Hunyuan VRAM (load-on-demand); wire Hunyuan-I2V for continuity.
 
 ---
 
+## Iteration 43 — agent runtime: LLM "prompt director" step (ADR 0016, per user)
+
+The user clarified the "LLM prompt director" is **already in OpenMontage's architecture** (the
+Layer-3 video skills) and asked to keep executing the agent-runtime integration per ADR 0016 —
+i.e. *activate* that existing knowledge, not invent a new step. Implemented in
+`services/agent_runtime/run.py`:
+
+- **`_resolve_video_skill(registry)`** — reads the `agent_skills` advertised by the video provider
+  `video_selector` routes to (e.g. `ai-video-gen`/`ltx2`) from `.agents/skills` or `.claude/skills`,
+  and feeds that prompting guidance to the LLM. So per-scene prompts are written the way the chosen
+  model wants, sourced from the skills OpenMontage already ships.
+- **`direct_video_prompt(llm, scene, brief, skill)`** — rewrites each scene description into ONE
+  rich native-T2V prompt (subject+action+setting+lighting+camera+style, ~40–70 words, positive
+  phrasing). This is the cross-attention insight from ADR 0015: **enrich the text condition and
+  hand a strong prompt to native T2V** instead of refining a weak low-res seed via I2V. Falls back
+  to the raw scene on any LLM error. Needs **no video GPU** — runs as soon as the LLM is up.
+- Directed prompts are persisted to `assets/director_prompts.json` for transparency; the assets
+  loop now generates from the **directed** prompt (not the bare scene description).
+
+**Validation (offline, no keys/GPU):** `tests/tools/test_agent_runtime.py` extended — the stub LLM
+handles the "video prompt director" system prompt and the test asserts each scene received a
+directed prompt (`startswith "directed cinematic shot, "`, `!= raw`) via `director_prompts.json`.
+`2 passed`. Still blocked on a live head LLM endpoint (Kakeya/Gemma) for the real `mode=agent` run.
+
+---
+
 ## Open follow-ups (next iterations)
 - **Phase 2b — native gRPC transport.** Add an optional `kakeya` Python SDK transport
   for the bounded-memory long-context path (W3), behind the same tool, once the proto
diff --git a/docs/adr/0016-autonomous-agent-runtime.md b/docs/adr/0016-autonomous-agent-runtime.md
index df65f7bff..32e56d776 100644
--- a/docs/adr/0016-autonomous-agent-runtime.md
+++ b/docs/adr/0016-autonomous-agent-runtime.md
@@ -31,9 +31,18 @@ New module `services/agent_runtime/` exposed as `AGENT_RUNTIME_CMD`
   - LLM produces **brief → script → scene_plan** (each schema-validated via `schemas.artifacts`
     and checkpointed via `lib/checkpoint`, exactly like the governed pipeline; director-skill text
     is loaded as LLM context — "intelligence in the skills").
+  - **prompt director** (`direct_video_prompt`): before each clip, the LLM rewrites the scene
+    description into ONE rich, model-appropriate **native-T2V** prompt (subject+action+setting+
+    lighting+camera+style, ~40–70 words, positive phrasing). Guidance is the **Layer-3 skill the
+    video provider advertises** (`_resolve_video_skill` reads the provider's `agent_skills`, e.g.
+    `ai-video-gen`/`ltx2`, from `.agents/skills` or `.claude/skills`) — i.e. it activates the prompt-
+    director knowledge OpenMontage already ships rather than inventing a new one. This is the
+    cross-attention insight from ADR 0015: enrich the text condition and hand a strong prompt to T2V,
+    instead of refining a weak low-res seed via I2V. Directed prompts are persisted to
+    `assets/director_prompts.json` for transparency. Needs no video GPU — runs as soon as the LLM is up.
   - assets: per-scene video via **`video_selector`** (auto-routes to the BEST available provider —
-    premium Seedance/Kling/Veo if configured, else the local WAN draft cluster), narration via
-    **`tts_selector`**, one music track via **`music_gen`** (skipped when unconfigured).
+    premium Seedance/Kling/Veo if configured, else the local WAN draft cluster) using the **directed**
+    prompt, narration via **`tts_selector`**, one music track via **`music_gen`** (skipped when unconfigured).
   - deterministic **edit_decisions** → **`audio_mixer`** + **`video_compose`** (ffmpeg) → final mp4.
 - `AGENT_RUNTIME_FAKE_ASSETS=1` swaps providers for ffmpeg fixtures so the harness is CI-testable
   with no keys/GPU.
@@ -57,9 +66,10 @@ agent uses a cloud LLM (light CPU/network) — it does **not** steal GPU from vi
 ## 5. Validation
 
 `tests/tools/test_agent_runtime.py` (offline, no keys/GPU): stub LLM emits valid brief/script/
-scene_plan; `AGENT_RUNTIME_FAKE_ASSETS=1` → full plumbing (plan → validate → checkpoint → per-scene
-assets → audio mix → ffmpeg compose) → **valid h264 mp4** + checkpoints for idea/script/scene_plan/
-assets/edit. JSON-extraction unit test. 2 pass (23 with gateway+pipeline suites).
+scene_plan; `AGENT_RUNTIME_FAKE_ASSETS=1` → full plumbing (plan → validate → **prompt director** →
+checkpoint → per-scene assets → audio mix → ffmpeg compose) → **valid h264 mp4** + checkpoints for
+idea/script/scene_plan/assets/edit. The test asserts each scene received a **directed** prompt
+(enriched, `!= raw` description) via `director_prompts.json`. JSON-extraction unit test. 2 pass.
 
 ## 6. Next
 
diff --git a/services/agent_runtime/run.py b/services/agent_runtime/run.py
index b3aa2dcdb..e2d3dc6cf 100644
--- a/services/agent_runtime/run.py
+++ b/services/agent_runtime/run.py
@@ -85,6 +85,75 @@ def _skill(path: str) -> str:
     return ""
 
 
+def _skill_named(name: str) -> str:
+    """Read a named Layer-3 skill (SKILL.md) from the skills trees (truncated)."""
+    for base in (REPO / ".agents" / "skills", REPO / ".claude" / "skills"):
+        cand = base / name / "SKILL.md"
+        try:
+            return cand.read_text()
+        except OSError:
+            continue
+    return ""
+
+
+def _resolve_video_skill(registry) -> str:
+    """Layer-3 prompting guidance for the active video provider.
+
+    OpenMontage already ships the "prompt director" knowledge as Layer-3 skills (ai-video-gen,
+    ltx2, seedance-2-0, ...). The agent reads the skill(s) advertised by the video provider that
+    `video_selector` routes to and feeds them to the LLM so per-scene prompts are written the way
+    that model wants — instead of passing a bare scene description to the generator."""
+    names: list[str] = []
+    sel = registry.get("video_selector")
+    if sel is not None:
+        try:
+            info = sel.get_info()
+            # If the selector can name the provider it would route to, prefer that provider's skill.
+            chosen = info.get("selected_tool") or info.get("provider")
+            if chosen:
+                prov = registry.get(chosen)
+                if prov is not None:
+                    names = list(getattr(prov, "agent_skills", []) or [])
+            if not names:
+                names = list(info.get("agent_skills", []) or [])
+        except Exception:  # noqa: BLE001
+            pass
+    if not names:
+        names = ["ai-video-gen"]
+    txt = ""
+    for n in names[:2]:
+        s = _skill_named(n)
+        if s:
+            txt += f"\n# Provider guidance: {n}\n{s[:4500]}\n"
+    return txt[:8000]
+
+
+def direct_video_prompt(llm: LLM, scene_desc: str, brief: dict, skill_text: str) -> str:
+    """LLM "prompt director": rewrite a scene into ONE rich, model-appropriate T2V prompt.
+
+    Native T2V conditions generation via cross-attention over the text embeddings, so a longer,
+    concrete, positive prompt (subject+action+setting+lighting+camera+style) yields a far stronger
+    result than a bare scene description. Guidance comes from the provider's Layer-3 skill."""
+    if not scene_desc:
+        return scene_desc
+    sys_p = (
+        "You are OpenMontage's video prompt director. Rewrite the scene into ONE rich, concrete "
+        "text-to-video prompt for a native text-to-video model. Include subject, action, setting, "
+        "lighting, camera framing/movement, and style. Be vivid and specific, ~40-70 words, fully "
+        "positive phrasing (describe what to SHOW, never what to avoid), no contradictions. "
+        "Output ONLY the prompt text — no quotes, no JSON, no preamble." + skill_text
+    )
+    user = (f"Scene: {scene_desc}\nOverall tone: {brief.get('tone', '')}\n"
+            f"Style: {brief.get('style', '')}\nTitle: {brief.get('title', '')}")
+    try:
+        out = llm.complete(sys_p, user)
+    except Exception as exc:  # noqa: BLE001
+        log(f"prompt-director fallback (raw scene): {exc}")
+        return scene_desc
+    out = " ".join((out or "").strip().strip('"').strip("`").split())
+    return out[:600] or scene_desc
+
+
 # --------------------------------------------------------------------------- #
 # asset helpers (real providers via the registry, or ffmpeg fixtures for tests)
 # --------------------------------------------------------------------------- #
@@ -212,12 +281,17 @@ def run(prompt: str, out: str, llm: LLM | None = None, project_dir: Path | None
 
     # 4) assets: per-scene video (video_selector -> best provider) + narration; one music track
     sect_by_id = {s["id"]: s for s in script.get("sections", [])}
-    asset_list, cuts = [], []
+    vskill = _resolve_video_skill(registry)
+    log(f"prompt-director: loaded provider guidance ({len(vskill)} chars)")
+    asset_list, cuts, director_prompts = [], [], []
     for i, sc in enumerate(scenes):
         dur = max(2.0, float(sc.get("end_seconds", 0)) - float(sc.get("start_seconds", 0)) or 5.0)
         vid = str(assets / f"scene_{i}.mp4")
-        log(f"scene {i}: generating clip ({dur:.0f}s) — {sc.get('description','')[:60]}")
-        vid = generate_clip(registry, sc.get("description", prompt), vid, dur, i)
+        raw = sc.get("description", prompt)
+        vprompt = direct_video_prompt(llm, raw, brief, vskill)
+        director_prompts.append({"scene_id": sc.get("id", f"sc{i}"), "raw": raw, "prompt": vprompt})
+        log(f"scene {i}: directed prompt — {vprompt[:80]}")
+        vid = generate_clip(registry, vprompt, vid, dur, i)
         asset_list.append({"id": f"v{i}", "type": "video", "path": vid, "source_tool": "video_selector",
                            "scene_id": sc.get("id", f"sc{i}")})
         cuts.append({"id": f"cut{i}", "source": vid, "in_seconds": 0.0, "out_seconds": dur, "speed": 1.0})
@@ -237,6 +311,10 @@ def run(prompt: str, out: str, llm: LLM | None = None, project_dir: Path | None
                            "scene_id": scenes[0].get("id", "sc0") if scenes else "sc0"})
     asset_manifest = {"version": "1.0", "assets": asset_list}
     validate_artifact("asset_manifest", asset_manifest); ck("assets", {"asset_manifest": asset_manifest})
+    try:
+        (assets / "director_prompts.json").write_text(json.dumps(director_prompts, indent=2))
+    except OSError:
+        pass
 
     # 5) edit_decisions (deterministic: clips in order, ffmpeg runtime)
     narrations = [a["path"] for a in asset_list if a["type"] == "narration"]
diff --git a/tests/tools/test_agent_runtime.py b/tests/tools/test_agent_runtime.py
index a832e486d..f44ff019a 100644
--- a/tests/tools/test_agent_runtime.py
+++ b/tests/tools/test_agent_runtime.py
@@ -26,6 +26,10 @@
 
 def _stub(system: str, user: str) -> str:
     s = system[:120].lower()  # match the prompt PREFIX, before appended skill text
+    if "video prompt director" in s:
+        # echo the raw scene so the test can verify the DIRECTED prompt is what reaches the generator
+        scene = user.split("\n", 1)[0].removeprefix("Scene: ").strip()
+        return f"directed cinematic shot, {scene}, golden hour rim light, slow dolly-in, 35mm film"
     if "scene director" in s:
         return json.dumps({"version": "1.0", "scenes": [
             {"id": "sc1", "type": "generated", "description": "a red fox walking into a snowy forest, cinematic",
@@ -70,6 +74,12 @@ def test_agent_runtime_end_to_end(tmp_path, monkeypatch):
     names = {c.stem for c in cks}
     assert {"checkpoint_idea", "checkpoint_script", "checkpoint_scene_plan",
             "checkpoint_assets", "checkpoint_edit"} <= names, names
+    # prompt-director step ran: each scene got a DIRECTED prompt (enriched, != raw description)
+    dp = json.loads((tmp_path / "proj" / "assets" / "director_prompts.json").read_text())
+    assert len(dp) == 2, dp
+    for entry in dp:
+        assert entry["prompt"].startswith("directed cinematic shot, "), entry
+        assert entry["prompt"] != entry["raw"], entry
 
 
 def test_llm_json_extraction():

From d5aaf0f0822c05a8c3b532f9a59e1e88eb8bbf97 Mon Sep 17 00:00:00 2001
From: Cursor Agent <cursoragent@cursor.com>
Date: Tue, 23 Jun 2026 13:44:40 +0000
Subject: [PATCH 06/28] agent runtime: add --check-llm endpoint smoke test

One tiny LLM round-trip to verify the reasoner (head Kakeya/Gemma gRPC runtime via
OpenAI HTTP shim) before committing to a full pipeline run. Exit 0/1/2 = ok/error/
no-endpoint. Docs updated with the live mode=agent verification recipe.

Co-authored-by: FluffyAIcode <FluffyAIcode@users.noreply.github.com>
---
 docs/adr/0001-kakeya-integration-loop-log.md | 11 ++++++-
 docs/adr/0016-autonomous-agent-runtime.md    |  9 ++++--
 services/agent_runtime/run.py                | 32 ++++++++++++++++++--
 3 files changed, 47 insertions(+), 5 deletions(-)

diff --git a/docs/adr/0001-kakeya-integration-loop-log.md b/docs/adr/0001-kakeya-integration-loop-log.md
index 15aced870..cebff951a 100644
--- a/docs/adr/0001-kakeya-integration-loop-log.md
+++ b/docs/adr/0001-kakeya-integration-loop-log.md
@@ -1187,7 +1187,16 @@ i.e. *activate* that existing knowledge, not invent a new step. Implemented in
 **Validation (offline, no keys/GPU):** `tests/tools/test_agent_runtime.py` extended — the stub LLM
 handles the "video prompt director" system prompt and the test asserts each scene received a
 directed prompt (`startswith "directed cinematic shot, "`, `!= raw`) via `director_prompts.json`.
-`2 passed`. Still blocked on a live head LLM endpoint (Kakeya/Gemma) for the real `mode=agent` run.
+`2 passed`.
+
+**LLM endpoint smoke test (`--check-llm`).** The head now runs the Kakeya **Gemma 26B** runtime
+(gRPC :51051, kept alive via `caffeinate`) next to the gateway (:8088, `/healthz agent_runtime:true`).
+Added `python -m services.agent_runtime.run --check-llm`: one tiny LLM round-trip prints
+provider/model + latency, exits 0 (OK) / 1 (endpoint error) / 2 (no endpoint, provider=stub) — so the
+reasoner is verified in ~seconds before committing to a 20–30 min pipeline run. Cloud agent cannot
+submit `mode=agent` directly (gateway enforces `X-API-Key`); the live planning+director run is driven
+by the key holder. The agent runtime reaches the runtime via the OpenAI-compatible HTTP shim
+(`AGENT_LLM=kakeya` + `KAKEYA_ENDPOINT`); the gRPC `RuntimeService` is token-id level, not text I/O.
 
 ---
 
diff --git a/docs/adr/0016-autonomous-agent-runtime.md b/docs/adr/0016-autonomous-agent-runtime.md
index 32e56d776..2b7c9bd13 100644
--- a/docs/adr/0016-autonomous-agent-runtime.md
+++ b/docs/adr/0016-autonomous-agent-runtime.md
@@ -73,8 +73,13 @@ idea/script/scene_plan/assets/edit. The test asserts each scene received a **dir
 
 ## 6. Next
 
-- Deploy: set `AGENT_RUNTIME_CMD="<venv> -m services.agent_runtime.run"` + a cloud LLM key on the
-  head; live-test `mode=agent` via `agent.kakeya.ai`.
+- Deploy: set `AGENT_RUNTIME_CMD="<venv> -m services.agent_runtime.run"` + an LLM endpoint on the
+  head; live-test `mode=agent` via `agent.kakeya.ai`. **LLM endpoint smoke test:** run
+  `python -m services.agent_runtime.run --check-llm` first — one tiny round-trip prints
+  provider/model + latency and exits non-zero on failure, so the reasoner is verified before a full
+  (slow) pipeline run. With the head's Kakeya/Gemma runtime, point `AGENT_LLM=kakeya` +
+  `KAKEYA_ENDPOINT` at the OpenAI-compatible HTTP shim (the stable text surface; the gRPC
+  `RuntimeService` is token-id level).
 - Configure premium video provider for production-grade clips (the quality answer from the
   draft-vs-premium discussion).
 - Grow toward the governed pipeline (proposal/approval/reviewer, more director skills) and more
diff --git a/services/agent_runtime/run.py b/services/agent_runtime/run.py
index e2d3dc6cf..4fd2a0c01 100644
--- a/services/agent_runtime/run.py
+++ b/services/agent_runtime/run.py
@@ -370,11 +370,39 @@ def _build_audio(narrations: list[str], music: str | None, out: str):
     subprocess.run(["ffmpeg", "-y", "-i", src, out], capture_output=True, check=True)
 
 
+def check_llm() -> int:
+    """Fast LLM-endpoint smoke test: one tiny round-trip. Verify the reasoner is reachable
+    BEFORE committing to a full (slow) pipeline run. Returns a process exit code."""
+    import time
+    llm = LLM()
+    log(f"LLM provider={llm.provider} model={llm.model or '(default)'}")
+    if llm.provider == "stub":
+        log("LLM check: provider=stub — no real endpoint configured "
+            "(set AGENT_LLM + KAKEYA_ENDPOINT/ANTHROPIC_API_KEY/OPENAI_API_KEY).")
+        return 2
+    t0 = time.time()
+    try:
+        out = llm.complete("You are a health check. Reply with exactly: OK",
+                           "Reply with the single word OK.", max_tokens=16)
+    except Exception as exc:  # noqa: BLE001
+        log(f"LLM check FAILED ({type(exc).__name__}): {exc}")
+        return 1
+    dt = time.time() - t0
+    log(f"LLM check OK in {dt:.1f}s — reply: {(out or '').strip()[:80]!r}")
+    return 0
+
+
 def main():
     ap = argparse.ArgumentParser()
-    ap.add_argument("--prompt", required=True)
-    ap.add_argument("--out", required=True)
+    ap.add_argument("--prompt", help="brief/idea to drive the pipeline")
+    ap.add_argument("--out", help="output mp4 path")
+    ap.add_argument("--check-llm", action="store_true",
+                    help="only verify the LLM endpoint round-trips, then exit (no GPU/pipeline)")
     args = ap.parse_args()
+    if args.check_llm:
+        raise SystemExit(check_llm())
+    if not args.prompt or not args.out:
+        ap.error("--prompt and --out are required (unless --check-llm)")
     Path(args.out).parent.mkdir(parents=True, exist_ok=True)
     run(args.prompt, args.out)
 

From c46922fa365395480c445f3f5a6ca33371290468 Mon Sep 17 00:00:00 2001
From: Cursor Agent <cursoragent@cursor.com>
Date: Tue, 23 Jun 2026 14:07:16 +0000
Subject: [PATCH 07/28] agent runtime: add kakeya_grpc provider (direct Kakeya
 gRPC RuntimeService)

Fold the head's working transport into the repo: AGENT_LLM=kakeya_grpc talks token-id
level to the user-run Kakeya engine (Gemma 26B @ :51051) via its gRPC SDK. External
dependency contract (KAKEYA_REPO required; kakeya/transformers imported lazily) so grpc/
transformers never become core deps. Clear RuntimeError when KAKEYA_REPO unset; offline
guard test. Verified live by owner (--check-llm -> OK).

Co-authored-by: FluffyAIcode <FluffyAIcode@users.noreply.github.com>
---
 docs/adr/0001-kakeya-integration-loop-log.md | 15 ++++-
 docs/adr/0016-autonomous-agent-runtime.md    | 26 ++++++---
 services/agent_runtime/llm.py                | 59 +++++++++++++++++++-
 services/agent_runtime/run.py                |  4 +-
 tests/tools/test_agent_runtime.py            | 12 ++++
 5 files changed, 103 insertions(+), 13 deletions(-)

diff --git a/docs/adr/0001-kakeya-integration-loop-log.md b/docs/adr/0001-kakeya-integration-loop-log.md
index cebff951a..a9f247c62 100644
--- a/docs/adr/0001-kakeya-integration-loop-log.md
+++ b/docs/adr/0001-kakeya-integration-loop-log.md
@@ -1195,8 +1195,19 @@ Added `python -m services.agent_runtime.run --check-llm`: one tiny LLM round-tri
 provider/model + latency, exits 0 (OK) / 1 (endpoint error) / 2 (no endpoint, provider=stub) — so the
 reasoner is verified in ~seconds before committing to a 20–30 min pipeline run. Cloud agent cannot
 submit `mode=agent` directly (gateway enforces `X-API-Key`); the live planning+director run is driven
-by the key holder. The agent runtime reaches the runtime via the OpenAI-compatible HTTP shim
-(`AGENT_LLM=kakeya` + `KAKEYA_ENDPOINT`); the gRPC `RuntimeService` is token-id level, not text I/O.
+by the key holder.
+
+**Live transport = `kakeya_grpc` (direct gRPC, not the HTTP shim).** Folded the head's working
+provider into the repo: `AGENT_LLM=kakeya_grpc` calls the Kakeya engine's native gRPC `RuntimeService`
+at token-id level (the stable surface — the OpenAI HTTP shim is deprecated/single-session). It is an
+**external dependency contract** — `KAKEYA_REPO` must point at a Kakeya checkout (top-level `kakeya`
+pkg + `sdks/python`); `kakeya`/`transformers` import **lazily** only when selected, so grpc/transformers
+never enter OpenMontage's core deps (D5/D7). Flow: `apply_chat_template(enable_thinking=False)` →
+`Client.create_session(eos_token_ids)` → `append` → `generate(max_tokens)` → `decode`. Env:
+`KAKEYA_GRPC_ADDRESS=127.0.0.1:51051`, `KAKEYA_TOKENIZER_ID` (→ `KAKEYA_VERIFIER_ID` fallback). Head
+deps in `~/.venv-distwan`: grpcio 1.81, transformers 5.12, mlx-lm 0.31. **Owner confirmed live:**
+`--check-llm` → `provider=kakeya_grpc model=kakeya-local` → `OK`. Offline guard test asserts a missing
+`KAKEYA_REPO` yields a clear `RuntimeError` (no obscure ImportError, no core-dep creep). `3 passed`.
 
 ---
 
diff --git a/docs/adr/0016-autonomous-agent-runtime.md b/docs/adr/0016-autonomous-agent-runtime.md
index 2b7c9bd13..0f6bdd86c 100644
--- a/docs/adr/0016-autonomous-agent-runtime.md
+++ b/docs/adr/0016-autonomous-agent-runtime.md
@@ -24,9 +24,21 @@ runtime**.
 New module `services/agent_runtime/` exposed as `AGENT_RUNTIME_CMD`
 (`python -m services.agent_runtime.run --prompt … --out …`):
 
-- **`llm.py`** — dependency-free (urllib) cloud-LLM client: `AGENT_LLM = anthropic|openai|kakeya|stub`
-  (auto-detected from `ANTHROPIC_API_KEY`/`OPENAI_API_KEY`/`KAKEYA_ENDPOINT`). `complete_json` parses
-  + retries. Creative reasoning runs here; the host agent is no longer required at runtime.
+- **`llm.py`** — LLM client: `AGENT_LLM = anthropic|openai|kakeya|kakeya_grpc|stub` (auto-detected
+  from `ANTHROPIC_API_KEY`/`OPENAI_API_KEY`/`KAKEYA_GRPC_ADDRESS`/`KAKEYA_ENDPOINT`). `complete_json`
+  parses + retries. Creative reasoning runs here; the host agent is no longer required at runtime.
+  - The cloud + `kakeya` (HTTP shim) providers are **urllib-only** (no heavy deps).
+  - **`kakeya_grpc`** (the head's live path) talks **directly** to the user-run Kakeya engine over
+    its native gRPC `RuntimeService` (token-id level — the real stable surface; the HTTP shim is
+    deprecated/single-session). It is an **external dependency contract**: OpenMontage does **not**
+    vendor the Kakeya SDK/proto — point `KAKEYA_REPO` at a checkout (must expose a top-level `kakeya`
+    package + `sdks/python`) and the `kakeya`/`transformers` imports happen **lazily** only when this
+    provider is selected, so grpc/transformers never become core deps (ADR 0001 D5/D7). Env:
+    `KAKEYA_GRPC_ADDRESS` (e.g. `127.0.0.1:51051`), `KAKEYA_TOKENIZER_ID` (HF/MLX tokenizer dir;
+    falls back to `KAKEYA_VERIFIER_ID`). Flow: tokenizer `apply_chat_template(enable_thinking=False)`
+    → `Client.create_session(eos_token_ids=…)` → `append` → `generate(max_tokens)` → `decode`.
+    Host deps (head `~/.venv-distwan`): `grpcio`, `transformers`, `mlx`/`mlx-lm`. Missing
+    `KAKEYA_REPO` → clear actionable `RuntimeError`.
 - **`run.py`** — minimal unattended orchestrator that reuses the real machinery:
   - LLM produces **brief → script → scene_plan** (each schema-validated via `schemas.artifacts`
     and checkpointed via `lib/checkpoint`, exactly like the governed pipeline; director-skill text
@@ -76,10 +88,10 @@ idea/script/scene_plan/assets/edit. The test asserts each scene received a **dir
 - Deploy: set `AGENT_RUNTIME_CMD="<venv> -m services.agent_runtime.run"` + an LLM endpoint on the
   head; live-test `mode=agent` via `agent.kakeya.ai`. **LLM endpoint smoke test:** run
   `python -m services.agent_runtime.run --check-llm` first — one tiny round-trip prints
-  provider/model + latency and exits non-zero on failure, so the reasoner is verified before a full
-  (slow) pipeline run. With the head's Kakeya/Gemma runtime, point `AGENT_LLM=kakeya` +
-  `KAKEYA_ENDPOINT` at the OpenAI-compatible HTTP shim (the stable text surface; the gRPC
-  `RuntimeService` is token-id level).
+  provider/model + latency and exits 0/1/2 (ok/error/no-endpoint), so the reasoner is verified
+  before a full (slow) pipeline run. The head's live config uses **`AGENT_LLM=kakeya_grpc`** against
+  the Gemma 26B gRPC runtime (`KAKEYA_GRPC_ADDRESS=127.0.0.1:51051`, `KAKEYA_REPO`, `KAKEYA_TOKENIZER_ID`)
+  — `--check-llm` prints `OK`, confirming end-to-end reachability.
 - Configure premium video provider for production-grade clips (the quality answer from the
   draft-vs-premium discussion).
 - Grow toward the governed pipeline (proposal/approval/reviewer, more director skills) and more
diff --git a/services/agent_runtime/llm.py b/services/agent_runtime/llm.py
index cc86642bf..934c7f264 100644
--- a/services/agent_runtime/llm.py
+++ b/services/agent_runtime/llm.py
@@ -4,12 +4,18 @@
 UNATTENDED behind the gateway's mode=agent, we need our own cloud-LLM reasoner. This is a
 thin, dependency-free client (urllib only) over the providers' HTTP APIs, selected by env:
 
-  AGENT_LLM = anthropic | openai | kakeya | stub   (auto-detected from keys if unset)
-  ANTHROPIC_API_KEY / OPENAI_API_KEY / KAKEYA_ENDPOINT
+  AGENT_LLM = anthropic | openai | kakeya | kakeya_grpc | stub  (auto-detected from keys if unset)
+  ANTHROPIC_API_KEY / OPENAI_API_KEY / KAKEYA_ENDPOINT / KAKEYA_GRPC_ADDRESS
   AGENT_LLM_MODEL  (optional override)
 
 Creative reasoning (brief/script/scene_plan/edit) runs here; mechanical text can still use
 the in-repo kakeya_llm tool. `complete_json` returns a parsed dict, retrying once on bad JSON.
+
+The `kakeya_grpc` provider talks DIRECTLY to a user-run Kakeya inference server over its native
+gRPC RuntimeService (token-id level — the real stable surface; the HTTP shim is deprecated). It is
+an EXTERNAL dependency contract: OpenMontage does NOT vendor the Kakeya SDK/proto or add
+grpc/transformers as core deps — point `KAKEYA_REPO` at a checkout of the Kakeya engine and those
+imports happen lazily, only when this provider is selected. Other providers stay urllib-only.
 """
 
 from __future__ import annotations
@@ -24,6 +30,7 @@
     "anthropic": "claude-3-5-sonnet-latest",
     "openai": "gpt-4o-mini",
     "kakeya": "kakeya-local",
+    "kakeya_grpc": "kakeya-local",
 }
 
 
@@ -39,6 +46,8 @@ def __init__(self, provider: Optional[str] = None, model: Optional[str] = None,
                 self.provider = "anthropic"
             elif os.environ.get("OPENAI_API_KEY"):
                 self.provider = "openai"
+            elif os.environ.get("KAKEYA_GRPC_ADDRESS"):
+                self.provider = "kakeya_grpc"
             elif os.environ.get("KAKEYA_ENDPOINT"):
                 self.provider = "kakeya"
             else:
@@ -57,6 +66,8 @@ def complete(self, system: str, user: str, max_tokens: int = 3000) -> str:
             return self._openai(system, user, max_tokens)
         if self.provider == "kakeya":
             return self._openai(system, user, max_tokens, kakeya=True)
+        if self.provider == "kakeya_grpc":
+            return self._kakeya_grpc(system, user, max_tokens)
         raise RuntimeError(f"unknown AGENT_LLM provider: {self.provider}")
 
     def _post(self, url: str, headers: dict, body: dict, timeout: int = 180) -> dict:
@@ -87,6 +98,50 @@ def _openai(self, system: str, user: str, max_tokens: int, kakeya: bool = False)
                                        {"role": "user", "content": user}]})
         return out["choices"][0]["message"]["content"]
 
+    def _kakeya_grpc(self, system: str, user: str, max_tokens: int) -> str:
+        """Direct token-id-level generation over the Kakeya gRPC RuntimeService.
+
+        External dependency contract: the Kakeya SDK/proto are NOT vendored — set KAKEYA_REPO to a
+        checkout of the engine (it must contain a top-level `kakeya` package and `sdks/python`).
+        Needs `grpcio` + `transformers` (and the tokenizer at KAKEYA_TOKENIZER_ID) on the host;
+        these are imported lazily so they never become core OpenMontage deps."""
+        import sys
+        from pathlib import Path
+
+        repo = os.environ.get("KAKEYA_REPO")
+        if not repo:
+            raise RuntimeError(
+                "kakeya_grpc requires KAKEYA_REPO (path to a Kakeya inference-engine checkout) so "
+                "its gRPC SDK can be imported. Also set KAKEYA_GRPC_ADDRESS (e.g. 127.0.0.1:51051) "
+                "and KAKEYA_TOKENIZER_ID (HF/MLX tokenizer dir).")
+        kakeya_root = Path(repo).expanduser()
+        for p in (str(kakeya_root), str(kakeya_root / "sdks" / "python")):
+            if p not in sys.path:
+                sys.path.insert(0, p)
+        from kakeya import Client  # type: ignore  # noqa: E402
+        from transformers import AutoTokenizer  # noqa: E402
+
+        address = os.environ.get("KAKEYA_GRPC_ADDRESS", "127.0.0.1:51051")
+        tokenizer_id = os.environ.get(
+            "KAKEYA_TOKENIZER_ID", os.environ.get("KAKEYA_VERIFIER_ID", self.model))
+        tokenizer = AutoTokenizer.from_pretrained(tokenizer_id)
+        eos = tokenizer.eos_token_id
+        eos_ids = [int(eos)] if eos is not None else []
+        token_ids = tokenizer.apply_chat_template(
+            [{"role": "system", "content": system}, {"role": "user", "content": user}],
+            add_generation_prompt=True, tokenize=True, return_dict=False, enable_thinking=False)
+        with Client(address) as client:
+            session = client.create_session(eos_token_ids=eos_ids, client_label="openmontage-agent")
+            try:
+                session.append(token_ids)
+                generated = list(session.generate(max_tokens=max_tokens))
+            finally:
+                try:
+                    session.close()
+                except Exception:  # noqa: BLE001
+                    pass
+        return tokenizer.decode(generated, skip_special_tokens=True)
+
     # --- JSON helper --------------------------------------------------------
     @staticmethod
     def _extract_json(text: str) -> dict:
diff --git a/services/agent_runtime/run.py b/services/agent_runtime/run.py
index 4fd2a0c01..f460cc360 100644
--- a/services/agent_runtime/run.py
+++ b/services/agent_runtime/run.py
@@ -377,8 +377,8 @@ def check_llm() -> int:
     llm = LLM()
     log(f"LLM provider={llm.provider} model={llm.model or '(default)'}")
     if llm.provider == "stub":
-        log("LLM check: provider=stub — no real endpoint configured "
-            "(set AGENT_LLM + KAKEYA_ENDPOINT/ANTHROPIC_API_KEY/OPENAI_API_KEY).")
+        log("LLM check: provider=stub — no real endpoint configured (set AGENT_LLM + one of "
+            "KAKEYA_GRPC_ADDRESS / KAKEYA_ENDPOINT / ANTHROPIC_API_KEY / OPENAI_API_KEY).")
         return 2
     t0 = time.time()
     try:
diff --git a/tests/tools/test_agent_runtime.py b/tests/tools/test_agent_runtime.py
index f44ff019a..868a86d3c 100644
--- a/tests/tools/test_agent_runtime.py
+++ b/tests/tools/test_agent_runtime.py
@@ -86,3 +86,15 @@ def test_llm_json_extraction():
     from services.agent_runtime.llm import LLM
     llm = LLM(stub=lambda s, u: 'prefix ```json\n{"a": 1, "b": {"c": 2}}\n``` suffix')
     assert llm.complete_json("sys", "user") == {"a": 1, "b": {"c": 2}}
+
+
+def test_kakeya_grpc_requires_repo(monkeypatch):
+    """kakeya_grpc is an external-dependency contract: without KAKEYA_REPO it must fail with a
+    clear, actionable error (not an obscure ImportError) and never become a core dep."""
+    from services.agent_runtime.llm import LLM
+    monkeypatch.delenv("KAKEYA_REPO", raising=False)
+    monkeypatch.setenv("KAKEYA_GRPC_ADDRESS", "127.0.0.1:51051")
+    llm = LLM(provider="kakeya_grpc")
+    assert llm.provider == "kakeya_grpc"
+    with pytest.raises(RuntimeError, match="KAKEYA_REPO"):
+        llm.complete("sys", "user")

From 4656fa3bdc49ea937a5869260d3c950bdbd169d2 Mon Sep 17 00:00:00 2001
From: Cursor Agent <cursoragent@cursor.com>
Date: Wed, 24 Jun 2026 03:17:55 +0000
Subject: [PATCH 08/28] agent runtime: add --plan-only (validate Gemma +
 director without GPU/gateway/key)

The public gateway enforces X-API-Key which the owner cannot retrieve, blocking
mode=agent smoke tests. The runtime is a plain module; --plan-only runs LLM planning
+ prompt director, writes director_prompts.json, and exits before any render. Refactor
run() to compute directed prompts up front. Offline plan-only test.

Co-authored-by: FluffyAIcode <FluffyAIcode@users.noreply.github.com>
---
 docs/adr/0001-kakeya-integration-loop-log.md | 12 +++++-
 docs/adr/0016-autonomous-agent-runtime.md    |  6 +++
 services/agent_runtime/run.py                | 44 ++++++++++++++------
 tests/tools/test_agent_runtime.py            | 18 ++++++++
 4 files changed, 66 insertions(+), 14 deletions(-)

diff --git a/docs/adr/0001-kakeya-integration-loop-log.md b/docs/adr/0001-kakeya-integration-loop-log.md
index a9f247c62..cd6cc810a 100644
--- a/docs/adr/0001-kakeya-integration-loop-log.md
+++ b/docs/adr/0001-kakeya-integration-loop-log.md
@@ -1207,7 +1207,17 @@ never enter OpenMontage's core deps (D5/D7). Flow: `apply_chat_template(enable_t
 `KAKEYA_GRPC_ADDRESS=127.0.0.1:51051`, `KAKEYA_TOKENIZER_ID` (→ `KAKEYA_VERIFIER_ID` fallback). Head
 deps in `~/.venv-distwan`: grpcio 1.81, transformers 5.12, mlx-lm 0.31. **Owner confirmed live:**
 `--check-llm` → `provider=kakeya_grpc model=kakeya-local` → `OK`. Offline guard test asserts a missing
-`KAKEYA_REPO` yields a clear `RuntimeError` (no obscure ImportError, no core-dep creep). `3 passed`.
+`KAKEYA_REPO` yields a clear `RuntimeError` (no obscure ImportError, no core-dep creep).
+
+**`--plan-only` (unblock testing without the gateway key).** The public gateway enforces
+`X-API-Key` (`AGENT_GATEWAY_API_KEY`), which the owner couldn't retrieve — repeatedly blocking
+`mode=agent` smoke tests via `agent.kakeya.ai`. Key insight: the agent runtime is a plain module;
+the gateway is just a wrapper. Added `--plan-only` — runs LLM planning (brief→script→scene_plan) +
+the prompt director, writes `director_prompts.json`, and **exits before any video render**. So the
+owner validates Gemma + director on the head in minutes with **no GPU, no gateway, no key**:
+`python -m services.agent_runtime.run --prompt "…" --plan-only`. Refactored `run()` to compute all
+directed prompts up front (cleaner separation of director vs. generation). Offline test asserts
+plan-only emits 2 directed prompts and renders no clips. `4 passed`.
 
 ---
 
diff --git a/docs/adr/0016-autonomous-agent-runtime.md b/docs/adr/0016-autonomous-agent-runtime.md
index 0f6bdd86c..32e2c042e 100644
--- a/docs/adr/0016-autonomous-agent-runtime.md
+++ b/docs/adr/0016-autonomous-agent-runtime.md
@@ -92,6 +92,12 @@ idea/script/scene_plan/assets/edit. The test asserts each scene received a **dir
   before a full (slow) pipeline run. The head's live config uses **`AGENT_LLM=kakeya_grpc`** against
   the Gemma 26B gRPC runtime (`KAKEYA_GRPC_ADDRESS=127.0.0.1:51051`, `KAKEYA_REPO`, `KAKEYA_TOKENIZER_ID`)
   — `--check-llm` prints `OK`, confirming end-to-end reachability.
+- **`--plan-only`**: runs the LLM planning (brief→script→scene_plan) + the prompt director and writes
+  `director_prompts.json`, then exits **before any video render**. This is the fast,
+  **no-GPU / no-gateway / no-API-key** way to validate that the LLM + director actually work on the
+  head: `python -m services.agent_runtime.run --prompt "…" --plan-only`. The gateway's `X-API-Key`
+  (set as `AGENT_GATEWAY_API_KEY`) is only needed for the *public* path; the runtime itself is a plain
+  module and needs no key — run it directly to verify the feature.
 - Configure premium video provider for production-grade clips (the quality answer from the
   draft-vs-premium discussion).
 - Grow toward the governed pipeline (proposal/approval/reviewer, more director skills) and more
diff --git a/services/agent_runtime/run.py b/services/agent_runtime/run.py
index f460cc360..43d3c2d4d 100644
--- a/services/agent_runtime/run.py
+++ b/services/agent_runtime/run.py
@@ -243,7 +243,8 @@ def plan_scenes(llm: LLM, script: dict) -> dict:
 # --------------------------------------------------------------------------- #
 # orchestration
 # --------------------------------------------------------------------------- #
-def run(prompt: str, out: str, llm: LLM | None = None, project_dir: Path | None = None) -> str:
+def run(prompt: str, out: str, llm: LLM | None = None, project_dir: Path | None = None,
+        plan_only: bool = False) -> str:
     from lib.checkpoint import write_checkpoint
     from schemas.artifacts import validate_artifact
     from tools.tool_registry import registry
@@ -281,17 +282,32 @@ def run(prompt: str, out: str, llm: LLM | None = None, project_dir: Path | None
 
     # 4) assets: per-scene video (video_selector -> best provider) + narration; one music track
     sect_by_id = {s["id"]: s for s in script.get("sections", [])}
+
+    # 3b) prompt director (LLM, no GPU): rewrite each scene into a rich native-T2V prompt up front,
+    # so the directed prompts can be inspected (and validated via --plan-only) before any rendering.
     vskill = _resolve_video_skill(registry)
     log(f"prompt-director: loaded provider guidance ({len(vskill)} chars)")
-    asset_list, cuts, director_prompts = [], [], []
+    director_prompts = []
     for i, sc in enumerate(scenes):
-        dur = max(2.0, float(sc.get("end_seconds", 0)) - float(sc.get("start_seconds", 0)) or 5.0)
-        vid = str(assets / f"scene_{i}.mp4")
         raw = sc.get("description", prompt)
         vprompt = direct_video_prompt(llm, raw, brief, vskill)
         director_prompts.append({"scene_id": sc.get("id", f"sc{i}"), "raw": raw, "prompt": vprompt})
         log(f"scene {i}: directed prompt — {vprompt[:80]}")
-        vid = generate_clip(registry, vprompt, vid, dur, i)
+    dp_path = assets / "director_prompts.json"
+    try:
+        dp_path.write_text(json.dumps(director_prompts, indent=2))
+    except OSError:
+        pass
+    if plan_only:
+        log(f"plan-only: planning + director done, skipping asset generation/compose -> {dp_path}")
+        return str(dp_path)
+
+    # 4) assets: per-scene video (video_selector -> best provider) from the DIRECTED prompt
+    asset_list, cuts = [], []
+    for i, sc in enumerate(scenes):
+        dur = max(2.0, float(sc.get("end_seconds", 0)) - float(sc.get("start_seconds", 0)) or 5.0)
+        vid = str(assets / f"scene_{i}.mp4")
+        vid = generate_clip(registry, director_prompts[i]["prompt"], vid, dur, i)
         asset_list.append({"id": f"v{i}", "type": "video", "path": vid, "source_tool": "video_selector",
                            "scene_id": sc.get("id", f"sc{i}")})
         cuts.append({"id": f"cut{i}", "source": vid, "in_seconds": 0.0, "out_seconds": dur, "speed": 1.0})
@@ -311,10 +327,6 @@ def run(prompt: str, out: str, llm: LLM | None = None, project_dir: Path | None
                            "scene_id": scenes[0].get("id", "sc0") if scenes else "sc0"})
     asset_manifest = {"version": "1.0", "assets": asset_list}
     validate_artifact("asset_manifest", asset_manifest); ck("assets", {"asset_manifest": asset_manifest})
-    try:
-        (assets / "director_prompts.json").write_text(json.dumps(director_prompts, indent=2))
-    except OSError:
-        pass
 
     # 5) edit_decisions (deterministic: clips in order, ffmpeg runtime)
     narrations = [a["path"] for a in asset_list if a["type"] == "narration"]
@@ -398,13 +410,19 @@ def main():
     ap.add_argument("--out", help="output mp4 path")
     ap.add_argument("--check-llm", action="store_true",
                     help="only verify the LLM endpoint round-trips, then exit (no GPU/pipeline)")
+    ap.add_argument("--plan-only", action="store_true",
+                    help="run LLM planning + prompt-director only (no video render); writes "
+                         "director_prompts.json. Fast, no-GPU, no-gateway, no-key validation.")
     args = ap.parse_args()
     if args.check_llm:
         raise SystemExit(check_llm())
-    if not args.prompt or not args.out:
-        ap.error("--prompt and --out are required (unless --check-llm)")
-    Path(args.out).parent.mkdir(parents=True, exist_ok=True)
-    run(args.prompt, args.out)
+    if not args.prompt:
+        ap.error("--prompt is required (unless --check-llm)")
+    if not args.plan_only and not args.out:
+        ap.error("--out is required (unless --check-llm or --plan-only)")
+    if args.out:
+        Path(args.out).parent.mkdir(parents=True, exist_ok=True)
+    print(run(args.prompt, args.out or "", plan_only=args.plan_only))
 
 
 if __name__ == "__main__":
diff --git a/tests/tools/test_agent_runtime.py b/tests/tools/test_agent_runtime.py
index 868a86d3c..9f6fa71e1 100644
--- a/tests/tools/test_agent_runtime.py
+++ b/tests/tools/test_agent_runtime.py
@@ -82,6 +82,24 @@ def test_agent_runtime_end_to_end(tmp_path, monkeypatch):
         assert entry["prompt"] != entry["raw"], entry
 
 
+def test_agent_runtime_plan_only(tmp_path):
+    """--plan-only path: LLM planning + prompt-director, NO video render. Validates the directed
+    prompts (Gemma + director) can be produced/inspected without any GPU/gateway/key."""
+    import importlib
+    import services.agent_runtime.run as run_mod
+    importlib.reload(run_mod)
+    from services.agent_runtime.llm import LLM
+
+    res = run_mod.run("a red fox in a snowy forest", "", llm=LLM(stub=_stub),
+                      project_dir=tmp_path / "proj", plan_only=True)
+    dp_path = Path(res)
+    assert dp_path.name == "director_prompts.json" and dp_path.exists()
+    dp = json.loads(dp_path.read_text())
+    assert len(dp) == 2 and all(e["prompt"].startswith("directed cinematic shot, ") for e in dp)
+    # no clips/compose happened
+    assert not list((tmp_path / "proj" / "assets").glob("scene_*.mp4"))
+
+
 def test_llm_json_extraction():
     from services.agent_runtime.llm import LLM
     llm = LLM(stub=lambda s, u: 'prefix ```json\n{"a": 1, "b": {"c": 2}}\n``` suffix')

From a6f9d3bb1600445aee12b3c35d62df9f297920ad Mon Sep 17 00:00:00 2001
From: Cursor Agent <cursoragent@cursor.com>
Date: Wed, 24 Jun 2026 03:53:12 +0000
Subject: [PATCH 09/28] agent runtime: harden script stage against LLM schema
 drift

Gemma's 4-bit script stage emitted enhancement_cues.type='visual' (out of the
script schema enum), failing validation. Tighten plan_script prompt to the exact
enum + minimal sections, and add defensive _sanitize_script() that drops unknown
section keys and out-of-enum/malformed cues before validation. Folds owner's local
plan_script patch into the repo. Offline test reproduces the drift. Live --plan-only
on the head (real Gemma via kakeya_grpc) confirmed exit 0.

Co-authored-by: FluffyAIcode <FluffyAIcode@users.noreply.github.com>
---
 docs/adr/0001-kakeya-integration-loop-log.md | 18 +++++++++-
 services/agent_runtime/run.py                | 36 ++++++++++++++++++--
 tests/tools/test_agent_runtime.py            | 23 +++++++++++++
 3 files changed, 74 insertions(+), 3 deletions(-)

diff --git a/docs/adr/0001-kakeya-integration-loop-log.md b/docs/adr/0001-kakeya-integration-loop-log.md
index cd6cc810a..479585034 100644
--- a/docs/adr/0001-kakeya-integration-loop-log.md
+++ b/docs/adr/0001-kakeya-integration-loop-log.md
@@ -1217,7 +1217,23 @@ the prompt director, writes `director_prompts.json`, and **exits before any vide
 owner validates Gemma + director on the head in minutes with **no GPU, no gateway, no key**:
 `python -m services.agent_runtime.run --prompt "…" --plan-only`. Refactored `run()` to compute all
 directed prompts up front (cleaner separation of director vs. generation). Offline test asserts
-plan-only emits 2 directed prompts and renders no clips. `4 passed`.
+plan-only emits 2 directed prompts and renders no clips.
+
+**LIVE validation on the head (real Gemma, no GPU/gateway/key).** Owner ran `--plan-only` against the
+`kakeya_grpc` runtime → exit 0. Log: `provider=kakeya_grpc model=kakeya-local fake_assets=False` →
+`planning brief…` (`brief: The Dawn Hunter`) → `planning script/scenes…` → `3 scenes` →
+`prompt-director: loaded provider guidance (8000 chars)` → 3× `scene i: directed prompt — <rich
+cinematic prompt>` → `director_prompts.json`. **Gap B core loop (Gemma planning + Layer-3 prompt
+director) is proven live.**
+
+**Schema-drift fix (`enhancement_cues.type`).** Gemma's 4-bit script stage emitted an out-of-enum
+`enhancement_cues.type="visual"`, failing the strict `script.schema.json` (enum: overlay/broll/diagram/
+stat_card/code_snippet/animation; sections `additionalProperties:false`). Hardened `plan_script`: the
+prompt now lists the exact enum + "keep sections minimal, no unknown fields", AND a defensive
+`_sanitize_script()` strips unknown section keys and drops out-of-enum/malformed cues before
+validation (prompt steering alone is fragile for a 4-bit model). Offline test reproduces the
+`type="visual"` drift and asserts the sanitized script passes the schema. `5 passed`. (Folds the
+owner's local plan_script patch into the repo so head==repo, no divergence.)
 
 ---
 
diff --git a/services/agent_runtime/run.py b/services/agent_runtime/run.py
index 43d3c2d4d..92dab13eb 100644
--- a/services/agent_runtime/run.py
+++ b/services/agent_runtime/run.py
@@ -222,13 +222,45 @@ def plan_brief(llm: LLM, prompt: str) -> dict:
     return llm.complete_json(sys_p, f"User brief: {prompt}")
 
 
+# script.schema.json: sections are strict (additionalProperties:false) and enhancement_cues[].type
+# is an enum. A 4-bit local model (Gemma) drifts — e.g. it emitted type="visual" — so we both steer
+# the prompt AND defensively sanitize before validation so a stray field never fails the pipeline.
+_SCRIPT_SECTION_KEYS = {"id", "label", "text", "start_seconds", "end_seconds",
+                        "speaker_directions", "enhancement_cues", "pronunciation_guides", "source_ref"}
+_CUE_TYPES = {"overlay", "broll", "diagram", "stat_card", "code_snippet", "animation"}
+
+
+def _sanitize_script(script: dict) -> dict:
+    """Drop fields that would fail the strict script schema (LLM drift guard).
+
+    Keeps only known section keys and filters enhancement_cues to the allowed `type` enum (dropping
+    cues that are malformed or out-of-enum, e.g. the observed type="visual")."""
+    for sec in script.get("sections", []):
+        if not isinstance(sec, dict):
+            continue
+        for k in [k for k in sec if k not in _SCRIPT_SECTION_KEYS]:
+            sec.pop(k, None)
+        cues = sec.get("enhancement_cues")
+        if isinstance(cues, list):
+            kept = [c for c in cues if isinstance(c, dict) and c.get("type") in _CUE_TYPES
+                    and c.get("description")]
+            if kept:
+                sec["enhancement_cues"] = kept
+            else:
+                sec.pop("enhancement_cues", None)
+    return script
+
+
 def plan_script(llm: LLM, brief: dict) -> dict:
     dur = int(brief.get("target_duration_seconds", 20))
     sys_p = ("You are OpenMontage's script director. Output a `script` JSON artifact. Required: "
              "version='1.0', title, total_duration_seconds (integer), sections (array). Each section: "
-             "id, text, start_seconds, end_seconds (contiguous, covering 0..total). Keep it concise. "
+             "id, text, start_seconds, end_seconds (contiguous, covering 0..total). Keep sections "
+             "MINIMAL (only those fields). The schema is STRICT: do not add unknown fields. Omit "
+             "`enhancement_cues` unless needed; if present, each cue.type MUST be exactly one of "
+             "[overlay, broll, diagram, stat_card, code_snippet, animation] (NOT 'visual'). "
              + _skill("pipelines/explainer/script-director"))
-    return llm.complete_json(sys_p, f"Brief:\n{json.dumps(brief)}\nTotal duration ~{dur}s.")
+    return _sanitize_script(llm.complete_json(sys_p, f"Brief:\n{json.dumps(brief)}\nTotal duration ~{dur}s."))
 
 
 def plan_scenes(llm: LLM, script: dict) -> dict:
diff --git a/tests/tools/test_agent_runtime.py b/tests/tools/test_agent_runtime.py
index 9f6fa71e1..77a238229 100644
--- a/tests/tools/test_agent_runtime.py
+++ b/tests/tools/test_agent_runtime.py
@@ -106,6 +106,29 @@ def test_llm_json_extraction():
     assert llm.complete_json("sys", "user") == {"a": 1, "b": {"c": 2}}
 
 
+def test_sanitize_script_drops_invalid_cues():
+    """Guard the observed Gemma drift: an out-of-enum enhancement_cues.type (e.g. 'visual') and
+    unknown section fields must be stripped so the strict script schema still validates."""
+    import services.agent_runtime.run as run_mod
+    from schemas.artifacts import validate_artifact
+    script = {
+        "version": "1.0", "title": "T", "total_duration_seconds": 10,
+        "sections": [
+            {"id": "s1", "text": "a", "start_seconds": 0, "end_seconds": 5, "bogus_field": 1,
+             "enhancement_cues": [{"type": "visual", "description": "bad"},
+                                  {"type": "broll", "description": "ok"}]},
+            {"id": "s2", "text": "b", "start_seconds": 5, "end_seconds": 10,
+             "enhancement_cues": [{"type": "nope", "description": "x"}]},
+        ],
+    }
+    out = run_mod._sanitize_script(script)
+    s1, s2 = out["sections"]
+    assert "bogus_field" not in s1
+    assert [c["type"] for c in s1["enhancement_cues"]] == ["broll"]  # 'visual' dropped
+    assert "enhancement_cues" not in s2  # all-invalid -> removed
+    validate_artifact("script", out)  # now passes the strict schema
+
+
 def test_kakeya_grpc_requires_repo(monkeypatch):
     """kakeya_grpc is an external-dependency contract: without KAKEYA_REPO it must fail with a
     clear, actionable error (not an obscure ImportError) and never become a core dep."""

From b991653d00f632c9fd958564c259a0bee9aa6eb0 Mon Sep 17 00:00:00 2001
From: Cursor Agent <cursoragent@cursor.com>
Date: Wed, 24 Jun 2026 04:18:28 +0000
Subject: [PATCH 10/28] gateway UI: remember API key in browser (localStorage)

The X-API-Key requirement repeatedly blocked the web UI because the key had to be
re-typed each time. Prefill the key field from localStorage on load and persist it
on submit, so it is entered once per browser.

Co-authored-by: FluffyAIcode <FluffyAIcode@users.noreply.github.com>
---
 services/agent_gateway/server.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/services/agent_gateway/server.py b/services/agent_gateway/server.py
index 6745fa2ef..27bc08ed6 100644
--- a/services/agent_gateway/server.py
+++ b/services/agent_gateway/server.py
@@ -430,6 +430,8 @@ def index():
 <script>
 const $=id=>document.getElementById(id);
 let timer=null;
+// Remember the API key in this browser so it only has to be entered once.
+try{const sk=localStorage.getItem('om_api_key');if(sk)$('key').value=sk;}catch(e){}
 async function poll(jid){
   const r=await fetch('/v1/jobs/'+jid); const j=await r.json();
   $('stage').textContent=j.status+' · '+j.stage+' · '+Math.round((j.pct||0)*100)+'%';
@@ -444,7 +446,8 @@ def index():
   const prompt=$('p').value.trim(); if(prompt.length<3)return;
   $('go').disabled=true;$('status').style.display='block';$('vid').style.display='none';
   $('stage').textContent='submitting…';$('fill').style.width='0';$('log').textContent='';
-  const h={'Content-Type':'application/json'}; const k=$('key').value.trim(); if(k)h['X-API-Key']=k;
+  const h={'Content-Type':'application/json'}; const k=$('key').value.trim();
+  if(k){h['X-API-Key']=k; try{localStorage.setItem('om_api_key',k);}catch(e){}}
   const r=await fetch('/v1/videos',{method:'POST',headers:h,body:JSON.stringify({prompt})});
   if(!r.ok){$('go').disabled=false;$('stage').textContent='error: '+r.status+' '+(await r.text());return;}
   const {job_id}=await r.json();

From 1cabae6a34484883b294b73723d9bb46bdf9d545 Mon Sep 17 00:00:00 2001
From: Cursor Agent <cursoragent@cursor.com>
Date: Wed, 24 Jun 2026 04:32:32 +0000
Subject: [PATCH 11/28] gateway UI: add mode (video/agent) + quality + length
 controls

Web UI can now trigger the full agent pipeline (mode=agent) and pick quality/length
for direct video, without hand-crafting API calls. Quality/length auto-disable in
agent mode (the LLM plans its own scenes/pacing).

Co-authored-by: FluffyAIcode <FluffyAIcode@users.noreply.github.com>
---
 services/agent_gateway/server.py | 44 +++++++++++++++++++++++++++++---
 1 file changed, 41 insertions(+), 3 deletions(-)

diff --git a/services/agent_gateway/server.py b/services/agent_gateway/server.py
index 27bc08ed6..67d1bc278 100644
--- a/services/agent_gateway/server.py
+++ b/services/agent_gateway/server.py
@@ -412,15 +412,42 @@ def index():
   border:1px solid #1c2942;border-radius:10px;padding:12px;font-size:12.5px;color:#a9c2e6}
 video{width:100%;border-radius:12px;margin-top:12px;background:#000}
 input[type=password]{padding:10px;border-radius:8px;border:1px solid #2a3a59;background:#0e1830;color:#e7eefc}
+select{padding:9px 10px;border-radius:8px;border:1px solid #2a3a59;background:#0e1830;color:#e7eefc;font:inherit}
+label.fld{display:flex;flex-direction:column;gap:4px;font-size:12px;color:#8fa6c8}
+label.fld select:disabled{opacity:.45}
 </style></head><body><div class="wrap">
 <h1>OpenMontage — Agent Video</h1>
 <p class="sub">Describe a clip. The distributed cluster (MLX proposer + CUDA refiner) renders it.</p>
 <textarea id="p" placeholder="a red fox walking through a snowy forest, cinematic, soft winter light"></textarea>
+<div class="row">
+  <label class="fld">Mode
+    <select id="mode">
+      <option value="video">video · direct (fast)</option>
+      <option value="agent">agent · full pipeline</option>
+    </select>
+  </label>
+  <label class="fld">Quality
+    <select id="quality">
+      <option value="draft">draft · 640×384</option>
+      <option value="standard" selected>standard · 832×480</option>
+      <option value="high">high · 1280×720</option>
+    </select>
+  </label>
+  <label class="fld">Length
+    <select id="len">
+      <option value="">auto</option>
+      <option value="3">3s</option>
+      <option value="5">5s</option>
+      <option value="8">8s</option>
+      <option value="10">10s</option>
+    </select>
+  </label>
+</div>
 <div class="row">
   <button id="go">Generate</button>
-  <input id="key" type="password" placeholder="API key (if required)" style="flex:1;min-width:160px">
+  <input id="key" type="password" placeholder="API key (remembered in this browser)" style="flex:1;min-width:160px">
 </div>
-<p class="muted">Direct text→video via <code>/v1/videos</code>. First render can take a few minutes.</p>
+<p class="muted" id="hint">Direct text→video via <code>/v1/videos</code>. First render can take a few minutes.</p>
 <div class="card" id="status" style="display:none">
   <div id="stage" class="muted">queued…</div>
   <div class="bar"><i id="fill"></i></div>
@@ -432,6 +459,15 @@ def index():
 let timer=null;
 // Remember the API key in this browser so it only has to be entered once.
 try{const sk=localStorage.getItem('om_api_key');if(sk)$('key').value=sk;}catch(e){}
+function syncMode(){
+  const agent=$('mode').value==='agent';
+  // quality/length apply to direct video; agent mode plans its own scenes, pacing and length.
+  $('quality').disabled=agent; $('len').disabled=agent;
+  $('hint').innerHTML=agent
+    ? 'Agent mode: the LLM plans script → scenes → directed prompts, the cluster renders each, then composes one video. Quality/Length below apply to <em>direct video</em> mode only. This takes several minutes.'
+    : 'Direct text→video via <code>/v1/videos</code>. First render can take a few minutes.';
+}
+$('mode').onchange=syncMode; syncMode();
 async function poll(jid){
   const r=await fetch('/v1/jobs/'+jid); const j=await r.json();
   $('stage').textContent=j.status+' · '+j.stage+' · '+Math.round((j.pct||0)*100)+'%';
@@ -448,7 +484,9 @@ def index():
   $('stage').textContent='submitting…';$('fill').style.width='0';$('log').textContent='';
   const h={'Content-Type':'application/json'}; const k=$('key').value.trim();
   if(k){h['X-API-Key']=k; try{localStorage.setItem('om_api_key',k);}catch(e){}}
-  const r=await fetch('/v1/videos',{method:'POST',headers:h,body:JSON.stringify({prompt})});
+  const mode=$('mode').value; const body={prompt,mode};
+  if(mode==='video'){ body.quality=$('quality').value; const s=$('len').value; if(s)body.seconds=parseFloat(s); }
+  const r=await fetch('/v1/videos',{method:'POST',headers:h,body:JSON.stringify(body)});
   if(!r.ok){$('go').disabled=false;$('stage').textContent='error: '+r.status+' '+(await r.text());return;}
   const {job_id}=await r.json();
   timer=setInterval(()=>poll(job_id),2500);poll(job_id);

From e90d6f5eb9a6d944d455305baa5a5cfbcec837fb Mon Sep 17 00:00:00 2001
From: Cursor Agent <cursoragent@cursor.com>
Date: Wed, 24 Jun 2026 05:02:16 +0000
Subject: [PATCH 12/28] vast supervisor: preload CUDA worker + silence HF
 progress bars

First framework/refine RPC was hitting INTERNAL BrokenPipe because the cold ~30s
model download/load happened mid-RPC over the held-SSH pipe. Add --preload (load at
boot; server binds only after, so supervisor keeps it out of the pool until ready)
and HF_HUB_DISABLE_PROGRESS_BARS/TQDM_DISABLE to avoid flooding the SSH stderr pipe.

Co-authored-by: FluffyAIcode <FluffyAIcode@users.noreply.github.com>
---
 services/agent_gateway/deploy/vast_refiner_supervisor.sh | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/services/agent_gateway/deploy/vast_refiner_supervisor.sh b/services/agent_gateway/deploy/vast_refiner_supervisor.sh
index 3bf2af603..90e8628cc 100755
--- a/services/agent_gateway/deploy/vast_refiner_supervisor.sh
+++ b/services/agent_gateway/deploy/vast_refiner_supervisor.sh
@@ -87,9 +87,13 @@ once() {
     if ! pgrep -f "KAKEYA_HELD=$VAST_SSH_HOST" >/dev/null 2>&1; then
       pkill -f "KAKEYA_HELD=" 2>/dev/null   # clean stale holds (e.g. old host)
       OPS="framework,refine,i2v"; [ -z "${VAST_I2V_MODEL:-}" ] && OPS="framework,refine"
+      # --preload: load the T2V model at worker BOOT (server binds only after load), so the first
+      # framework/refine RPC is instant. Without it the first call triggers a ~30s cold download/load
+      # over the held-SSH pipe, which has caused INTERNAL BrokenPipe on the very first job. HF/tqdm
+      # progress bars are silenced (HF_HUB_DISABLE_PROGRESS_BARS) to avoid flooding the SSH stderr pipe.
       nohup ssh $SSH_OPTS -o ServerAliveInterval=20 -o ServerAliveCountMax=1000 \
           -p "$VAST_SSH_PORT" "$VAST_SSH_USER@$VAST_SSH_HOST" \
-          "cd /workspace/distwan && KAKEYA_HELD=$VAST_SSH_HOST HF_HOME=${VAST_HF_HOME:-/root/.hf_home} CUDA_I2V_MODEL='${VAST_I2V_MODEL:-}' CUDA_I2V_OFFLOAD='${VAST_I2V_OFFLOAD:-0}' exec ${VAST_VENV_PY:-/venv/main/bin/python} grpc_worker.py --backend cuda --host 0.0.0.0 --port $VAST_REMOTE_PORT --ops $OPS" \
+          "cd /workspace/distwan && KAKEYA_HELD=$VAST_SSH_HOST HF_HOME=${VAST_HF_HOME:-/root/.hf_home} HF_HUB_DISABLE_PROGRESS_BARS=1 TQDM_DISABLE=1 CUDA_I2V_MODEL='${VAST_I2V_MODEL:-}' CUDA_I2V_OFFLOAD='${VAST_I2V_OFFLOAD:-0}' exec ${VAST_VENV_PY:-/venv/main/bin/python} grpc_worker.py --backend cuda --host 0.0.0.0 --port $VAST_REMOTE_PORT --ops $OPS --preload" \
           >> "$HOME/.openmontage-logs/vast_held_worker.log" 2>&1 &
       log "spawned held worker ssh -> $VAST_SSH_HOST (ops=$OPS); model load may take minutes"
       sleep 3

From 8a693e14301079079f101a488e262660bb763e44 Mon Sep 17 00:00:00 2001
From: Cursor Agent <cursoragent@cursor.com>
Date: Wed, 24 Jun 2026 05:07:00 +0000
Subject: [PATCH 13/28] grpc_worker: log full traceback on op error (debug
 broken-pipe over tunnel)

Co-authored-by: FluffyAIcode <FluffyAIcode@users.noreply.github.com>
---
 services/distributed_wan/grpc_worker.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/services/distributed_wan/grpc_worker.py b/services/distributed_wan/grpc_worker.py
index 64877c6d0..5b2d12067 100644
--- a/services/distributed_wan/grpc_worker.py
+++ b/services/distributed_wan/grpc_worker.py
@@ -483,7 +483,10 @@ def run():
                 result["frames"] = op_fn(request, q)
                 result["t"] = time.time() - t
             except Exception as exc:  # noqa: BLE001
+                import traceback as _tb
                 result["err"] = f"{type(exc).__name__}: {exc}"
+                print(f"[grpc_worker] op error: {result['err']}", flush=True)
+                _tb.print_exc()
             finally:
                 q.put(("done", None))
 

From a284dfde5d972318ad3cc66d4beb5b4a882fc593 Mon Sep 17 00:00:00 2001
From: Cursor Agent <cursoragent@cursor.com>
Date: Wed, 24 Jun 2026 05:18:45 +0000
Subject: [PATCH 14/28] vast supervisor: redirect held worker output to a file
 on the box (fix BrokenPipe)

The worker's stdout/stderr was piped over the held SSH; diffusion progress bars on
stderr both raised BrokenPipeError when the channel hiccuped and the heavy traffic
triggered 'Connection reset by peer' that killed the worker mid-refine. Redirect
worker output to a local file so the held SSH carries ~no data and only keeps the
process alive.

Co-authored-by: FluffyAIcode <FluffyAIcode@users.noreply.github.com>
---
 .../deploy/vast_refiner_supervisor.sh             | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/services/agent_gateway/deploy/vast_refiner_supervisor.sh b/services/agent_gateway/deploy/vast_refiner_supervisor.sh
index 90e8628cc..5cd36293e 100755
--- a/services/agent_gateway/deploy/vast_refiner_supervisor.sh
+++ b/services/agent_gateway/deploy/vast_refiner_supervisor.sh
@@ -87,13 +87,18 @@ once() {
     if ! pgrep -f "KAKEYA_HELD=$VAST_SSH_HOST" >/dev/null 2>&1; then
       pkill -f "KAKEYA_HELD=" 2>/dev/null   # clean stale holds (e.g. old host)
       OPS="framework,refine,i2v"; [ -z "${VAST_I2V_MODEL:-}" ] && OPS="framework,refine"
-      # --preload: load the T2V model at worker BOOT (server binds only after load), so the first
-      # framework/refine RPC is instant. Without it the first call triggers a ~30s cold download/load
-      # over the held-SSH pipe, which has caused INTERNAL BrokenPipe on the very first job. HF/tqdm
-      # progress bars are silenced (HF_HUB_DISABLE_PROGRESS_BARS) to avoid flooding the SSH stderr pipe.
+      # Two hardening fixes for INTERNAL BrokenPipe on real jobs:
+      #  1) --preload: load the T2V model at BOOT (server binds only after), so the first RPC is
+      #     instant instead of triggering a ~30s cold load mid-RPC.
+      #  2) Redirect the worker's stdout/stderr to a FILE ON THE BOX (not through the held-SSH pipe).
+      #     The diffusion writes progress bars to stderr; sending that over the SSH channel both
+      #     (a) raised BrokenPipeError inside the op when the channel hiccuped, and (b) the heavy
+      #     traffic itself triggered "Connection reset by peer" that killed the worker mid-job.
+      #     With output on a local file the held SSH carries ~no data and just keeps the process alive.
+      #     HF/tqdm bars are also disabled as defense in depth. Read the worker log at $VAST_WORKER_LOG.
       nohup ssh $SSH_OPTS -o ServerAliveInterval=20 -o ServerAliveCountMax=1000 \
           -p "$VAST_SSH_PORT" "$VAST_SSH_USER@$VAST_SSH_HOST" \
-          "cd /workspace/distwan && KAKEYA_HELD=$VAST_SSH_HOST HF_HOME=${VAST_HF_HOME:-/root/.hf_home} HF_HUB_DISABLE_PROGRESS_BARS=1 TQDM_DISABLE=1 CUDA_I2V_MODEL='${VAST_I2V_MODEL:-}' CUDA_I2V_OFFLOAD='${VAST_I2V_OFFLOAD:-0}' exec ${VAST_VENV_PY:-/venv/main/bin/python} grpc_worker.py --backend cuda --host 0.0.0.0 --port $VAST_REMOTE_PORT --ops $OPS --preload" \
+          "cd /workspace/distwan && KAKEYA_HELD=$VAST_SSH_HOST HF_HOME=${VAST_HF_HOME:-/root/.hf_home} HF_HUB_DISABLE_PROGRESS_BARS=1 TQDM_DISABLE=1 CUDA_I2V_MODEL='${VAST_I2V_MODEL:-}' CUDA_I2V_OFFLOAD='${VAST_I2V_OFFLOAD:-0}' exec ${VAST_VENV_PY:-/venv/main/bin/python} grpc_worker.py --backend cuda --host 0.0.0.0 --port $VAST_REMOTE_PORT --ops $OPS --preload > ${VAST_WORKER_LOG:-/workspace/distwan/worker.log} 2>&1" \
           >> "$HOME/.openmontage-logs/vast_held_worker.log" 2>&1 &
       log "spawned held worker ssh -> $VAST_SSH_HOST (ops=$OPS); model load may take minutes"
       sleep 3

From 77cdf0910d865ebbda1f63d917ed53e01dfcf775 Mon Sep 17 00:00:00 2001
From: Cursor Agent <cursoragent@cursor.com>
Date: Wed, 24 Jun 2026 05:47:09 +0000
Subject: [PATCH 15/28] loop-log: iter 44 - Blackwell node + BrokenPipe root
 cause (held-SSH) -> tmux mode

Co-authored-by: FluffyAIcode <FluffyAIcode@users.noreply.github.com>
---
 docs/adr/0001-kakeya-integration-loop-log.md | 28 ++++++++++++++++++++
 1 file changed, 28 insertions(+)

diff --git a/docs/adr/0001-kakeya-integration-loop-log.md b/docs/adr/0001-kakeya-integration-loop-log.md
index 479585034..10aa5a115 100644
--- a/docs/adr/0001-kakeya-integration-loop-log.md
+++ b/docs/adr/0001-kakeya-integration-loop-log.md
@@ -1237,6 +1237,34 @@ owner's local plan_script patch into the repo so head==repo, no divergence.)
 
 ---
 
+## Iteration 44 — new Blackwell vast node + BrokenPipe root-cause (held-SSH) → tmux mode
+
+Wired a new vast box (RTX PRO 6000 **Blackwell**, 96 GB) as the 3rd node and chased an
+`INTERNAL: BrokenPipeError` that killed real jobs.
+
+- **Onboard:** authorized the head key (fixed a recurring *merged authorized_keys line* — the existing
+  key lacked a trailing newline, so an appended key concatenated and broke both), repointed
+  `~/.kakeya/vast.env`, verified Blackwell torch (2.12.1+cu130, sm_120 matmul OK), deps preinstalled.
+- **BrokenPipe root cause:** the **held-worker** mode runs the worker as a child of a persistent SSH
+  with stdout/stderr piped over that SSH. Diffusion writes progress bars to stderr; the fragile SSH
+  channel got `Connection reset by peer`, so the worker's write raised `BrokenPipeError` (surfaced as
+  the op error) AND the reset killed the worker mid-op (no Python traceback → killed by signal). The
+  heavy stderr traffic likely *caused* the resets. Verified the op itself is fine: framework AND v2v
+  refine run to completion **directly on the box** (`REFINE_OK`, ~6 s).
+- **Fix:** this box's image (unlike the earlier one) **keeps tmux/processes alive after SSH logout**
+  (tested). So switched it to non-held **tmux** mode (`VAST_HOLD_WORKER=0`): the worker is detached
+  from SSH, output goes to a tmux pane/file, and SSH resets no longer kill it. Verified: the worker
+  now survives and the 720p refine runs to 14/14 on the worker (previously it was killed → BrokenPipe).
+  Draft `mode=video` jobs complete end-to-end over the public gateway. Held-mode path was also
+  hardened (--preload, worker output → file, HF/tqdm bars off) for boxes that need it.
+- **Remaining (separate) issue:** for `quality=high` (full-frame 720p single-refine) the worker
+  computes the result but the gRPC progress/result **stream wedges over the SSH `-L` tunnel after
+  ~8 messages** — a transport flow-control deadlock, NOT the original crash. Needs a tunnel/transport
+  fix (gRPC window/keepalive tuning on the orchestrator channel, chunked result, or a sturdier
+  forward). Draft works; standard (tiled) likely avoids the single large stream.
+
+---
+
 ## Open follow-ups (next iterations)
 - **Phase 2b — native gRPC transport.** Add an optional `kakeya` Python SDK transport
   for the bounded-memory long-context path (W3), behind the same tool, once the proto

From 7b48bf15b3dedad64d49545c8d1d761ddc1c4f4f Mon Sep 17 00:00:00 2001
From: Cursor Agent <cursoragent@cursor.com>
Date: Wed, 24 Jun 2026 05:52:02 +0000
Subject: [PATCH 16/28] grpc_worker: instrument final-message send (debug 720p
 tunnel wedge)

Co-authored-by: FluffyAIcode <FluffyAIcode@users.noreply.github.com>
---
 services/distributed_wan/grpc_worker.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/services/distributed_wan/grpc_worker.py b/services/distributed_wan/grpc_worker.py
index 5b2d12067..ba3197630 100644
--- a/services/distributed_wan/grpc_worker.py
+++ b/services/distributed_wan/grpc_worker.py
@@ -509,8 +509,12 @@ def run():
         if "err" in result:
             context.abort(grpc.StatusCode.INTERNAL, result["err"])
             return
-        yield pb.Progress(pct=1.0, stage="done", done=True,
-                          mp4=frames_to_mp4(result["frames"]), gen_seconds=float(result["t"]))
+        nfr = len(result["frames"]) if result.get("frames") is not None else 0
+        print(f"[grpc_worker] denoise done ({nfr} frames); encoding mp4...", flush=True)
+        _mp4 = frames_to_mp4(result["frames"])
+        print(f"[grpc_worker] encoded mp4 {len(_mp4)} bytes; sending final message...", flush=True)
+        yield pb.Progress(pct=1.0, stage="done", done=True, mp4=_mp4, gen_seconds=float(result["t"]))
+        print("[grpc_worker] final message sent OK", flush=True)
 
     def GenerateFramework(self, request, context):
         yield from self._stream(self.backend.framework, request, context)

From 4b679a9f97e850982639add269df61061ef55741 Mon Sep 17 00:00:00 2001
From: Cursor Agent <cursoragent@cursor.com>
Date: Wed, 24 Jun 2026 06:05:15 +0000
Subject: [PATCH 17/28] grpc_worker: enable VAE tiling+slicing on CUDA backend
 (fix 720p refine hang)

The high-quality path upscales the proposer to 1280x720 before refine, so the Wan VAE
must encode+decode a full-frame 720p clip in fp32. Untiled this allocates a huge tensor
and HANGS after the denoise loop (worker stuck in VAE decode, GPU 0%, final message never
sent -> orchestrator wedged at 58%). enable_tiling()+enable_slicing() processes spatial
tiles / temporal slices, keeping memory bounded. Applied to T2V/v2v and I2V pipelines.

Co-authored-by: FluffyAIcode <FluffyAIcode@users.noreply.github.com>
---
 services/distributed_wan/grpc_worker.py | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/services/distributed_wan/grpc_worker.py b/services/distributed_wan/grpc_worker.py
index ba3197630..961728b3a 100644
--- a/services/distributed_wan/grpc_worker.py
+++ b/services/distributed_wan/grpc_worker.py
@@ -101,6 +101,15 @@ def load(self):
         self.pipe = WanPipeline.from_pretrained(self.MODEL, vae=vae, torch_dtype=torch.bfloat16)
         self.pipe.scheduler = UniPCMultistepScheduler.from_config(self.pipe.scheduler.config, flow_shift=3.0)
         self.pipe.to("cuda")
+        # VAE tiling/slicing: REQUIRED for the 720p refine path. The Wan VAE encode+decode of a
+        # full-frame 1280x720 clip in fp32 otherwise allocates a huge contiguous tensor and HANGS
+        # (observed: refine denoise completes, then the worker is stuck in VAE decode at GPU 0%).
+        # Tiling processes the frame in spatial tiles; slicing splits the temporal batch.
+        for _vae in (vae,):
+            try:
+                _vae.enable_tiling(); _vae.enable_slicing()
+            except Exception:  # noqa: BLE001
+                pass
         self.pipe.load_lora_weights(self.LORA_REPO, weight_name=self.LORA_FILE, adapter_name="causvid")
         self.v2v = WanVideoToVideoPipeline(**{k: self.pipe.components[k] for k in
                                               ("tokenizer", "text_encoder", "transformer", "vae", "scheduler")})
@@ -120,6 +129,10 @@ def _load_i2v(self):
         pipe = WanImageToVideoPipeline.from_pretrained(self.i2v_model, vae=vae, image_encoder=image_encoder,
                                                        torch_dtype=torch.bfloat16)
         pipe.scheduler = UniPCMultistepScheduler.from_config(pipe.scheduler.config, flow_shift=5.0)
+        try:  # tiling/slicing keeps 720P I2V VAE encode/decode within memory (see load())
+            vae.enable_tiling(); vae.enable_slicing()
+        except Exception:  # noqa: BLE001
+            pass
         if os.environ.get("CUDA_I2V_OFFLOAD", "0").lower() in ("1", "true", "yes"):
             pipe.enable_model_cpu_offload()
         else:

From d0a417d74ae69001f9244a6528f8c8f70809a29d Mon Sep 17 00:00:00 2001
From: Cursor Agent <cursoragent@cursor.com>
Date: Wed, 24 Jun 2026 06:21:17 +0000
Subject: [PATCH 18/28] orchestrator: cap single-refine resolution
 (REFINE_MAX_DIM=512) then upscale

Full-frame v2v at 1280x720 hangs in the Wan VAE (denoise completes, worker stuck at GPU
0%, never returns -> job wedges at 58%). Refine at a capped longest-side (default 512,
proven stable ~6s) and let the existing post-refine LANCZOS upscale reach the 720p target.
Env REFINE_MAX_DIM tunable; 0 = original full-res behavior.

Co-authored-by: FluffyAIcode <FluffyAIcode@users.noreply.github.com>
---
 services/distributed_wan/grpc_orchestrator.py | 20 ++++++++++++++-----
 1 file changed, 15 insertions(+), 5 deletions(-)

diff --git a/services/distributed_wan/grpc_orchestrator.py b/services/distributed_wan/grpc_orchestrator.py
index 9ba6cd678..c9aa4e4d1 100644
--- a/services/distributed_wan/grpc_orchestrator.py
+++ b/services/distributed_wan/grpc_orchestrator.py
@@ -346,14 +346,24 @@ def _to_out(frames):  # ensure every chunk is the same display resolution for cl
         F = framework.shape[0]
         t_idx = (np.round(np.linspace(0, F - 1, args.frames)).astype(int)
                  if F != args.frames else np.arange(args.frames))
-        # Upscale the proposer to the TARGET resolution before refine. The CUDA V2V pipeline keeps
-        # its INPUT resolution, so to get high-res generative detail the input must already be at
-        # out_w x out_h (MLX SR also honors this). This is what makes 'high'/single truly hi-res.
+        # Refine at a CAPPED resolution, then SR-upscale to the display target below. Full-frame
+        # v2v at 1280x720 makes the Wan VAE encode/decode allocate a huge fp32 tensor and HANG
+        # (denoise completes, then the worker is stuck at GPU 0% and never returns). Refining at
+        # <=REFINE_MAX_DIM (longest side; 512 is proven stable/fast ~6s) then LANCZOS-upscaling is
+        # reliable and still applies generative detail. Raise REFINE_MAX_DIM on stacks that handle
+        # higher; 0 disables the cap (original full-res behavior).
+        rw, rh = args.out_width, args.out_height
+        cap = int(os.environ.get("REFINE_MAX_DIM", "512"))
+        if cap and max(rw, rh) > cap:
+            s = cap / float(max(rw, rh))
+            rw = max(16, int(round(rw * s)) // 16 * 16)
+            rh = max(16, int(round(rh * s)) // 16 * 16)
         proposer = np.stack([np.asarray(Image.fromarray(framework[t_idx[i]]).resize(
-            (args.out_width, args.out_height), Image.BICUBIC)) for i in range(args.frames)])
+            (rw, rh), Image.BICUBIC)) for i in range(args.frames)])
+        print(f"[orch] refine @ {rw}x{rh} (cap={cap}) -> upscale to {args.out_width}x{args.out_height}", flush=True)
         mp4b, rg = _consume(refiner.stub.RefineTile(pb.RefineRequest(
             prompt=args.prompt, mp4=_frames_to_mp4_bytes(proposer),
-            width=args.out_width, height=args.out_height, num_frames=args.frames,
+            width=rw, height=rh, num_frames=args.frames,
             steps=args.refine_steps, strength=args.strength, seed=args.seed + 10)), "refine")
         final = _mp4_to_frames(mp4b)
         # Generative refiners (WAN 1.3B V2V) output at the model's NATIVE resolution (~832x480)

From 4d5b6671ddd9283fc12e35bffa52ce73748d75d3 Mon Sep 17 00:00:00 2001
From: Cursor Agent <cursoragent@cursor.com>
Date: Wed, 24 Jun 2026 06:36:46 +0000
Subject: [PATCH 19/28] high quality: native T2V @720p DIRECT on CUDA (v2v
 refine hangs in worker thread)

The distributed v2v refine op wedges in the worker's gRPC thread after denoise (VAE
decode never returns; confirmed framework/native-T2V @720p completes in-thread and the
1MB result traverses the tunnel fine). Repoint 'high' to native 1280x720 generation on
the strongest worker (Blackwell CUDA), no refine -- reliable + aligns with ADR 0015's
'native T2V beats low-res seed + refine'. Verified end-to-end: public download = h264
1280x720. (Also keeps REFINE_MAX_DIM cap as defense for any single-refine use.)

Co-authored-by: FluffyAIcode <FluffyAIcode@users.noreply.github.com>
---
 services/agent_gateway/server.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/services/agent_gateway/server.py b/services/agent_gateway/server.py
index 67d1bc278..af56525ab 100644
--- a/services/agent_gateway/server.py
+++ b/services/agent_gateway/server.py
@@ -160,9 +160,14 @@ def _auth(x_api_key: Optional[str] = Header(default=None)):
     "standard": {"fw_width": 480, "fw_height": 256, "fw_frames": 13, "proposer_steps": 6,
                  "refine_steps": 16, "out_width": 832, "out_height": 480, "frames": 25,
                  "fps": 12, "interpolate": 1, "interp_method": "linear", "refine_mode": ""},
-    "high": {"fw_width": 512, "fw_height": 288, "fw_frames": 17, "proposer_steps": 8,
+    # 'high' = NATIVE T2V generation at 1280x720 directly on the strongest framework worker (the
+    # CUDA box), no v2v refine. The distributed v2v refine op hangs in the worker's gRPC thread
+    # after denoise (VAE decode never returns; framework/native-T2V at 720p works fine in-thread),
+    # so the reliable hi-res path is direct native generation — which also matches the "native T2V
+    # beats low-res-seed + I2V/refine" finding (ADR 0015). 25 frames (4n+1) + mci interpolation.
+    "high": {"fw_width": 1280, "fw_height": 720, "fw_frames": 25, "proposer_steps": 8,
              "refine_steps": 24, "out_width": 1280, "out_height": 720, "frames": 25,
-             "fps": 24, "interpolate": 2, "interp_method": "mci", "refine_mode": "single"},
+             "fps": 24, "interpolate": 2, "interp_method": "mci", "refine_mode": "direct"},
 }
 
 

From 826a4993e82d8c2a88010f7118040fa549da8e90 Mon Sep 17 00:00:00 2001
From: Cursor Agent <cursoragent@cursor.com>
Date: Wed, 24 Jun 2026 06:40:08 +0000
Subject: [PATCH 20/28] standard quality: native T2V @832x480 DIRECT (tiled v2v
 also hangs in worker thread)

All three tiers now use the reliable direct native-T2V path (draft 384x224 / standard
832x480 / high 1280x720), differing by resolution+frames. The v2v refine op (single AND
tiled) wedges in the worker's gRPC thread, so it's removed from the default tiers.

Co-authored-by: FluffyAIcode <FluffyAIcode@users.noreply.github.com>
---
 services/agent_gateway/server.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/services/agent_gateway/server.py b/services/agent_gateway/server.py
index af56525ab..3daaf8023 100644
--- a/services/agent_gateway/server.py
+++ b/services/agent_gateway/server.py
@@ -157,9 +157,11 @@ def _auth(x_api_key: Optional[str] = Header(default=None)):
     "draft": {"fw_width": 384, "fw_height": 224, "fw_frames": 13, "proposer_steps": 5,
               "refine_steps": 12, "out_width": 640, "out_height": 384, "frames": 25,
               "fps": 12, "interpolate": 1, "interp_method": "linear", "refine_mode": "direct"},
-    "standard": {"fw_width": 480, "fw_height": 256, "fw_frames": 13, "proposer_steps": 6,
+    # standard = native T2V @ 832x480 DIRECT (same reliable path as 'high'; the tiled v2v refine
+    # hangs in the worker thread, so all tiers use direct native generation, differing by res/frames).
+    "standard": {"fw_width": 832, "fw_height": 480, "fw_frames": 25, "proposer_steps": 6,
                  "refine_steps": 16, "out_width": 832, "out_height": 480, "frames": 25,
-                 "fps": 12, "interpolate": 1, "interp_method": "linear", "refine_mode": ""},
+                 "fps": 16, "interpolate": 1, "interp_method": "linear", "refine_mode": "direct"},
     # 'high' = NATIVE T2V generation at 1280x720 directly on the strongest framework worker (the
     # CUDA box), no v2v refine. The distributed v2v refine op hangs in the worker's gRPC thread
     # after denoise (VAE decode never returns; framework/native-T2V at 720p works fine in-thread),

From 495c43c8dbbd3d09aa21862f9f07bc38ad2883ac Mon Sep 17 00:00:00 2001
From: Cursor Agent <cursoragent@cursor.com>
Date: Wed, 24 Jun 2026 06:43:09 +0000
Subject: [PATCH 21/28] loop-log: iter 45 - high/standard tunnel-stall was
 v2v-in-worker-thread hang; tiers -> native T2V direct

Co-authored-by: FluffyAIcode <FluffyAIcode@users.noreply.github.com>
---
 docs/adr/0001-kakeya-integration-loop-log.md | 27 ++++++++++++++++++++
 1 file changed, 27 insertions(+)

diff --git a/docs/adr/0001-kakeya-integration-loop-log.md b/docs/adr/0001-kakeya-integration-loop-log.md
index 10aa5a115..0462af752 100644
--- a/docs/adr/0001-kakeya-integration-loop-log.md
+++ b/docs/adr/0001-kakeya-integration-loop-log.md
@@ -1265,6 +1265,33 @@ Wired a new vast box (RTX PRO 6000 **Blackwell**, 96 GB) as the 3rd node and cha
 
 ---
 
+## Iteration 45 — "tunnel stall" was a v2v-in-worker-thread hang → all tiers use native T2V DIRECT
+
+Chased the `quality=high` wedge (job stuck at refine 58%). It was **not** the tunnel:
+
+- The progress reaching 58% is expected (14 effective steps / 24 `total` for strength 0.6). The
+  **final result never arrives** because the worker is stuck **after** the denoise loop.
+- Bisected it: `framework`/native-T2V at **1280x720** completes in the worker (final 1 MB mp4 sent
+  over the tunnel OK, public download = h264 1280x720); **standalone v2v** at 512x288 completes in a
+  plain process; but the **`refine` (v2v) op hangs in the worker's gRPC servicer thread** after
+  denoise (VAE decode/return never completes, GPU 0%) — at 720p AND at the capped 512x288. So it is a
+  v2v-op × gRPC-thread interaction, independent of resolution and of the SSH tunnel. VAE
+  tiling/slicing (added) did not help.
+- **Fix:** drop v2v from the default tiers and use the **reliable native-T2V DIRECT** path on the
+  strongest worker (the Blackwell CUDA box) for all tiers — which also matches ADR 0015's finding
+  that native T2V beats a low-res seed + I2V/refine. Presets: draft 384x224, **standard 832x480**,
+  **high 1280x720**, all `refine_mode=direct`, 25 frames (4n+1) + interpolation.
+- **Verified end-to-end via `agent.kakeya.ai`** (public download + ffprobe): standard = h264
+  832x480x25; high = h264 1280x720, 47 frames (~2 s). Draft also fine.
+- Left in place as defense/diagnostics: `REFINE_MAX_DIM` cap on the single-refine path, VAE
+  tiling on the CUDA backend, and worker op-error tracebacks + final-message logging.
+
+**Open:** the v2v generative refine (extra detail over native T2V) remains disabled pending a
+root-cause of the in-thread VAE-decode hang (suspect CUDA × gRPC C-core threading; py-spy blocked by
+no SYS_PTRACE in the vast container). Native-T2V direct is the reliable quality path meanwhile.
+
+---
+
 ## Open follow-ups (next iterations)
 - **Phase 2b — native gRPC transport.** Add an optional `kakeya` Python SDK transport
   for the bounded-memory long-context path (W3), behind the same tool, once the proto

From 9fd13b9be0c56f471954bcb32ce3dc9eb60b9253 Mon Sep 17 00:00:00 2001
From: Cursor Agent <cursoragent@cursor.com>
Date: Wed, 24 Jun 2026 06:55:38 +0000
Subject: [PATCH 22/28] route high -> Hunyuan native 720p via --prefer-backend
 (ADR 0017 premium layer)

Add orchestrator --prefer-backend (pin the generator to a backend substring, e.g. hunyuan;
falls back to fastest). Gateway: prefer_backend param + high preset prefer_backend=hunyuan,
native 1280x720 DIRECT, 30 steps / 45 frames. draft/standard stay on fast WAN; high routes
to the Hunyuan worker when present.

Co-authored-by: FluffyAIcode <FluffyAIcode@users.noreply.github.com>
---
 services/agent_gateway/server.py              | 21 ++++++++++++-------
 services/distributed_wan/grpc_orchestrator.py | 17 ++++++++++++++-
 2 files changed, 29 insertions(+), 9 deletions(-)

diff --git a/services/agent_gateway/server.py b/services/agent_gateway/server.py
index 3daaf8023..9c4924dd1 100644
--- a/services/agent_gateway/server.py
+++ b/services/agent_gateway/server.py
@@ -162,14 +162,14 @@ def _auth(x_api_key: Optional[str] = Header(default=None)):
     "standard": {"fw_width": 832, "fw_height": 480, "fw_frames": 25, "proposer_steps": 6,
                  "refine_steps": 16, "out_width": 832, "out_height": 480, "frames": 25,
                  "fps": 16, "interpolate": 1, "interp_method": "linear", "refine_mode": "direct"},
-    # 'high' = NATIVE T2V generation at 1280x720 directly on the strongest framework worker (the
-    # CUDA box), no v2v refine. The distributed v2v refine op hangs in the worker's gRPC thread
-    # after denoise (VAE decode never returns; framework/native-T2V at 720p works fine in-thread),
-    # so the reliable hi-res path is direct native generation — which also matches the "native T2V
-    # beats low-res-seed + I2V/refine" finding (ADR 0015). 25 frames (4n+1) + mci interpolation.
-    "high": {"fw_width": 1280, "fw_height": 720, "fw_frames": 25, "proposer_steps": 8,
-             "refine_steps": 24, "out_width": 1280, "out_height": 720, "frames": 25,
-             "fps": 24, "interpolate": 2, "interp_method": "mci", "refine_mode": "direct"},
+    # 'high' = NATIVE 720p generation on the HunyuanVideo (13B) worker — the premium quality layer
+    # (ADR 0017). Routed via prefer_backend='hunyuan' (DIRECT generation; no v2v refine). Hunyuan is
+    # not distilled, so ~30 steps; 45 frames (4n+1) native. Falls back to the fastest worker (WAN) if
+    # no hunyuan worker is up. WAN native 1280x720 direct remains the fallback for high.
+    "high": {"fw_width": 1280, "fw_height": 720, "fw_frames": 45, "proposer_steps": 30,
+             "refine_steps": 24, "out_width": 1280, "out_height": 720, "frames": 45,
+             "fps": 24, "interpolate": 1, "interp_method": "linear", "refine_mode": "direct",
+             "prefer_backend": "hunyuan"},
 }
 
 
@@ -192,6 +192,7 @@ class VideoRequest(BaseModel):
     out_width: Optional[int] = None
     out_height: Optional[int] = None
     refine_mode: Optional[str] = Field(None, pattern="^(auto|direct|single|tiled)$")
+    prefer_backend: Optional[str] = Field(None, pattern="^[a-z0-9_-]{0,24}$")
     # Long-form (ADR 0015 Phase 2): chunks>1 (or longform=true + seconds) = autoregressive I2V gen.
     longform: bool = False
     chunks: Optional[int] = Field(None, ge=1, le=20)
@@ -220,6 +221,8 @@ def resolved(self) -> dict:
                 p["chunks"] = max(1, math.ceil((want - p["chunk_overlap"]) / net))
         if p["chunks"] == 1 and p["seconds"] > 0:  # single-pass: seconds drives frame count
             p["frames"] = max(2, round(p["seconds"] * p["fps"]))
+        if self.prefer_backend is not None:
+            p["prefer_backend"] = self.prefer_backend
         p["quality"] = self.quality
         return p
 
@@ -259,6 +262,8 @@ def _run_video_job(job: Job):
         cmd += ["--seconds", str(p["seconds"])]
     if p.get("refine_mode"):  # quality preset / explicit override of refine topology
         cmd += ["--refine-mode", p["refine_mode"]]
+    if p.get("prefer_backend"):  # pin a tier to a model worker (e.g. 'high' -> hunyuan)
+        cmd += ["--prefer-backend", p["prefer_backend"]]
     if p.get("chunks", 1) > 1:  # long-form (autoregressive I2V continuity)
         cmd += ["--chunks", str(p["chunks"]), "--chunk-frames", str(p["chunk_frames"]),
                 "--chunk-overlap", str(p["chunk_overlap"])]
diff --git a/services/distributed_wan/grpc_orchestrator.py b/services/distributed_wan/grpc_orchestrator.py
index c9aa4e4d1..de5f1c483 100644
--- a/services/distributed_wan/grpc_orchestrator.py
+++ b/services/distributed_wan/grpc_orchestrator.py
@@ -138,6 +138,18 @@ def _png_bytes(frame):
     buf = io.BytesIO(); Image.fromarray(frame).save(buf, format="PNG"); return buf.getvalue()
 
 
+def _pick_fw(workers, prefer=""):
+    """Fastest framework/generator worker, optionally pinned to a backend substring (e.g. 'hunyuan').
+
+    Lets the gateway route a tier (e.g. 'high') to a specific model worker — Hunyuan for premium
+    quality — while other tiers fall through to the fastest available (WAN). No match -> fastest."""
+    if prefer:
+        m = [w for w in workers if prefer.lower() in (getattr(w, "backend", "") or "").lower()]
+        if m:
+            return max(m, key=lambda w: w.speed)
+    return max(workers, key=lambda w: w.speed)
+
+
 def _stitch(clips, overlap):
     """Concatenate chunks with an `overlap`-frame crossfade so chunk boundaries don't jump-cut.
 
@@ -198,6 +210,9 @@ def main():
     ap.add_argument("--proposer-steps", type=int, default=6)
     ap.add_argument("--refine-steps", type=int, default=16)
     ap.add_argument("--strength", type=float, default=0.6)
+    ap.add_argument("--prefer-backend", default=os.environ.get("PREFER_BACKEND", ""),
+                    help="prefer a framework/generator worker whose backend contains this substring "
+                         "(e.g. 'hunyuan'); falls back to the fastest worker if none match.")
     ap.add_argument("--seed", type=int, default=11)
     ap.add_argument("--no-refine", action="store_true",
                     help="single-worker DIRECT T2V (e.g. Mac-only): one generation at fw dims, "
@@ -306,7 +321,7 @@ def _to_out(frames):  # ensure every chunk is the same display resolution for cl
     # This is the Mac-only / single-GPU path (mlx-video has no vid2vid). Auto-enabled when no
     # refine worker is present, so the service still produces video with just the Mac.
     if args.no_refine or args.refine_mode == "direct" or not refine_workers:
-        fw = max(fw_workers, key=lambda w: w.speed)
+        fw = _pick_fw(fw_workers, args.prefer_backend)
         nf = args.fw_frames
         print(f"[orch] DIRECT (no-refine) on {fw.addr} ({fw.backend}) "
               f"@ {args.fw_width}x{args.fw_height}x{nf}", flush=True)

From 50b934fc275ed9e2632e4f369a6f757a455893da Mon Sep 17 00:00:00 2001
From: Cursor Agent <cursoragent@cursor.com>
Date: Wed, 24 Jun 2026 07:13:37 +0000
Subject: [PATCH 23/28] high: keep WAN native 720p (Hunyuan blocked by
 in-worker-thread VAE-decode hang)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Hunyuan native 720p produces断崖式 better quality standalone, but its heavy VAE decode
HANGS when run inside the worker's gRPC daemon thread (identical post-denoise GPU-0%
stall as WAN v2v; both work standalone). Needs the worker to run the heavy op off the
daemon thread (main thread/subprocess). Until then high stays on reliable WAN native;
--prefer-backend=hunyuan routing is wired and ready to flip on once that's fixed.

Co-authored-by: FluffyAIcode <FluffyAIcode@users.noreply.github.com>
---
 services/agent_gateway/server.py | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/services/agent_gateway/server.py b/services/agent_gateway/server.py
index 9c4924dd1..6f1a8542d 100644
--- a/services/agent_gateway/server.py
+++ b/services/agent_gateway/server.py
@@ -162,14 +162,15 @@ def _auth(x_api_key: Optional[str] = Header(default=None)):
     "standard": {"fw_width": 832, "fw_height": 480, "fw_frames": 25, "proposer_steps": 6,
                  "refine_steps": 16, "out_width": 832, "out_height": 480, "frames": 25,
                  "fps": 16, "interpolate": 1, "interp_method": "linear", "refine_mode": "direct"},
-    # 'high' = NATIVE 720p generation on the HunyuanVideo (13B) worker — the premium quality layer
-    # (ADR 0017). Routed via prefer_backend='hunyuan' (DIRECT generation; no v2v refine). Hunyuan is
-    # not distilled, so ~30 steps; 45 frames (4n+1) native. Falls back to the fastest worker (WAN) if
-    # no hunyuan worker is up. WAN native 1280x720 direct remains the fallback for high.
-    "high": {"fw_width": 1280, "fw_height": 720, "fw_frames": 45, "proposer_steps": 30,
-             "refine_steps": 24, "out_width": 1280, "out_height": 720, "frames": 45,
-             "fps": 24, "interpolate": 1, "interp_method": "linear", "refine_mode": "direct",
-             "prefer_backend": "hunyuan"},
+    # 'high' = native WAN T2V @ 1280x720 DIRECT (reliable). Hunyuan (13B) would be断崖式 better, but
+    # its heavy VAE decode HANGS when run inside the worker's gRPC daemon thread (same stall as WAN
+    # v2v; standalone Hunyuan/v2v both work — only heavy decode in the servicer thread hangs). Until
+    # the worker runs the heavy op off the daemon thread (main thread / subprocess), high stays on the
+    # reliable WAN native path. The --prefer-backend='hunyuan' routing is wired and ready (ADR 0017);
+    # set prefer_backend below to flip to Hunyuan once the worker concurrency fix lands.
+    "high": {"fw_width": 1280, "fw_height": 720, "fw_frames": 25, "proposer_steps": 8,
+             "refine_steps": 24, "out_width": 1280, "out_height": 720, "frames": 25,
+             "fps": 24, "interpolate": 2, "interp_method": "mci", "refine_mode": "direct"},
 }
 
 

From 2c23ba2a2052798730f610ab79c82e38d35e8c7e Mon Sep 17 00:00:00 2001
From: Cursor Agent <cursoragent@cursor.com>
Date: Wed, 24 Jun 2026 07:21:38 +0000
Subject: [PATCH 24/28] loop-log: iter 46 - Hunyuan 720p routing wired; blocked
 by in-worker-thread VAE-decode hang

Co-authored-by: FluffyAIcode <FluffyAIcode@users.noreply.github.com>
---
 docs/adr/0001-kakeya-integration-loop-log.md | 25 ++++++++++++++++++++
 1 file changed, 25 insertions(+)

diff --git a/docs/adr/0001-kakeya-integration-loop-log.md b/docs/adr/0001-kakeya-integration-loop-log.md
index 0462af752..c0b4a13f8 100644
--- a/docs/adr/0001-kakeya-integration-loop-log.md
+++ b/docs/adr/0001-kakeya-integration-loop-log.md
@@ -1292,6 +1292,31 @@ no SYS_PTRACE in the vast container). Native-T2V direct is the reliable quality
 
 ---
 
+## Iteration 46 — Hunyuan 720p routing wired; blocked by the same in-worker-thread decode hang
+
+Tried to make `high` = HunyuanVideo (13B) native 720p (ADR 0017 premium layer). Built the routing
+and confirmed the blocker is the worker concurrency model, not Hunyuan:
+
+- **Wired end-to-end:** orchestrator `--prefer-backend <substr>` (pin the generator to a backend,
+  e.g. `hunyuan`; falls back to fastest) + gateway `prefer_backend` param + `high` preset. Downloaded
+  the 40 GB model on the Blackwell, ran a dedicated `--backend hunyuan` worker on :50053 (no offload),
+  tunneled it to the head, added it to the worker list. A public `high` job correctly routed:
+  `DIRECT on 127.0.0.1:50054 (hunyuan-diffusers) @ 1280x720x45`, and Hunyuan **generated all 30
+  steps** at 100% GPU (~14 s/step) — generation works.
+- **Blocker:** after denoise, the worker **hangs at GPU 0%** in the Hunyuan VAE decode — the
+  *identical* post-denoise stall as WAN v2v (iter 45). Standalone Hunyuan (the iter-42 A/B) AND
+  standalone WAN v2v both complete; they only hang when the **heavy CUDA decode runs inside the gRPC
+  servicer's daemon thread** (`_stream`'s `threading.Thread`). WAN native-T2V's tiny VAE is the only
+  heavy decode that slips through — which is why all reliable tiers use it. (Can't py-spy: vast
+  container lacks SYS_PTRACE.)
+- **Decision:** reverted `high` to reliable **WAN native 720p** (verified: public h264 1280x720).
+  The `--prefer-backend=hunyuan` routing is wired and one line away from flipping on. **Real fix:
+  run the heavy generation op OFF the daemon thread** — a CUDA subprocess worker (spawn) or a
+  main-thread work-queue in the gРПС worker — so the decode doesn't deadlock with the gRPC C-core
+  threads. That worker-concurrency change is the next focused task to unlock Hunyuan (and v2v).
+
+---
+
 ## Open follow-ups (next iterations)
 - **Phase 2b — native gRPC transport.** Add an optional `kakeya` Python SDK transport
   for the bounded-memory long-context path (W3), behind the same tool, once the proto

From 51d1c6d562850f53132419bcc73ece0f5d9d9a08 Mon Sep 17 00:00:00 2001
From: Cursor Agent <cursoragent@cursor.com>
Date: Wed, 24 Jun 2026 07:33:41 +0000
Subject: [PATCH 25/28] grpc_worker: --subproc backend (run heavy CUDA gen in a
 spawned child)

Fixes the in-worker-thread VAE-decode hang (Hunyuan, WAN v2v): the gRPC servicer thread
deadlocks on heavy CUDA decode, but the same op runs fine in a standalone process's main
thread. SubprocBackend spawns a persistent child (spawn ctx) that loads the model and runs
ops in its MAIN thread, streaming per-step progress back over a Queue and handing frames via
a temp .npy; the gRPC side does only IPC + a file read (no CUDA on the gRPC threads). Gated
by --subproc for cuda/hunyuan.

Co-authored-by: FluffyAIcode <FluffyAIcode@users.noreply.github.com>
---
 services/distributed_wan/grpc_worker.py | 125 +++++++++++++++++++++++-
 1 file changed, 124 insertions(+), 1 deletion(-)

diff --git a/services/distributed_wan/grpc_worker.py b/services/distributed_wan/grpc_worker.py
index 961728b3a..8ed1879db 100644
--- a/services/distributed_wan/grpc_worker.py
+++ b/services/distributed_wan/grpc_worker.py
@@ -536,6 +536,122 @@ def RefineTile(self, request, context):
         yield from self._stream(self.backend.refine, request, context)
 
 
+# --------------------------------------------------------------------------- #
+# Subprocess backend — runs the heavy CUDA generation in a CLEAN child process.
+#
+# Why: heavy video VAE encode/decode (Hunyuan, WAN v2v) HANGS when run inside the gRPC servicer's
+# worker thread (denoise completes, then stuck at GPU 0%). The same op runs fine in a standalone
+# process's main thread. So we run the real backend in a spawned child process whose MAIN thread does
+# all CUDA work; the gRPC side only does IPC + a small file read (no CUDA on the gRPC threads).
+# Progress is streamed back per step; the result frames are handed over via a temp .npy file.
+# --------------------------------------------------------------------------- #
+_REQ_TYPES = {"framework": pb.FrameworkRequest, "refine": pb.RefineRequest}
+
+
+def _make_real_backend(kind, ops):
+    if kind == "cuda":
+        return CudaBackend()
+    if kind == "hunyuan":
+        return HunyuanBackend(ops)
+    raise ValueError(f"subproc unsupported backend: {kind}")
+
+
+def _subproc_main(cmd_q, res_q, kind, ops):
+    """Child process (spawn): load the model and run ops in the MAIN thread. CUDA lives only here."""
+    import os
+    import numpy as np
+    import tempfile
+    import traceback
+    try:
+        backend = _make_real_backend(kind, ops)
+        if kind == "cuda":
+            backend.load()
+        elif kind == "hunyuan":
+            backend._load_t2v()
+        res_q.put(("ready", None))
+    except Exception as exc:  # noqa: BLE001
+        res_q.put(("fatal", f"{type(exc).__name__}: {exc}\n{traceback.format_exc()}"))
+        return
+
+    class _PQ:  # forwards backend progress (q.put_nowait(("p", pct))) to the parent
+        def put_nowait(self, item):
+            try:
+                res_q.put(("p", float(item[1])))
+            except Exception:  # noqa: BLE001
+                pass
+        put = put_nowait
+
+    tmpdir = os.environ.get("DISTWAN_TMP", "/workspace")
+    while True:
+        job = cmd_q.get()
+        if job is None:
+            break
+        op_name, req_bytes, req_type = job
+        try:
+            req = _REQ_TYPES[req_type]()
+            req.ParseFromString(req_bytes)
+            frames = getattr(backend, op_name)(req, _PQ())
+            fd, path = tempfile.mkstemp(suffix=".npy", dir=tmpdir)
+            os.close(fd)
+            np.save(path, frames)
+            res_q.put(("done", path))
+        except Exception as exc:  # noqa: BLE001
+            res_q.put(("err", f"{type(exc).__name__}: {exc}\n{traceback.format_exc()}"))
+
+
+class SubprocBackend:
+    """Proxy that runs `kind` in a persistent spawned child (CUDA off the gRPC threads)."""
+
+    def __init__(self, kind, ops):
+        import multiprocessing as mp
+        self.kind = kind
+        self.ops = ops or ["framework"]
+        self._lock = threading.Lock()
+        ctx = mp.get_context("spawn")
+        self.cmd_q = ctx.Queue()
+        self.res_q = ctx.Queue()
+        self.proc = ctx.Process(target=_subproc_main, args=(self.cmd_q, self.res_q, kind, self.ops),
+                                daemon=True)
+        self.proc.start()
+        print(f"[grpc_worker] subproc backend '{kind}' child pid={self.proc.pid} loading model...",
+              flush=True)
+
+    def health(self):
+        spd = float(os.environ.get("SUBPROC_RELATIVE_SPEED", "0.8" if self.kind == "hunyuan" else "1.0"))
+        return pb.HealthReply(device="cuda", backend=f"{self.kind}-subproc", model=self.kind,
+                              ops=self.ops, ready=self.proc.is_alive(),
+                              note=f"subprocess worker pid={self.proc.pid}", relative_speed=spd)
+
+    def _run(self, op_name, req, req_type, q):
+        import numpy as np
+        with self._lock:
+            self.cmd_q.put((op_name, req.SerializeToString(), req_type))
+            while True:
+                kind, val = self.res_q.get()
+                if kind == "p":
+                    try:
+                        q.put_nowait(("p", val))
+                    except Exception:  # noqa: BLE001
+                        pass
+                elif kind == "ready":
+                    continue  # model finished loading; keep waiting for this job's result
+                elif kind == "done":
+                    frames = np.load(val)
+                    try:
+                        os.unlink(val)
+                    except OSError:
+                        pass
+                    return frames
+                elif kind in ("err", "fatal"):
+                    raise RuntimeError(val)
+
+    def framework(self, req, q):
+        return self._run("framework", req, "framework", q)
+
+    def refine(self, req, q):
+        return self._run("refine", req, "refine", q)
+
+
 def main():
     ap = argparse.ArgumentParser()
     ap.add_argument("--backend", choices=["cuda", "mlx", "test", "hunyuan"], default="cuda")
@@ -546,9 +662,16 @@ def main():
     ap.add_argument("--mlx-ops", default=os.environ.get("MLX_OPS", "framework"))
     ap.add_argument("--ops", default=os.environ.get("WORKER_OPS", ""),
                     help="restrict advertised ops, e.g. 'refine' to make a CUDA box refine-only")
+    ap.add_argument("--subproc", action="store_true",
+                    help="run the (cuda/hunyuan) backend in a spawned child process so heavy CUDA "
+                         "decode runs off the gRPC threads (fixes the in-thread VAE-decode hang).")
     args = ap.parse_args()
 
-    if args.backend == "cuda":
+    if args.subproc and args.backend in ("cuda", "hunyuan"):
+        _default = "framework,refine,i2v" if args.backend == "cuda" else "framework,i2v"
+        _ops = [o.strip() for o in (args.ops or _default).split(",") if o.strip()]
+        backend = SubprocBackend(args.backend, _ops)
+    elif args.backend == "cuda":
         backend = CudaBackend()
         if args.preload:
             backend.load()

From 98e6d9d92f620e6d452f88c72c9fc4042f7abf51 Mon Sep 17 00:00:00 2001
From: Cursor Agent <cursoragent@cursor.com>
Date: Wed, 24 Jun 2026 07:54:52 +0000
Subject: [PATCH 26/28] high -> Hunyuan native 720p (subproc worker) +
 supervisor manages it durably

SubprocBackend fixed the in-worker decode hang, so flip 'high' to prefer_backend=hunyuan
(native 1280x720, 30 steps/45 frames; falls back to WAN native if no hunyuan worker).
Supervisor now (VAST_HUNYUAN=1) launches the hunyuan --subproc worker in tmux, maintains
its tunnel (VAST_HUNYUAN_LOCAL->REMOTE), and appends it to the worker list. Verified public
render: h264 1280x720x45.

Co-authored-by: FluffyAIcode <FluffyAIcode@users.noreply.github.com>
---
 .../deploy/vast_refiner_supervisor.sh         | 23 +++++++++++++++++--
 services/agent_gateway/server.py              | 19 +++++++--------
 2 files changed, 31 insertions(+), 11 deletions(-)

diff --git a/services/agent_gateway/deploy/vast_refiner_supervisor.sh b/services/agent_gateway/deploy/vast_refiner_supervisor.sh
index 5cd36293e..18babc339 100755
--- a/services/agent_gateway/deploy/vast_refiner_supervisor.sh
+++ b/services/agent_gateway/deploy/vast_refiner_supervisor.sh
@@ -123,10 +123,29 @@ once() {
     sleep 2
   fi
 
+  # Optional Hunyuan premium worker (subproc) for the 'high' tier (ADR 0017). Runs as a SECOND tmux
+  # worker on the same box (port VAST_HUNYUAN_REMOTE), tunneled to VAST_HUNYUAN_LOCAL on the head, and
+  # appended to the worker list so the gateway's --prefer-backend=hunyuan routing can reach it.
+  HY=""
+  if [ "${VAST_HUNYUAN:-0}" = "1" ]; then
+    HR="${VAST_HUNYUAN_REMOTE:-50053}"; HL="${VAST_HUNYUAN_LOCAL:-50054}"
+    ssh $SSH_OPTS -p "$VAST_SSH_PORT" "$VAST_SSH_USER@$VAST_SSH_HOST" \
+      "tmux has-session -t hunyuan 2>/dev/null || tmux new-session -d -s hunyuan \"cd /workspace/distwan && HF_HOME=${VAST_HF_HOME:-/root/.hf_home} HUNYUAN_OFFLOAD=${VAST_HUNYUAN_OFFLOAD:-0} DISTWAN_TMP=/workspace HF_HUB_DISABLE_PROGRESS_BARS=1 TQDM_DISABLE=1 ${VAST_VENV_PY:-/venv/main/bin/python} grpc_worker.py --backend hunyuan --subproc --ops framework --host 0.0.0.0 --port $HR > /workspace/distwan/hunyuan_sp.log 2>&1\"" 2>/dev/null
+    if ssh $SSH_OPTS -p "$VAST_SSH_PORT" "$VAST_SSH_USER@$VAST_SSH_HOST" \
+         "bash -c 'exec 3<>/dev/tcp/127.0.0.1/$HR' 2>/dev/null && echo up" 2>/dev/null | grep -q up; then
+      nc -z 127.0.0.1 "$HL" >/dev/null 2>&1 || {
+        pkill -f "L $HL:localhost:$HR" 2>/dev/null; sleep 1
+        ssh $SSH_OPTS -o ExitOnForwardFailure=yes -fN -L "$HL:localhost:$HR" \
+            -p "$VAST_SSH_PORT" "$VAST_SSH_USER@$VAST_SSH_HOST" 2>/dev/null; sleep 2; }
+      nc -z 127.0.0.1 "$HL" >/dev/null 2>&1 && HY=",127.0.0.1:$HL"
+    fi
+  fi
+
   if reachable_local; then
-    write_workers "$BASE_WORKERS,127.0.0.1:$VAST_LOCAL_PORT"; log "vast refiner ONLINE -> 3-node (added 127.0.0.1:$VAST_LOCAL_PORT)"
+    write_workers "$BASE_WORKERS,127.0.0.1:$VAST_LOCAL_PORT$HY"
+    log "vast refiner ONLINE -> $BASE_WORKERS,127.0.0.1:$VAST_LOCAL_PORT$HY"
   else
-    write_workers "$BASE_WORKERS"; log "tunnel failed -> 2-Mac fallback"
+    write_workers "$BASE_WORKERS$HY"; log "wan tunnel failed -> $BASE_WORKERS$HY"
   fi
 }
 
diff --git a/services/agent_gateway/server.py b/services/agent_gateway/server.py
index 6f1a8542d..56632fb59 100644
--- a/services/agent_gateway/server.py
+++ b/services/agent_gateway/server.py
@@ -162,15 +162,16 @@ def _auth(x_api_key: Optional[str] = Header(default=None)):
     "standard": {"fw_width": 832, "fw_height": 480, "fw_frames": 25, "proposer_steps": 6,
                  "refine_steps": 16, "out_width": 832, "out_height": 480, "frames": 25,
                  "fps": 16, "interpolate": 1, "interp_method": "linear", "refine_mode": "direct"},
-    # 'high' = native WAN T2V @ 1280x720 DIRECT (reliable). Hunyuan (13B) would be断崖式 better, but
-    # its heavy VAE decode HANGS when run inside the worker's gRPC daemon thread (same stall as WAN
-    # v2v; standalone Hunyuan/v2v both work — only heavy decode in the servicer thread hangs). Until
-    # the worker runs the heavy op off the daemon thread (main thread / subprocess), high stays on the
-    # reliable WAN native path. The --prefer-backend='hunyuan' routing is wired and ready (ADR 0017);
-    # set prefer_backend below to flip to Hunyuan once the worker concurrency fix lands.
-    "high": {"fw_width": 1280, "fw_height": 720, "fw_frames": 25, "proposer_steps": 8,
-             "refine_steps": 24, "out_width": 1280, "out_height": 720, "frames": 25,
-             "fps": 24, "interpolate": 2, "interp_method": "mci", "refine_mode": "direct"},
+    # 'high' = HunyuanVideo (13B) NATIVE 1280x720 — the premium quality layer (ADR 0017), routed via
+    # prefer_backend='hunyuan' to the subprocess Hunyuan worker (--backend hunyuan --subproc, which
+    # runs the heavy VAE decode off the gRPC threads — fixes the in-worker-thread hang). Hunyuan is
+    # not distilled, so ~30 steps; 45 frames (4n+1) native, no interpolation. If no hunyuan worker is
+    # up, _pick_fw falls back to the fastest worker = WAN native 1280x720 (still reliable). Verified
+    # end-to-end on agent.kakeya.ai: public h264 1280x720x45.
+    "high": {"fw_width": 1280, "fw_height": 720, "fw_frames": 45, "proposer_steps": 30,
+             "refine_steps": 24, "out_width": 1280, "out_height": 720, "frames": 45,
+             "fps": 24, "interpolate": 1, "interp_method": "linear", "refine_mode": "direct",
+             "prefer_backend": "hunyuan"},
 }
 
 

From bd070319f1e7aff2ae1a74fb0eab983c4f3a5773 Mon Sep 17 00:00:00 2001
From: Cursor Agent <cursoragent@cursor.com>
Date: Wed, 24 Jun 2026 08:07:04 +0000
Subject: [PATCH 27/28] loop-log: iter 47 - subproc worker fixes decode hang;
 Hunyuan native 720p live as high tier

Co-authored-by: FluffyAIcode <FluffyAIcode@users.noreply.github.com>
---
 docs/adr/0001-kakeya-integration-loop-log.md | 22 ++++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/docs/adr/0001-kakeya-integration-loop-log.md b/docs/adr/0001-kakeya-integration-loop-log.md
index c0b4a13f8..b3e01c7f7 100644
--- a/docs/adr/0001-kakeya-integration-loop-log.md
+++ b/docs/adr/0001-kakeya-integration-loop-log.md
@@ -1317,6 +1317,28 @@ and confirmed the blocker is the worker concurrency model, not Hunyuan:
 
 ---
 
+## Iteration 47 — subprocess worker fixes the decode hang; Hunyuan 720p LIVE as the `high` tier
+
+Implemented the worker-concurrency fix and shipped Hunyuan as the premium tier.
+
+- **`SubprocBackend` (`--subproc`):** runs the real backend in a persistent **spawned child process**
+  whose MAIN thread does all CUDA work (load + denoise + the heavy VAE decode). The gRPC servicer
+  thread only does IPC: it streams per-step progress back over a `Queue` and receives the result
+  frames via a temp `.npy` handoff — **no CUDA on the gRPC threads**, so the post-denoise decode no
+  longer hangs. (Root finding: the decode also *eventually* completes non-subproc but unreliably/very
+  slowly with GPU idle; the child-main-thread path is the robust fix.)
+- **Hunyuan 720p productized:** `high` → `prefer_backend=hunyuan` (native 1280x720, 30 steps, 45
+  frames; WAN-native fallback via `_pick_fw` if no hunyuan worker). The supervisor (`VAST_HUNYUAN=1`)
+  launches the `--backend hunyuan --subproc` worker in tmux on the box, maintains a 2nd tunnel
+  (`VAST_HUNYUAN_LOCAL`→`REMOTE`), and appends it to the dynamic worker list — durable + auto-recover.
+- **Verified end-to-end on `agent.kakeya.ai`** (public download + ffprobe): a plain `{"quality":"high"}`
+  job routed to `hunyuan-subproc @ 1280x720x45` and produced **h264 1280x720, 45 frames** (also an
+  explicit-param run). Generation ~30×14 s + decode; the worker no longer wedges.
+- Model (40 GB `hunyuanvideo-community/HunyuanVideo`) cached on the box; HUNYUAN_OFFLOAD=0 (Blackwell
+  96 GB). Tiers: draft 384x224 / standard 832x480 (WAN native) / **high 1280x720 (Hunyuan native)**.
+
+---
+
 ## Open follow-ups (next iterations)
 - **Phase 2b — native gRPC transport.** Add an optional `kakeya` Python SDK transport
   for the bounded-memory long-context path (W3), behind the same tool, once the proto

From cfa9bc65f6dc883ad19a93484cd2e431e68fb0b4 Mon Sep 17 00:00:00 2001
From: Cursor Agent <cursoragent@cursor.com>
Date: Wed, 24 Jun 2026 13:54:57 +0000
Subject: [PATCH 28/28] agent runtime: quality passthrough + cross-scene
 consistency + governance reviewer

(1) generate_clip renders via the gateway /v1/videos when AGENT_GATEWAY_URL is set
    (quality=AGENT_VIDEO_QUALITY; high -> Hunyuan native 720p), else video_selector with a
    resolution hint. Director/editor (runtime) + render farm (gateway) split.
(2) plan_style_bible: one fixed subject+style 'continuity bible' embedded into every directed
    prompt for cross-scene consistency (assets/style_bible.json; AGENT_CONSISTENCY).
(3) review_plan: LLM reviewer over brief+prompts (consistency/coherence/quality) with optional
    auto-revise (AGENT_AUTO_REVISE) and human-approval gate (AGENT_REQUIRE_APPROVAL);
    assets/review.json. All shown under --plan-only. Tests updated (5 passed).

Co-authored-by: FluffyAIcode <FluffyAIcode@users.noreply.github.com>
---
 docs/adr/0016-autonomous-agent-runtime.md |  19 +++
 services/agent_runtime/run.py             | 181 +++++++++++++++++++++-
 tests/tools/test_agent_runtime.py         |  10 ++
 3 files changed, 204 insertions(+), 6 deletions(-)

diff --git a/docs/adr/0016-autonomous-agent-runtime.md b/docs/adr/0016-autonomous-agent-runtime.md
index 32e2c042e..335cc0066 100644
--- a/docs/adr/0016-autonomous-agent-runtime.md
+++ b/docs/adr/0016-autonomous-agent-runtime.md
@@ -63,6 +63,25 @@ New module `services/agent_runtime/` exposed as `AGENT_RUNTIME_CMD`
 (cloud LLM) next to the gateway; headless Mac stays a video worker; vast does heavy generation. The
 agent uses a cloud LLM (light CPU/network) — it does **not** steal GPU from video.
 
+## 3b. Quality, consistency, governance (added)
+
+Three upgrades toward production-grade, all in `run.py` (LLM-driven, schema-light sidecars):
+
+- **(1) Render quality passthrough.** `generate_clip` renders each scene through the **gateway**
+  `/v1/videos` when `AGENT_GATEWAY_URL` is set (`quality=AGENT_VIDEO_QUALITY`, default `standard`;
+  `high` → Hunyuan native 720p — the tier from ADR 0017), reusing the whole distributed pipeline.
+  Without a gateway URL it falls back to `video_selector` with a `resolution` hint. The runtime is the
+  director/editor; the gateway is the render farm. (`AGENT_GATEWAY_API_KEY` or `~/.kakeya/...` key.)
+- **(2) Cross-scene consistency.** `plan_style_bible` produces ONE fixed `subject`+`style` (a textual
+  "continuity bible") that `direct_video_prompt` embeds verbatim into EVERY shot's prompt, so the
+  recurring subject/style stays consistent across scenes (mitigates the "different dog each shot"
+  gap). Written to `assets/style_bible.json`. Toggle `AGENT_CONSISTENCY` (default on).
+- **(3) Governance reviewer.** `review_plan` is an LLM reviewer over the brief + directed prompts
+  (subject consistency / coherence / prompt quality) → `{approved, issues, revised_prompts}`. With
+  `AGENT_AUTO_REVISE` (default on) flagged prompts are auto-fixed; `AGENT_REQUIRE_APPROVAL=1` gates
+  (halts with the plan written for human approval). Written to `assets/review.json`. Toggle
+  `AGENT_REVIEW` (default on). All visible under `--plan-only` (no GPU).
+
 ## 4. Honest scope / limits
 
 - This is an **MVP "auto" path**, not the full governed `animated-explainer` pipeline (no
diff --git a/services/agent_runtime/run.py b/services/agent_runtime/run.py
index 92dab13eb..2feeb7b0a 100644
--- a/services/agent_runtime/run.py
+++ b/services/agent_runtime/run.py
@@ -44,6 +44,24 @@
 CLIP_H = int(os.environ.get("AGENT_CLIP_HEIGHT", "720"))
 
 
+def _flag(name: str, default: str = "1") -> bool:
+    return os.environ.get(name, default).lower() in ("1", "true", "yes")
+
+
+# (1) per-scene render quality. When AGENT_GATEWAY_URL is set, clips render through the gateway's
+#     /v1/videos (quality=AGENT_VIDEO_QUALITY -> e.g. 'high' = Hunyuan native 720p); else video_selector
+#     gets a resolution hint. (2) AGENT_CONSISTENCY -> a shared subject/style "bible" injected into every
+#     directed prompt for cross-scene consistency. (3) AGENT_REVIEW -> an LLM reviewer pass (+ optional
+#     auto-revise / human-approval gate).
+VIDEO_QUALITY = os.environ.get("AGENT_VIDEO_QUALITY", "standard").strip().lower()
+GATEWAY_URL = os.environ.get("AGENT_GATEWAY_URL", "").strip().rstrip("/")
+CONSISTENCY = _flag("AGENT_CONSISTENCY", "1")
+REVIEW = _flag("AGENT_REVIEW", "1")
+AUTO_REVISE = _flag("AGENT_AUTO_REVISE", "1")
+REQUIRE_APPROVAL = _flag("AGENT_REQUIRE_APPROVAL", "0")
+_QUALITY_RES = {"draft": "480p", "standard": "480p", "high": "720p"}
+
+
 def log(msg: str):
     print(f"[agent] {msg}", flush=True)
 
@@ -128,20 +146,51 @@ def _resolve_video_skill(registry) -> str:
     return txt[:8000]
 
 
-def direct_video_prompt(llm: LLM, scene_desc: str, brief: dict, skill_text: str) -> str:
+def plan_style_bible(llm: LLM, brief: dict, scenes: list) -> dict:
+    """LLM "continuity director": one fixed subject + style description reused across ALL shots.
+
+    Cross-scene consistency is the biggest gap for un-anchored multi-shot generation (the dog looks
+    different each shot). We can't rely on the video model alone, so we pin a textual "bible": a
+    detailed, fixed appearance of the recurring subject + a fixed look, injected verbatim into every
+    directed prompt so the model renders a consistent subject/style across scenes."""
+    sys_p = (
+        "You are OpenMontage's continuity director. Output a JSON object with exactly two string "
+        "fields: 'subject' = a fixed, highly specific visual description of the MAIN recurring "
+        "subject (species/breed, exact colors, distinctive markings, build, age) so it can be "
+        "rendered IDENTICALLY in every shot; 'style' = a fixed look (lighting, color palette, film "
+        "stock/medium, mood). Keep each under 40 words, concrete and positive. Output ONLY the JSON.")
+    descs = " | ".join(s.get("description", "") for s in scenes)[:1500]
+    try:
+        b = llm.complete_json(sys_p, f"Title: {brief.get('title','')}\nTone: {brief.get('tone','')}\n"
+                                     f"Style: {brief.get('style','')}\nScenes: {descs}")
+        return {"subject": str(b.get("subject", "")).strip(), "style": str(b.get("style", "")).strip()}
+    except Exception as exc:  # noqa: BLE001
+        log(f"style-bible skipped ({exc})")
+        return {}
+
+
+def direct_video_prompt(llm: LLM, scene_desc: str, brief: dict, skill_text: str,
+                        bible: dict | None = None) -> str:
     """LLM "prompt director": rewrite a scene into ONE rich, model-appropriate T2V prompt.
 
     Native T2V conditions generation via cross-attention over the text embeddings, so a longer,
     concrete, positive prompt (subject+action+setting+lighting+camera+style) yields a far stronger
-    result than a bare scene description. Guidance comes from the provider's Layer-3 skill."""
+    result than a bare scene description. Guidance comes from the provider's Layer-3 skill. When a
+    continuity `bible` is given, its fixed subject+style are embedded so the subject stays consistent
+    across shots."""
     if not scene_desc:
         return scene_desc
+    anchor = ""
+    if bible and (bible.get("subject") or bible.get("style")):
+        anchor = ("\nCONTINUITY — keep IDENTICAL across all shots; OPEN the prompt by restating the "
+                  f"subject's fixed appearance: SUBJECT: {bible.get('subject','')}. "
+                  f"STYLE: {bible.get('style','')}.")
     sys_p = (
         "You are OpenMontage's video prompt director. Rewrite the scene into ONE rich, concrete "
         "text-to-video prompt for a native text-to-video model. Include subject, action, setting, "
         "lighting, camera framing/movement, and style. Be vivid and specific, ~40-70 words, fully "
         "positive phrasing (describe what to SHOW, never what to avoid), no contradictions. "
-        "Output ONLY the prompt text — no quotes, no JSON, no preamble." + skill_text
+        "Output ONLY the prompt text — no quotes, no JSON, no preamble." + anchor + skill_text
     )
     user = (f"Scene: {scene_desc}\nOverall tone: {brief.get('tone', '')}\n"
             f"Style: {brief.get('style', '')}\nTitle: {brief.get('title', '')}")
@@ -154,6 +203,29 @@ def direct_video_prompt(llm: LLM, scene_desc: str, brief: dict, skill_text: str)
     return out[:600] or scene_desc
 
 
+def review_plan(llm: LLM, brief: dict, scene_plan: dict, prompts: list) -> dict:
+    """LLM "reviewer": governance pass over the plan + directed prompts before rendering.
+
+    Checks subject consistency across shots, coherence/on-brief, and concrete positive prompts.
+    Returns {approved: bool, issues: [...], revised_prompts: [{scene_id, prompt}]}."""
+    sys_p = (
+        "You are OpenMontage's plan reviewer. Given the brief, scene plan, and per-scene directed "
+        "prompts, verify: (1) EVERY shot depicts the SAME subject with consistent appearance, "
+        "(2) the shots form a coherent on-brief sequence, (3) each prompt is concrete and positive. "
+        "Output ONLY a JSON object: {\"approved\": bool, \"issues\": [strings], \"revised_prompts\": "
+        "[{\"scene_id\": str, \"prompt\": str}]}. Include revised_prompts ONLY for shots that need a "
+        "fix (e.g. to enforce subject consistency); omit or leave empty if all good.")
+    payload = {"brief": {k: brief.get(k) for k in ("title", "tone", "style")},
+               "prompts": prompts}
+    try:
+        r = llm.complete_json(sys_p, json.dumps(payload))
+        return {"approved": bool(r.get("approved", True)), "issues": list(r.get("issues", []) or []),
+                "revised_prompts": list(r.get("revised_prompts", []) or [])}
+    except Exception as exc:  # noqa: BLE001
+        log(f"review skipped ({exc})")
+        return {"approved": True, "issues": [], "revised_prompts": []}
+
+
 # --------------------------------------------------------------------------- #
 # asset helpers (real providers via the registry, or ffmpeg fixtures for tests)
 # --------------------------------------------------------------------------- #
@@ -169,15 +241,67 @@ def _ffmpeg_sine(out: str, seconds: float):
                    capture_output=True, check=True)
 
 
+def _read_gateway_key() -> str:
+    k = os.environ.get("AGENT_GATEWAY_API_KEY", "").strip()
+    if k:
+        return k
+    for p in (Path(os.environ.get("AGENT_GATEWAY_API_KEY_FILE", "")),
+              Path.home() / ".kakeya" / "agent_gateway_api_key"):
+        try:
+            return p.read_text().strip()
+        except OSError:
+            continue
+    return ""
+
+
+def _gateway_clip(prompt: str, out: str, seconds: float, quality: str) -> str:
+    """Render one clip through the gateway's /v1/videos (so 'high' = Hunyuan native 720p).
+
+    The agent runtime is the director/editor; the gateway is the render farm. This reuses the whole
+    distributed pipeline + quality tiers instead of re-implementing provider selection here."""
+    import time
+    import urllib.request
+    headers = {"Content-Type": "application/json"}
+    key = _read_gateway_key()
+    if key:
+        headers["X-API-Key"] = key
+    body = json.dumps({"prompt": prompt, "mode": "video", "quality": quality,
+                       "seconds": float(max(2.0, seconds))}).encode()
+    req = urllib.request.Request(GATEWAY_URL + "/v1/videos", data=body, headers=headers)
+    with urllib.request.urlopen(req, timeout=60) as r:
+        job = json.loads(r.read())
+    jid = job["job_id"]
+    deadline = time.time() + int(os.environ.get("AGENT_GATEWAY_TIMEOUT", "1800"))
+    poll = urllib.request.Request(f"{GATEWAY_URL}/v1/jobs/{jid}", headers=headers)
+    while time.time() < deadline:
+        time.sleep(5)
+        with urllib.request.urlopen(poll, timeout=30) as r:
+            st = json.loads(r.read())
+        if st.get("status") == "done":
+            dl = urllib.request.Request(f"{GATEWAY_URL}/v1/jobs/{jid}/video", headers=headers)
+            with urllib.request.urlopen(dl, timeout=120) as r, open(out, "wb") as f:
+                f.write(r.read())
+            return out
+        if st.get("status") == "error":
+            raise RuntimeError(f"gateway clip failed: {st.get('error')}")
+    raise RuntimeError("gateway clip timed out")
+
+
+def _quality_to_res(quality: str) -> str:
+    return _QUALITY_RES.get(quality, "480p")
+
+
 def generate_clip(registry, prompt: str, out: str, seconds: float, idx: int) -> str:
     if FAKE:
         _ffmpeg_clip(out, seconds, ["darkblue", "darkgreen", "darkred", "purple", "teal"][idx % 5])
         return out
+    if GATEWAY_URL:  # render via the gateway (quality tier -> e.g. Hunyuan native 720p for 'high')
+        return _gateway_clip(prompt, out, seconds, VIDEO_QUALITY)
     sel = registry.get("video_selector")
     if sel is None:
         raise RuntimeError("no video_selector tool available")
     res = sel.execute({"prompt": prompt, "aspect_ratio": "16:9", "duration": str(int(seconds)),
-                       "output_path": out})
+                       "resolution": _quality_to_res(VIDEO_QUALITY), "output_path": out})
     if not res.success:
         raise RuntimeError(f"video generation failed: {res.error}")
     return res.data.get("output") or res.data.get("output_path") or out
@@ -317,21 +441,66 @@ def run(prompt: str, out: str, llm: LLM | None = None, project_dir: Path | None
 
     # 3b) prompt director (LLM, no GPU): rewrite each scene into a rich native-T2V prompt up front,
     # so the directed prompts can be inspected (and validated via --plan-only) before any rendering.
+    # 3a) continuity bible (cross-scene consistency): one fixed subject+style for ALL shots.
+    bible = {}
+    if CONSISTENCY:
+        bible = plan_style_bible(llm, brief, scenes)
+        if bible.get("subject") or bible.get("style"):
+            log(f"continuity: subject='{bible.get('subject','')[:60]}' style='{bible.get('style','')[:40]}'")
+            try:
+                (assets / "style_bible.json").write_text(json.dumps(bible, indent=2))
+            except OSError:
+                pass
+
+    # 3b) prompt director (LLM, no GPU): rewrite each scene into a rich native-T2V prompt up front
+    # (with the continuity anchor), so the directed prompts can be inspected (--plan-only) and
+    # reviewed before any rendering.
     vskill = _resolve_video_skill(registry)
     log(f"prompt-director: loaded provider guidance ({len(vskill)} chars)")
     director_prompts = []
     for i, sc in enumerate(scenes):
         raw = sc.get("description", prompt)
-        vprompt = direct_video_prompt(llm, raw, brief, vskill)
+        vprompt = direct_video_prompt(llm, raw, brief, vskill, bible)
         director_prompts.append({"scene_id": sc.get("id", f"sc{i}"), "raw": raw, "prompt": vprompt})
         log(f"scene {i}: directed prompt — {vprompt[:80]}")
+
+    # 3c) governance: reviewer pass over the plan + directed prompts (consistency / coherence /
+    # prompt quality). Optionally auto-revise flagged prompts, and/or gate on human approval.
+    if REVIEW:
+        rev = review_plan(llm, brief, scene_plan, director_prompts)
+        log(f"review: approved={rev['approved']} issues={len(rev['issues'])}")
+        for it in rev["issues"][:5]:
+            log(f"  - {str(it)[:100]}")
+        if AUTO_REVISE and rev.get("revised_prompts"):
+            by_id = {d["scene_id"]: d for d in director_prompts}
+            n = 0
+            for rp in rev["revised_prompts"]:
+                d = by_id.get(rp.get("scene_id"))
+                if d and rp.get("prompt"):
+                    d["prompt"] = " ".join(str(rp["prompt"]).split())[:600]; n += 1
+            if n:
+                log(f"review: auto-revised {n} prompt(s)")
+        try:
+            (assets / "review.json").write_text(json.dumps(rev, indent=2))
+        except OSError:
+            pass
+        if REQUIRE_APPROVAL and not rev["approved"]:
+            dp_path = assets / "director_prompts.json"
+            try:
+                dp_path.write_text(json.dumps(director_prompts, indent=2))
+            except OSError:
+                pass
+            log(f"APPROVAL REQUIRED: review not approved; plan written to {dp_path}. "
+                "Set AGENT_REQUIRE_APPROVAL=0 (or fix the brief) and re-run to proceed.")
+            raise SystemExit(3)
+
     dp_path = assets / "director_prompts.json"
     try:
         dp_path.write_text(json.dumps(director_prompts, indent=2))
     except OSError:
         pass
     if plan_only:
-        log(f"plan-only: planning + director done, skipping asset generation/compose -> {dp_path}")
+        log(f"plan-only: planning + director + review done, skipping render -> {dp_path}")
         return str(dp_path)
 
     # 4) assets: per-scene video (video_selector -> best provider) from the DIRECTED prompt
diff --git a/tests/tools/test_agent_runtime.py b/tests/tools/test_agent_runtime.py
index 77a238229..db0642cc8 100644
--- a/tests/tools/test_agent_runtime.py
+++ b/tests/tools/test_agent_runtime.py
@@ -26,6 +26,11 @@
 
 def _stub(system: str, user: str) -> str:
     s = system[:120].lower()  # match the prompt PREFIX, before appended skill text
+    if "continuity director" in s:  # style/character bible (cross-scene consistency)
+        return json.dumps({"subject": "a black-and-white Border Collie with a white blaze and amber eyes",
+                            "style": "cinematic, soft cool winter light, 35mm film, muted palette"})
+    if "plan reviewer" in s:  # governance review pass
+        return json.dumps({"approved": True, "issues": [], "revised_prompts": []})
     if "video prompt director" in s:
         # echo the raw scene so the test can verify the DIRECTED prompt is what reaches the generator
         scene = user.split("\n", 1)[0].removeprefix("Scene: ").strip()
@@ -98,6 +103,11 @@ def test_agent_runtime_plan_only(tmp_path):
     assert len(dp) == 2 and all(e["prompt"].startswith("directed cinematic shot, ") for e in dp)
     # no clips/compose happened
     assert not list((tmp_path / "proj" / "assets").glob("scene_*.mp4"))
+    # continuity bible + governance review artifacts were produced
+    bible = json.loads((tmp_path / "proj" / "assets" / "style_bible.json").read_text())
+    assert bible.get("subject") and bible.get("style")
+    review = json.loads((tmp_path / "proj" / "assets" / "review.json").read_text())
+    assert review.get("approved") is True
 
 
 def test_llm_json_extraction():