Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 24 additions & 0 deletions docs/adr/0001-kakeya-integration-loop-log.md
Original file line number Diff line number Diff line change
Expand Up @@ -1122,6 +1122,30 @@ auto-recovery. (Caveat: the ~84 G I2V model re-downloads unless `HF_HOME` is on

---

## Iteration 40 — Gap A closed: deploy PR#5 to agent.kakeya.ai + wire quality=high → true 720p I2V

**Honesty correction:** Iter 36–39 results were from DIRECT orchestrator runs. The live public gateway
was still PR#4 (no presets/refine-mode/chunks/mci) — confirmed by `grep` on the deployed code +
`healthz mode=distributed`. So `agent.kakeya.ai` was serving distributed-tiled (~480p mix), NOT the
720p I2V I had summarized. Owned + fixed.

**Deployed:** head repo → `main` (PR#5) then the `quality=high→I2V` follow-up; gateway restarted.
Wired `quality=high` to the **true-720p hero path**: orchestrator `--longform` forces the I2V
generative path even at chunks=1; `high` preset → `longform=True` (+ mci x2). Multi-chunk continuity
via `chunks=N` / `longform+seconds`.

**Verified through the PUBLIC endpoint** (`agent.kakeya.ai`, not direct):
- multi-chunk: `{chunks:2,out 1280x720}` → `mode=longform continuity=i2v px=[720,1280] frames=46` →
public download ffprobe **h264 1280×720, 46f, 2.875s**.
- one-flag hero: `{quality:"high"}` → `mode=longform chunks=1 px=[720,1280] frames=47` → public
ffprobe **h264 1280×720, 47f, 1.96s**, gen≈211 s.

**Honest tiers now:** default/`standard` = fast distributed tiled (~480p, ~2 s); `high` = true 720p I2V
generative + optical-flow (~3.5 min); `longform`/`chunks` = multi-chunk continuity (minutes×N). Gap B
(full OpenMontage agent + all skills behind the gateway; `agent_runtime=false`) remains — separate plan.

---

## Open follow-ups (next iterations)
- **Phase 2b — native gRPC transport.** Add an optional `kakeya` Python SDK transport
for the bounded-memory long-context path (W3), behind the same tool, once the proto
Expand Down
50 changes: 37 additions & 13 deletions services/agent_gateway/server.py
Original file line number Diff line number Diff line change
Expand Up @@ -160,9 +160,12 @@ def _auth(x_api_key: Optional[str] = Header(default=None)):
"standard": {"fw_width": 480, "fw_height": 256, "fw_frames": 13, "proposer_steps": 6,
"refine_steps": 16, "out_width": 832, "out_height": 480, "frames": 25,
"fps": 12, "interpolate": 1, "interp_method": "linear", "refine_mode": ""},
# high = the true 720p generative hero path: single-chunk I2V (longform) on the CUDA i2v worker,
# + optical-flow (mci) smoothing. Slower (~minutes) but matches the validated quality.
"high": {"fw_width": 512, "fw_height": 288, "fw_frames": 17, "proposer_steps": 8,
"refine_steps": 24, "out_width": 1280, "out_height": 720, "frames": 25,
"fps": 24, "interpolate": 2, "interp_method": "mci", "refine_mode": "single"},
"refine_steps": 22, "out_width": 1280, "out_height": 720, "frames": 25,
"fps": 24, "interpolate": 2, "interp_method": "mci", "refine_mode": "",
"longform": True, "chunk_frames": 25, "chunk_overlap": 4},
}


Expand Down Expand Up @@ -203,15 +206,17 @@ def resolved(self) -> dict:
p["seed"] = self.seed
p["seconds"] = self.seconds or 0.0
p["chunks"] = self.chunks or 1
p["chunk_frames"] = self.chunk_frames or 25
p["chunk_overlap"] = self.chunk_overlap if self.chunk_overlap is not None else 4
# Long-form (opt-in): derive chunk count from target seconds when longform is requested.
if self.longform and p["chunks"] == 1 and p["seconds"] > 0:
p["chunk_frames"] = self.chunk_frames or p.get("chunk_frames", 25)
p["chunk_overlap"] = self.chunk_overlap if self.chunk_overlap is not None else p.get("chunk_overlap", 4)
# longform = explicit request OR preset (high). Single-chunk longform = the true-720p I2V path.
p["longform"] = bool(self.longform or p.get("longform", False))
# Long-form: derive chunk count from target seconds when longform is requested.
if p["longform"] and p["chunks"] == 1 and p["seconds"] > 0:
net = max(1, p["chunk_frames"] - p["chunk_overlap"])
want = round(p["seconds"] * p["fps"])
if want > p["chunk_frames"]:
p["chunks"] = max(1, math.ceil((want - p["chunk_overlap"]) / net))
if p["chunks"] == 1 and p["seconds"] > 0: # single-pass: seconds drives frame count
if not p["longform"] and p["chunks"] == 1 and p["seconds"] > 0: # single-pass: seconds->frames
p["frames"] = max(2, round(p["seconds"] * p["fps"]))
p["quality"] = self.quality
return p
Expand Down Expand Up @@ -252,9 +257,9 @@ def _run_video_job(job: Job):
cmd += ["--seconds", str(p["seconds"])]
if p.get("refine_mode"): # quality preset / explicit override of refine topology
cmd += ["--refine-mode", p["refine_mode"]]
if p.get("chunks", 1) > 1: # long-form (autoregressive I2V continuity)
cmd += ["--chunks", str(p["chunks"]), "--chunk-frames", str(p["chunk_frames"]),
"--chunk-overlap", str(p["chunk_overlap"])]
if p.get("longform") or p.get("chunks", 1) > 1: # I2V generative path (single or multi-chunk)
cmd += ["--longform", "--chunks", str(p.get("chunks", 1)),
"--chunk-frames", str(p["chunk_frames"]), "--chunk-overlap", str(p["chunk_overlap"])]
if POOL_MODE:
cmd.append("--no-refine") # each pooled worker is a single framework-only GPU (Mac MLX)
elif GW_MODE == "pipeline":
Expand Down Expand Up @@ -412,15 +417,32 @@ def index():
border:1px solid #1c2942;border-radius:10px;padding:12px;font-size:12.5px;color:#a9c2e6}
video{width:100%;border-radius:12px;margin-top:12px;background:#000}
input[type=password]{padding:10px;border-radius:8px;border:1px solid #2a3a59;background:#0e1830;color:#e7eefc}
select{padding:10px;border-radius:8px;border:1px solid #2a3a59;background:#0e1830;color:#e7eefc;font:inherit}
label{color:#8fa6c8;font-size:13px;display:block;margin-bottom:4px}
.opts{display:flex;gap:14px;flex-wrap:wrap;margin-top:12px}
</style></head><body><div class="wrap">
<h1>OpenMontage — Agent Video</h1>
<p class="sub">Describe a clip. The distributed cluster (MLX proposer + CUDA refiner) renders it.</p>
<p class="sub">Describe a clip. The distributed cluster (MLX proposer + CUDA / I2V refiner) renders it.</p>
<textarea id="p" placeholder="a red fox walking through a snowy forest, cinematic, soft winter light"></textarea>
<div class="opts">
<div><label>Quality</label>
<select id="q">
<option value="high" selected>High — 720p generative (I2V) · ~3–4 min</option>
<option value="standard">Standard — ~480p · ~1–2 min</option>
<option value="draft">Draft — fast preview</option>
</select></div>
<div><label>Length</label>
<select id="len">
<option value="short" selected>Short (~2 s)</option>
<option value="5">Longer (~5 s, multi-shot)</option>
<option value="8">Long (~8 s, multi-shot)</option>
</select></div>
</div>
<div class="row">
<button id="go">Generate</button>
<input id="key" type="password" placeholder="API key (if required)" style="flex:1;min-width:160px">
</div>
<p class="muted">Direct text→video via <code>/v1/videos</code>. First render can take a few minutes.</p>
<p class="muted"><b>High</b> = native 720p generative (I2V) — best quality, a few minutes. <b>Standard</b> = fast ~480p draft. Longer = multi-shot I2V continuity (minutes × shots).</p>
<div class="card" id="status" style="display:none">
<div id="stage" class="muted">queued…</div>
<div class="bar"><i id="fill"></i></div>
Expand All @@ -445,7 +467,9 @@ def index():
$('go').disabled=true;$('status').style.display='block';$('vid').style.display='none';
$('stage').textContent='submitting…';$('fill').style.width='0';$('log').textContent='';
const h={'Content-Type':'application/json'}; const k=$('key').value.trim(); if(k)h['X-API-Key']=k;
const r=await fetch('/v1/videos',{method:'POST',headers:h,body:JSON.stringify({prompt})});
const body={prompt, quality:$('q').value};
const len=$('len').value; if(len!=='short'){body.longform=true; body.seconds=parseFloat(len);}
const r=await fetch('/v1/videos',{method:'POST',headers:h,body:JSON.stringify(body)});
if(!r.ok){$('go').disabled=false;$('stage').textContent='error: '+r.status+' '+(await r.text());return;}
const {job_id}=await r.json();
timer=setInterval(()=>poll(job_id),2500);poll(job_id);
Expand Down
6 changes: 5 additions & 1 deletion services/distributed_wan/grpc_orchestrator.py
Original file line number Diff line number Diff line change
Expand Up @@ -227,6 +227,9 @@ def main():
help="interpolation: 'linear' (blend) or 'mci' (ffmpeg motion-compensated / "
"optical-flow — RIFE-class smoothness; falls back to linear if unavailable).")
# Long-form (ADR 0015 Phase 2): chunked autoregressive generation with I2V continuity.
ap.add_argument("--longform", action="store_true",
help="force the I2V generative path even for a single chunk (chunks=1) — the "
"true-720p hero path on an i2v worker. >1 chunks is multi-chunk continuity.")
ap.add_argument("--chunks", type=int, default=1,
help=">1 enables long-form: N chunks generated autoregressively (chunk N+1 seeded "
"by the last frame of chunk N via I2V) and crossfade-stitched.")
Expand All @@ -250,7 +253,8 @@ def main():
# seeded by the last frame of chunk N via I2V (continuity), then crossfade-stitched. Runs on an
# i2v-capable worker (the CUDA box with CUDA_I2V_MODEL); falls back to independent T2V chunks
# (no continuity) with a warning if no i2v worker is present.
if args.chunks > 1:
if args.longform or args.chunks > 1:
args.chunks = max(1, args.chunks)
i2v_workers = [w for w in workers if "i2v" in w.ops]
cont = bool(i2v_workers)
ow, oh = args.out_width, args.out_height
Expand Down
10 changes: 7 additions & 3 deletions tests/tools/test_agent_gateway.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ def client(tmp_path, monkeypatch):
" ap.add_argument(a)\n"
"ap.add_argument('--no-refine',action='store_true')\n"
"ap.add_argument('--single-refine',action='store_true')\n"
"ap.add_argument('--longform',action='store_true')\n"
"ap.add_argument('--refine-spread')\n"
"x=ap.parse_args()\n"
"print('[orch] framework: 5%',flush=True)\n"
Expand Down Expand Up @@ -125,8 +126,9 @@ def _mode_fixture(tmp_path, monkeypatch, *, workers, mode_env):
"for a in ['--prompt','--out','--frames','--fw-width','--fw-height','--fw-frames','--proposer-steps','--refine-steps','--seed','--out-width','--out-height','--refine-spread','--fps','--interpolate','--interp-method','--refine-mode','--seconds','--chunks','--chunk-frames','--chunk-overlap']: ap.add_argument(a)\n"
"ap.add_argument('--no-refine',action='store_true')\n"
"ap.add_argument('--single-refine',action='store_true')\n"
"ap.add_argument('--longform',action='store_true')\n"
"x=ap.parse_args()\n"
"flags={'single_refine':x.single_refine,'no_refine':x.no_refine,'refine_spread':x.refine_spread,'workers':os.environ.get('WAN_WORKERS',''),'fps':x.fps,'interpolate':x.interpolate,'interp_method':x.interp_method,'refine_mode':x.refine_mode,'out_width':x.out_width,'frames':x.frames,'seconds':x.seconds,'refine_steps':x.refine_steps,'chunks':x.chunks,'chunk_frames':x.chunk_frames,'chunk_overlap':x.chunk_overlap}\n"
"flags={'single_refine':x.single_refine,'no_refine':x.no_refine,'refine_spread':x.refine_spread,'workers':os.environ.get('WAN_WORKERS',''),'fps':x.fps,'interpolate':x.interpolate,'interp_method':x.interp_method,'refine_mode':x.refine_mode,'out_width':x.out_width,'frames':x.frames,'seconds':x.seconds,'refine_steps':x.refine_steps,'chunks':x.chunks,'chunk_frames':x.chunk_frames,'chunk_overlap':x.chunk_overlap,'longform':x.longform}\n"
"print('[orch] refine: 100%',flush=True)\n"
"p=pathlib.Path(x.out); p.parent.mkdir(parents=True,exist_ok=True); p.write_bytes(b'MODEMP4')\n"
"print('FLAGS '+json.dumps(flags),flush=True)\n"
Expand Down Expand Up @@ -195,8 +197,9 @@ def test_quality_presets(tmp_path, monkeypatch):
# high: seam-free full-frame generative refine on the best refiner, hi-res, smoother
fh = _job_flags(c, c.post("/v1/videos", json={"prompt": "hero clip", "quality": "high"}).json()["job_id"])
assert fh["out_width"] == "1280" and fh["fps"] == "24" and fh["interpolate"] == "2"
assert fh["refine_mode"] == "single" and fh["refine_steps"] == "24"
assert fh["interp_method"] == "mci" # high uses optical-flow interpolation
assert fh["interp_method"] == "mci" # high uses optical-flow interpolation
assert fh["longform"] is True # high = true 720p I2V generative path
assert fh["refine_mode"] is None # longform supersedes refine topology
# draft: proposer-only direct
fd = _job_flags(c, c.post("/v1/videos", json={"prompt": "draft clip", "quality": "draft"}).json()["job_id"])
assert fd["refine_mode"] == "direct"
Expand Down Expand Up @@ -252,6 +255,7 @@ def test_pool_mode_two_macs(tmp_path, monkeypatch):
"for a in ['--prompt','--out','--frames','--fw-width','--fw-height','--fw-frames','--proposer-steps','--refine-steps','--seed','--out-width','--out-height','--fps','--interpolate','--interp-method','--refine-mode','--seconds','--chunks','--chunk-frames','--chunk-overlap']: ap.add_argument(a)\n"
"ap.add_argument('--no-refine',action='store_true')\n"
"ap.add_argument('--single-refine',action='store_true')\n"
"ap.add_argument('--longform',action='store_true')\n"
"ap.add_argument('--refine-spread')\n"
"x=ap.parse_args()\n"
"assert x.no_refine, 'pool mode must pass --no-refine'\n"
Expand Down
20 changes: 20 additions & 0 deletions tests/tools/test_distributed_wan_pipeline.py
Original file line number Diff line number Diff line change
Expand Up @@ -168,6 +168,26 @@ def test_orchestrator_longform_chunks(tmp_path):
gen.stop(0)


def test_orchestrator_longform_single_chunk(tmp_path):
"""--longform with chunks=1 (quality=high path) -> a single I2V generative clip at out-res."""
gen, p_gen = _serve(["framework", "t2v", "i2v"])
import time as _t
_t.sleep(0.5)
try:
out = tmp_path / "hero.mp4"
env = {**os.environ, "WAN_WORKERS": f"127.0.0.1:{p_gen}"}
proc = subprocess.run(
[sys.executable, str(DWAN / "grpc_orchestrator.py"), "--prompt", "a fox", "--longform",
"--chunks", "1", "--chunk-frames", "13", "--out-width", "64", "--out-height", "48",
"--out", str(out)], env=env, capture_output=True, text=True, timeout=120)
assert out.exists(), proc.stdout + proc.stderr
info = json.loads([l for l in proc.stdout.splitlines() if l.startswith("ORCH_DONE")][-1].split(" ", 1)[1])
assert info["mode"] == "longform" and info["continuity"] == "i2v" and info["chunks"] == 1
assert info["px"] == [48, 64]
finally:
gen.stop(0)


def test_orchestrator_refine_mode_single_fps_interp(tmp_path):
"""--refine-mode single (quality path) + --fps/--interpolate: forces a seam-free full-frame
refine on the best refiner and the output frame count reflects interpolation."""
Expand Down