From bc673d9a9fbfe492c829c13097367753c709f4fd Mon Sep 17 00:00:00 2001 From: Dennis Kruyt Date: Thu, 6 Aug 2026 16:08:47 +0200 Subject: [PATCH 1/5] internal/server: add video generation API support (/v1/videos) Add OpenAI/vLLM-omni-compatible video generation routing, capability metadata, and a Playground UI tab. - route POST /v1/videos and /v1/videos/sync (multipart, model in form field) like /v1/audio/transcriptions and /v1/images/edits - route GET/DELETE /v1/videos, /v1/videos/{video_id}, and /v1/videos/{video_id}/content via a required ?model= query param, same convention as /props, since job ids don't carry a model - add "video" as a valid capabilities modality and derive video_understanding/video_generation/image_to_video/video_to_video capability flags for /v1/models - add a Playground "Video" tab (VideoInterface.svelte, videoApi.ts) with sync/async generation, job status polling, and reference image/video upload for image-to-video and video-to-video - update README.md and config.example.yaml docs --- README.md | 5 + config.example.yaml | 4 +- internal/config/model_config.go | 5 +- internal/config/model_config_test.go | 38 ++- internal/server/api.go | 12 + internal/server/api_test.go | 36 ++ internal/server/inflight.go | 6 + internal/server/server.go | 20 ++ internal/server/server_test.go | 69 ++++ .../playground/VideoInterface.svelte | 310 ++++++++++++++++++ ui-svelte/src/lib/types.ts | 22 ++ ui-svelte/src/lib/videoApi.test.ts | 166 ++++++++++ ui-svelte/src/lib/videoApi.ts | 156 +++++++++ ui-svelte/src/routes/Playground.svelte | 2 + ui-svelte/src/stores/playground.ts | 3 +- ui-svelte/src/stores/playgroundActivity.ts | 6 +- 16 files changed, 847 insertions(+), 13 deletions(-) create mode 100644 ui-svelte/src/components/playground/VideoInterface.svelte create mode 100644 ui-svelte/src/lib/videoApi.test.ts create mode 100644 ui-svelte/src/lib/videoApi.ts diff --git a/README.md b/README.md index c58bc100e..cc6bb18ea 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,11 @@ Built in Go for performance and simplicity, llama-swap has zero dependencies and - `v1/audio/voices` - `v1/images/generations` - `v1/images/edits` + - `v1/videos` - create a video generation job ([vLLM-omni video API](https://docs.vllm.ai/projects/vllm-omni/en/latest/serving/videos_api/)) + - `v1/videos/sync` - create a video generation job and wait for the result + - `v1/videos/{video_id}`, `v1/videos/{video_id}/content` - poll status / download a video job + - the model isn't in the request, so these require `?model={model_id}`, same as `/props` + - `DELETE v1/videos/{video_id}` - delete a video job - also requires `?model={model_id}` - ✅ Anthropic API supported endpoints: - `v1/messages` - `v1/messages/count_tokens` diff --git a/config.example.yaml b/config.example.yaml index 30229e97e..9e8d9b944 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -417,14 +417,14 @@ models: capabilities: # in: list of modalities understood by the model # - default: [] - # - valid: text, audio, image + # - valid: text, audio, image, video in: - text - audio - image # out: list of modalities generated by the model # - default: [] - # - valid: text, audio, image + # - valid: text, audio, image, video out: - text - audio diff --git a/internal/config/model_config.go b/internal/config/model_config.go index 3a9408b60..c372296e1 100644 --- a/internal/config/model_config.go +++ b/internal/config/model_config.go @@ -15,6 +15,7 @@ var validModalities = map[string]struct{}{ "text": {}, "audio": {}, "image": {}, + "video": {}, } // ModelCapConfig defines what modalities and features a model supports. @@ -38,12 +39,12 @@ func (c ModelCapConfig) Empty() bool { func (c ModelCapConfig) Validate() error { for _, m := range c.In { if _, ok := validModalities[m]; !ok { - return fmt.Errorf("capabilities.in: invalid modality %q, must be one of: text, audio, image", m) + return fmt.Errorf("capabilities.in: invalid modality %q, must be one of: text, audio, image, video", m) } } for _, m := range c.Out { if _, ok := validModalities[m]; !ok { - return fmt.Errorf("capabilities.out: invalid modality %q, must be one of: text, audio, image", m) + return fmt.Errorf("capabilities.out: invalid modality %q, must be one of: text, audio, image, video", m) } } if c.Context < 0 { diff --git a/internal/config/model_config_test.go b/internal/config/model_config_test.go index 90b103e14..0c7d18400 100644 --- a/internal/config/model_config_test.go +++ b/internal/config/model_config_test.go @@ -264,6 +264,27 @@ models: assert.True(t, mc.Capabilities.Reranker) }) + t.Run("video fields", func(t *testing.T) { + content := ` +models: + model1: + cmd: path/to/cmd --port ${PORT} + capabilities: + in: + - text + - video + out: + - video +` + config, err := LoadConfigFromReader(strings.NewReader(content)) + assert.NoError(t, err) + + mc := config.Models["model1"] + assert.False(t, mc.Capabilities.Empty()) + assert.Equal(t, []string{"text", "video"}, mc.Capabilities.In) + assert.Equal(t, []string{"video"}, mc.Capabilities.Out) + }) + t.Run("reranker false is empty", func(t *testing.T) { content := ` models: @@ -297,19 +318,24 @@ func TestConfig_ModelCapabilities_Validate(t *testing.T) { }) t.Run("invalid_in_modality", func(t *testing.T) { - caps := ModelCapConfig{In: []string{"video"}} + caps := ModelCapConfig{In: []string{"haptic"}} err := caps.Validate() assert.Error(t, err) assert.Contains(t, err.Error(), "capabilities.in") - assert.Contains(t, err.Error(), "video") + assert.Contains(t, err.Error(), "haptic") }) t.Run("invalid_out_modality", func(t *testing.T) { - caps := ModelCapConfig{Out: []string{"video"}} + caps := ModelCapConfig{Out: []string{"haptic"}} err := caps.Validate() assert.Error(t, err) assert.Contains(t, err.Error(), "capabilities.out") - assert.Contains(t, err.Error(), "video") + assert.Contains(t, err.Error(), "haptic") + }) + + t.Run("video_modality_valid", func(t *testing.T) { + caps := ModelCapConfig{In: []string{"video"}, Out: []string{"video"}} + assert.NoError(t, caps.Validate()) }) t.Run("negative_context", func(t *testing.T) { @@ -327,11 +353,11 @@ models: capabilities: in: - text - - video + - haptic ` _, err := LoadConfigFromReader(strings.NewReader(content)) assert.Error(t, err) - assert.Contains(t, err.Error(), "video") + assert.Contains(t, err.Error(), "haptic") }) } diff --git a/internal/server/api.go b/internal/server/api.go index 1c882c28d..0e8072077 100644 --- a/internal/server/api.go +++ b/internal/server/api.go @@ -72,6 +72,9 @@ func renderCapabilities(caps config.ModelCapConfig) (arch map[string]any, capsMa if contains(caps.In, "image") { capsMap["vision"] = true } + if contains(caps.In, "video") { + capsMap["video_understanding"] = true + } } if hasIn && hasOut { if contains(caps.In, "audio") && contains(caps.Out, "text") { @@ -86,6 +89,15 @@ func renderCapabilities(caps config.ModelCapConfig) (arch map[string]any, capsMa if contains(caps.In, "image") && contains(caps.Out, "image") { capsMap["image_to_image"] = true } + if contains(caps.In, "text") && contains(caps.Out, "video") { + capsMap["video_generation"] = true + } + if contains(caps.In, "image") && contains(caps.Out, "video") { + capsMap["image_to_video"] = true + } + if contains(caps.In, "video") && contains(caps.Out, "video") { + capsMap["video_to_video"] = true + } } if caps.Tools { diff --git a/internal/server/api_test.go b/internal/server/api_test.go index bfb741760..91dc83bf8 100644 --- a/internal/server/api_test.go +++ b/internal/server/api_test.go @@ -667,6 +667,42 @@ func TestServer_HandleListModels_Capabilities(t *testing.T) { } }) + t.Run("video_understanding", func(t *testing.T) { + m := getModel(t, newServer(config.ModelConfig{ + Capabilities: config.ModelCapConfig{In: []string{"video"}}, + })) + if m.Capabilities == nil || m.Capabilities["video_understanding"] != true { + t.Error("expected video_understanding: true") + } + }) + + t.Run("video_generation", func(t *testing.T) { + m := getModel(t, newServer(config.ModelConfig{ + Capabilities: config.ModelCapConfig{In: []string{"text"}, Out: []string{"video"}}, + })) + if m.Capabilities == nil || m.Capabilities["video_generation"] != true { + t.Error("expected video_generation: true") + } + }) + + t.Run("image_to_video", func(t *testing.T) { + m := getModel(t, newServer(config.ModelConfig{ + Capabilities: config.ModelCapConfig{In: []string{"image"}, Out: []string{"video"}}, + })) + if m.Capabilities == nil || m.Capabilities["image_to_video"] != true { + t.Error("expected image_to_video: true") + } + }) + + t.Run("video_to_video", func(t *testing.T) { + m := getModel(t, newServer(config.ModelConfig{ + Capabilities: config.ModelCapConfig{In: []string{"video"}, Out: []string{"video"}}, + })) + if m.Capabilities == nil || m.Capabilities["video_to_video"] != true { + t.Error("expected video_to_video: true") + } + }) + t.Run("empty_skip", func(t *testing.T) { m := getModel(t, newServer(config.ModelConfig{})) if m.Architecture != nil { diff --git a/internal/server/inflight.go b/internal/server/inflight.go index 2150d7c13..a050fa16c 100644 --- a/internal/server/inflight.go +++ b/internal/server/inflight.go @@ -433,6 +433,12 @@ func isModelDispatchedRequest(method, path string) bool { return true } } + case http.MethodDelete: + for _, p := range modelDeleteRoutes { + if p == path { + return true + } + } } return false } diff --git a/internal/server/server.go b/internal/server/server.go index b98a1b9bc..fb2aab14c 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -117,6 +117,8 @@ var modelPostJSONRoutes = []string{ var modelPostFormRoutes = []string{ "/v1/audio/transcriptions", "/v1/images/edits", + "/v1/videos", + "/v1/videos/sync", } // modelGetRoutes are model-dispatched GET endpoints (the model arrives as a @@ -125,6 +127,16 @@ var modelGetRoutes = []string{ "/v1/audio/voices", "/sdapi/v1/loras", "/props", + "/v1/videos", + "/v1/videos/{video_id}", + "/v1/videos/{video_id}/content", +} + +// modelDeleteRoutes are model-dispatched DELETE endpoints. Like modelGetRoutes, +// the model arrives as a query parameter since the resource id alone (e.g. a +// video job id) doesn't identify which backend created it. +var modelDeleteRoutes = []string{ + "/v1/videos/{video_id}", } // isMetricsRecordPath reports whether path is one of the model-dispatched @@ -145,6 +157,11 @@ func isMetricsRecordPath(path string) bool { return true } } + for _, p := range modelDeleteRoutes { + if p == path { + return true + } + } return false } @@ -264,6 +281,9 @@ func (s *Server) routes() { for _, path := range modelGetRoutes { mux.Handle("GET "+path, modelChain.Then(dispatch)) } + for _, path := range modelDeleteRoutes { + mux.Handle("DELETE "+path, modelChain.Then(dispatch)) + } // llama-swap API + custom endpoints. mux.Handle("GET /v1/models", apiChain.ThenFunc(s.handleListModels)) diff --git a/internal/server/server_test.go b/internal/server/server_test.go index 2f76632b9..259f11313 100644 --- a/internal/server/server_test.go +++ b/internal/server/server_test.go @@ -1,9 +1,11 @@ package server import ( + "bytes" "context" "encoding/json" "io" + "mime/multipart" "net/http" "net/http/httptest" "strings" @@ -123,6 +125,23 @@ func chatRequest(model string) *http.Request { return req } +// multipartRequest builds a multipart/form-data POST request whose only field +// is "model", matching how /v1/videos, /v1/audio/transcriptions, and +// /v1/images/edits carry the model id. +func multipartRequest(path, model string) *http.Request { + var buf bytes.Buffer + mw := multipart.NewWriter(&buf) + if err := mw.WriteField("model", model); err != nil { + panic(err) + } + if err := mw.Close(); err != nil { + panic(err) + } + req := httptest.NewRequest(http.MethodPost, path, &buf) + req.Header.Set("Content-Type", mw.FormDataContentType()) + return req +} + func TestServer_New_GroupConfig(t *testing.T) { discard := logmon.NewWriter(io.Discard) cfg := config.Config{HealthCheckTimeout: 15} @@ -225,6 +244,56 @@ func TestServer_RouteToLocalModel_PrefersLocalCollision(t *testing.T) { } } +func TestServer_RouteVideoCreate_MultipartModelField(t *testing.T) { + s := newTestServer( + newStubRouter([]string{"local-model"}, `{"id":"video-123","status":"queued"}`), + newStubRouter(nil, ""), + ) + + for _, path := range []string{"/v1/videos", "/v1/videos/sync"} { + w := httptest.NewRecorder() + s.ServeHTTP(w, multipartRequest(path, "local-model")) + + if w.Code != http.StatusOK { + t.Fatalf("%s: status=%d body=%q", path, w.Code, w.Body.String()) + } + } +} + +func TestServer_RouteVideoStatus_QueryParamModel(t *testing.T) { + s := newTestServer( + newStubRouter([]string{"local-model"}, `{"id":"video-123","status":"completed"}`), + newStubRouter(nil, ""), + ) + + for _, path := range []string{ + "/v1/videos?model=local-model", + "/v1/videos/video-123?model=local-model", + "/v1/videos/video-123/content?model=local-model", + } { + w := httptest.NewRecorder() + s.ServeHTTP(w, httptest.NewRequest(http.MethodGet, path, nil)) + + if w.Code != http.StatusOK { + t.Errorf("%s: status=%d body=%q", path, w.Code, w.Body.String()) + } + } +} + +func TestServer_RouteVideoDelete_QueryParamModel(t *testing.T) { + s := newTestServer( + newStubRouter([]string{"local-model"}, ""), + newStubRouter(nil, ""), + ) + + w := httptest.NewRecorder() + s.ServeHTTP(w, httptest.NewRequest(http.MethodDelete, "/v1/videos/video-123?model=local-model", nil)) + + if w.Code != http.StatusOK { + t.Fatalf("status=%d body=%q", w.Code, w.Body.String()) + } +} + func TestServer_UnknownModelReturns404(t *testing.T) { s := newTestServer( newStubRouter([]string{"local-model"}, ""), diff --git a/ui-svelte/src/components/playground/VideoInterface.svelte b/ui-svelte/src/components/playground/VideoInterface.svelte new file mode 100644 index 000000000..65ee118a0 --- /dev/null +++ b/ui-svelte/src/components/playground/VideoInterface.svelte @@ -0,0 +1,310 @@ + + +
+ +
+ + + v && modeStore.set(v as "async" | "sync")}> + {$modeStore === "sync" ? "Sync" : "Async"} + + Async + Sync + + + + v && sizeStore.set(v)}> + {$sizeStore} + + + Landscape + 1280x720 (16:9) + 1920x1080 (16:9) + + + + Portrait + 720x1280 (9:16) + 480x854 (9:16) + + + + Square + 720x720 + + + + + + + +
+ + + {#if !$hasListedModels} + + {:else} + +
+ {#if isGenerating} +
+
+

Generating video{jobStatus ? ` (${jobStatus})` : ""}...

+ {#if $modeStore === "async"} +

Video jobs can take a while to complete.

+ {/if} +
+ {:else if $error} +
+

Error

+

{$error}

+
+ {:else if generatedVideoUrl} +
+
+
+ {#if generatedTimestamp} + {formatTimestamp(generatedTimestamp)} + {/if} +
+ +
+ + +
+ {:else} +
+ +

Enter a prompt below to generate a video

+
+ {/if} +
+ + +
+ {#if referenceFile} +
+ {referenceFile.name} + {formatFileSize(referenceFile.size)} +
+ + {:else} + Optional: drop a reference image or video (image-to-video / video-to-video) + + + {/if} +
+ + +
+ +
+ {#if isGenerating} + + {:else} + + + {/if} +
+
+ {/if} +
diff --git a/ui-svelte/src/lib/types.ts b/ui-svelte/src/lib/types.ts index 949292d3a..f0d39f208 100644 --- a/ui-svelte/src/lib/types.ts +++ b/ui-svelte/src/lib/types.ts @@ -9,6 +9,10 @@ export interface ModelCapabilities { audio_speech?: boolean; image_generation?: boolean; image_to_image?: boolean; + video_understanding?: boolean; + video_generation?: boolean; + image_to_video?: boolean; + video_to_video?: boolean; function_calling?: boolean; reranker?: boolean; } @@ -358,3 +362,21 @@ export interface SpeechGenerationRequest { input: string; voice: string; } + +// vLLM-omni video generation types: https://docs.vllm.ai/projects/vllm-omni/en/latest/serving/videos_api/ +export interface VideoGenerationParams { + size?: string; // "WIDTHxHEIGHT" + seconds?: number; + fps?: number; +} + +// Only "queued" and "completed" are documented; other values pass through +// unrecognized so status handling stays defensive. +export type VideoJobStatus = "queued" | "in_progress" | "completed" | "failed" | (string & {}); + +export interface VideoJob { + id: string; + status: VideoJobStatus; + created_at?: number; + error?: string | { message?: string }; +} diff --git a/ui-svelte/src/lib/videoApi.test.ts b/ui-svelte/src/lib/videoApi.test.ts new file mode 100644 index 000000000..1ffd0cdfe --- /dev/null +++ b/ui-svelte/src/lib/videoApi.test.ts @@ -0,0 +1,166 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { + createVideoJob, + createVideoSync, + deleteVideo, + generateVideoAsync, + getVideoContent, + getVideoStatus, +} from "./videoApi"; +import type { VideoJob } from "./types"; + +afterEach(() => { + vi.unstubAllGlobals(); +}); + +function jsonResponse(body: unknown, ok = true, status = 200) { + return { ok, status, json: async () => body, text: async () => JSON.stringify(body) }; +} + +function blobResponse(blob: Blob, ok = true, status = 200) { + return { ok, status, blob: async () => blob, text: async () => "error" }; +} + +describe("createVideoJob", () => { + it("posts a multipart form to /v1/videos and returns the job", async () => { + const job: VideoJob = { id: "video-1", status: "queued", created_at: 1 }; + const fetchMock = vi.fn().mockResolvedValue(jsonResponse(job)); + vi.stubGlobal("fetch", fetchMock); + + await expect( + createVideoJob("my-model", "a cat riding a skateboard", { size: "1280x720", seconds: 5, fps: 24 }) + ).resolves.toEqual(job); + + expect(fetchMock).toHaveBeenCalledTimes(1); + const [url, init] = fetchMock.mock.calls[0]; + expect(url).toBe("/v1/videos"); + expect(init.method).toBe("POST"); + const body = init.body as FormData; + expect(body.get("model")).toBe("my-model"); + expect(body.get("prompt")).toBe("a cat riding a skateboard"); + expect(body.get("size")).toBe("1280x720"); + expect(body.get("seconds")).toBe("5"); + expect(body.get("fps")).toBe("24"); + expect(body.get("input_reference")).toBeNull(); + }); + + it("includes the reference file when provided", async () => { + const job: VideoJob = { id: "video-1", status: "queued" }; + const fetchMock = vi.fn().mockResolvedValue(jsonResponse(job)); + vi.stubGlobal("fetch", fetchMock); + const file = new File(["data"], "ref.png", { type: "image/png" }); + + await createVideoJob("my-model", "animate this", {}, file); + + const body = fetchMock.mock.calls[0][1].body as FormData; + expect(body.get("input_reference")).toBe(file); + }); + + it("throws with the status and body on a non-ok response", async () => { + vi.stubGlobal("fetch", vi.fn().mockResolvedValue({ ok: false, status: 400, text: async () => "bad request" })); + + await expect(createVideoJob("my-model", "prompt", {})).rejects.toThrow("Video API error: 400 - bad request"); + }); +}); + +describe("createVideoSync", () => { + it("posts to /v1/videos/sync and returns the raw video bytes", async () => { + const blob = new Blob(["bytes"], { type: "video/mp4" }); + const fetchMock = vi.fn().mockResolvedValue(blobResponse(blob)); + vi.stubGlobal("fetch", fetchMock); + + await expect(createVideoSync("my-model", "prompt", { size: "1280x720" })).resolves.toBe(blob); + expect(fetchMock.mock.calls[0][0]).toBe("/v1/videos/sync"); + }); +}); + +describe("getVideoStatus / getVideoContent / deleteVideo", () => { + it("dispatches by a ?model= query param since the job id alone doesn't identify a backend", async () => { + const job: VideoJob = { id: "video-1", status: "completed" }; + const fetchMock = vi.fn().mockResolvedValue(jsonResponse(job)); + vi.stubGlobal("fetch", fetchMock); + + await expect(getVideoStatus("video-1", "my model")).resolves.toEqual(job); + expect(fetchMock).toHaveBeenCalledWith( + "/v1/videos/video-1?model=my%20model", + expect.objectContaining({ cache: "no-store" }) + ); + }); + + it("getVideoContent fetches the content endpoint and returns a blob", async () => { + const blob = new Blob(["bytes"]); + const fetchMock = vi.fn().mockResolvedValue(blobResponse(blob)); + vi.stubGlobal("fetch", fetchMock); + + await expect(getVideoContent("video-1", "my-model")).resolves.toBe(blob); + expect(fetchMock).toHaveBeenCalledWith("/v1/videos/video-1/content?model=my-model", expect.anything()); + }); + + it("deleteVideo issues a DELETE request", async () => { + const fetchMock = vi.fn().mockResolvedValue({ ok: true, status: 200, text: async () => "" }); + vi.stubGlobal("fetch", fetchMock); + + await deleteVideo("video-1", "my-model"); + const [url, init] = fetchMock.mock.calls[0]; + expect(url).toBe("/v1/videos/video-1?model=my-model"); + expect(init.method).toBe("DELETE"); + }); +}); + +describe("generateVideoAsync", () => { + it("polls until completed and returns the rendered video", async () => { + const fetchMock = vi + .fn() + // createVideoJob + .mockResolvedValueOnce(jsonResponse({ id: "video-1", status: "queued" })) + // getVideoStatus (still running) + .mockResolvedValueOnce(jsonResponse({ id: "video-1", status: "in_progress" })) + // getVideoStatus (done) + .mockResolvedValueOnce(jsonResponse({ id: "video-1", status: "completed" })) + // getVideoContent + .mockResolvedValueOnce(blobResponse(new Blob(["bytes"]))); + vi.stubGlobal("fetch", fetchMock); + + const statuses: string[] = []; + const controller = new AbortController(); + const blob = await generateVideoAsync( + "my-model", + "prompt", + {}, + undefined, + controller.signal, + (job) => statuses.push(job.status), + { pollIntervalMs: 0 } + ); + + expect(await blob.text()).toBe("bytes"); + expect(statuses).toEqual(["queued", "in_progress", "completed"]); + expect(fetchMock).toHaveBeenCalledTimes(4); + }); + + it("throws when the job reaches a terminal failure status", async () => { + const fetchMock = vi + .fn() + .mockResolvedValueOnce(jsonResponse({ id: "video-1", status: "queued" })) + .mockResolvedValueOnce(jsonResponse({ id: "video-1", status: "failed", error: "out of memory" })); + vi.stubGlobal("fetch", fetchMock); + + await expect( + generateVideoAsync("my-model", "prompt", {}, undefined, new AbortController().signal, undefined, { + pollIntervalMs: 0, + }) + ).rejects.toThrow("Video generation failed: out of memory"); + }); + + it("times out when the job never reaches a terminal status", async () => { + const fetchMock = vi.fn().mockResolvedValue(jsonResponse({ id: "video-1", status: "queued" })); + vi.stubGlobal("fetch", fetchMock); + + await expect( + generateVideoAsync("my-model", "prompt", {}, undefined, new AbortController().signal, undefined, { + pollIntervalMs: 0, + timeoutMs: -1, + }) + ).rejects.toThrow("Timed out waiting for video generation to complete"); + }); +}); diff --git a/ui-svelte/src/lib/videoApi.ts b/ui-svelte/src/lib/videoApi.ts new file mode 100644 index 000000000..79e392353 --- /dev/null +++ b/ui-svelte/src/lib/videoApi.ts @@ -0,0 +1,156 @@ +import type { VideoGenerationParams, VideoJob } from "./types"; +import { playgroundSessionHeaders } from "./playgroundSession"; + +const TERMINAL_FAILURE_STATUSES = new Set(["failed", "error", "cancelled", "canceled"]); +const DEFAULT_POLL_INTERVAL_MS = 2000; +const DEFAULT_POLL_TIMEOUT_MS = 10 * 60 * 1000; + +function buildVideoFormData( + model: string, + prompt: string, + params: VideoGenerationParams, + referenceFile?: File | null +): FormData { + const formData = new FormData(); + formData.append("model", model); + formData.append("prompt", prompt); + if (params.size) formData.append("size", params.size); + if (params.seconds !== undefined) formData.append("seconds", String(params.seconds)); + if (params.fps !== undefined) formData.append("fps", String(params.fps)); + if (referenceFile) formData.append("input_reference", referenceFile); + return formData; +} + +async function postVideoForm(path: string, formData: FormData, signal?: AbortSignal): Promise { + const response = await fetch(path, { + method: "POST", + headers: playgroundSessionHeaders, + body: formData, + signal, + }); + if (!response.ok) { + const errorText = await response.text(); + throw new Error(`Video API error: ${response.status} - ${errorText}`); + } + return response; +} + +/** POST /v1/videos - creates an async video generation job. */ +export async function createVideoJob( + model: string, + prompt: string, + params: VideoGenerationParams, + referenceFile?: File | null, + signal?: AbortSignal +): Promise { + const response = await postVideoForm("/v1/videos", buildVideoFormData(model, prompt, params, referenceFile), signal); + return response.json(); +} + +/** POST /v1/videos/sync - blocks until the video is generated. */ +export async function createVideoSync( + model: string, + prompt: string, + params: VideoGenerationParams, + referenceFile?: File | null, + signal?: AbortSignal +): Promise { + const response = await postVideoForm("/v1/videos/sync", buildVideoFormData(model, prompt, params, referenceFile), signal); + return response.blob(); +} + +// GET/DELETE /v1/videos/{id}... routes carry no model in the request body, so +// llama-swap dispatches them via a required ?model= query param (same +// convention as /props). + +export async function getVideoStatus(id: string, model: string, signal?: AbortSignal): Promise { + const url = `/v1/videos/${encodeURIComponent(id)}?model=${encodeURIComponent(model)}`; + const response = await fetch(url, { headers: playgroundSessionHeaders, signal, cache: "no-store" }); + if (!response.ok) { + const errorText = await response.text(); + throw new Error(`Video API error: ${response.status} - ${errorText}`); + } + return response.json(); +} + +export async function getVideoContent(id: string, model: string, signal?: AbortSignal): Promise { + const url = `/v1/videos/${encodeURIComponent(id)}/content?model=${encodeURIComponent(model)}`; + const response = await fetch(url, { headers: playgroundSessionHeaders, signal }); + if (!response.ok) { + const errorText = await response.text(); + throw new Error(`Video API error: ${response.status} - ${errorText}`); + } + return response.blob(); +} + +export async function deleteVideo(id: string, model: string, signal?: AbortSignal): Promise { + const url = `/v1/videos/${encodeURIComponent(id)}?model=${encodeURIComponent(model)}`; + const response = await fetch(url, { method: "DELETE", headers: playgroundSessionHeaders, signal }); + if (!response.ok) { + const errorText = await response.text(); + throw new Error(`Video API error: ${response.status} - ${errorText}`); + } +} + +function sleep(ms: number, signal: AbortSignal): Promise { + return new Promise((resolve, reject) => { + if (signal.aborted) { + reject(new DOMException("Aborted", "AbortError")); + return; + } + const timer = setTimeout(resolve, ms); + signal.addEventListener( + "abort", + () => { + clearTimeout(timer); + reject(new DOMException("Aborted", "AbortError")); + }, + { once: true } + ); + }); +} + +function formatJobError(error: VideoJob["error"]): string { + if (!error) return ""; + return typeof error === "string" ? error : (error.message ?? JSON.stringify(error)); +} + +/** + * Creates an async video job (POST /v1/videos) and polls it to completion, + * returning the rendered video bytes. onStatus is invoked after job creation + * and after every poll so callers can render progress. The vLLM-omni docs + * only confirm "queued" and "completed" status values, so polling treats + * "completed" as success, a small set of known failure strings as terminal + * failure, and anything else as still in progress. + */ +export async function generateVideoAsync( + model: string, + prompt: string, + params: VideoGenerationParams, + referenceFile: File | null | undefined, + signal: AbortSignal, + onStatus?: (job: VideoJob) => void, + options?: { pollIntervalMs?: number; timeoutMs?: number } +): Promise { + const pollIntervalMs = options?.pollIntervalMs ?? DEFAULT_POLL_INTERVAL_MS; + const timeoutMs = options?.timeoutMs ?? DEFAULT_POLL_TIMEOUT_MS; + + let current = await createVideoJob(model, prompt, params, referenceFile, signal); + onStatus?.(current); + + const deadline = Date.now() + timeoutMs; + while (current.status !== "completed") { + if (TERMINAL_FAILURE_STATUSES.has(current.status)) { + const detail = formatJobError(current.error); + throw new Error(`Video generation ${current.status}${detail ? `: ${detail}` : ""}`); + } + if (Date.now() > deadline) { + throw new Error("Timed out waiting for video generation to complete"); + } + await sleep(pollIntervalMs, signal); + current = await getVideoStatus(current.id, model, signal); + onStatus?.(current); + } + + return getVideoContent(current.id, model, signal); +} diff --git a/ui-svelte/src/routes/Playground.svelte b/ui-svelte/src/routes/Playground.svelte index 7331d6102..b6271a7e4 100644 --- a/ui-svelte/src/routes/Playground.svelte +++ b/ui-svelte/src/routes/Playground.svelte @@ -4,6 +4,7 @@ import ImageInterface from "../components/playground/ImageInterface.svelte"; import AudioInterface from "../components/playground/AudioInterface.svelte"; import SpeechInterface from "../components/playground/SpeechInterface.svelte"; + import VideoInterface from "../components/playground/VideoInterface.svelte"; import RerankInterface from "../components/playground/RerankInterface.svelte"; import ConcurrencyInterface from "../components/playground/ConcurrencyInterface.svelte"; import * as Card from "$lib/components/ui/card/index.js"; @@ -18,6 +19,7 @@ images: ImageInterface, speech: SpeechInterface, audio: AudioInterface, + video: VideoInterface, rerank: RerankInterface, concurrency: ConcurrencyInterface, }; diff --git a/ui-svelte/src/stores/playground.ts b/ui-svelte/src/stores/playground.ts index dcc9b2ae7..daf2ba251 100644 --- a/ui-svelte/src/stores/playground.ts +++ b/ui-svelte/src/stores/playground.ts @@ -1,12 +1,13 @@ import { persistentStore } from "./persistent"; -export type PlaygroundTab = "chat" | "images" | "speech" | "audio" | "rerank" | "concurrency"; +export type PlaygroundTab = "chat" | "images" | "speech" | "audio" | "video" | "rerank" | "concurrency"; export const playgroundTabs: { id: PlaygroundTab; label: string }[] = [ { id: "chat", label: "Chat" }, { id: "images", label: "Images" }, { id: "speech", label: "Speech" }, { id: "audio", label: "Transcription" }, + { id: "video", label: "Video" }, { id: "rerank", label: "Rerank" }, { id: "concurrency", label: "Load Test" }, ]; diff --git a/ui-svelte/src/stores/playgroundActivity.ts b/ui-svelte/src/stores/playgroundActivity.ts index ad0676844..01f122f55 100644 --- a/ui-svelte/src/stores/playgroundActivity.ts +++ b/ui-svelte/src/stores/playgroundActivity.ts @@ -5,10 +5,11 @@ const imageGenerating = writable(false); const speechGenerating = writable(false); const audioTranscribing = writable(false); const rerankLoading = writable(false); +const videoGenerating = writable(false); export const playgroundActivity = derived( - [chatStreaming, imageGenerating, speechGenerating, audioTranscribing, rerankLoading], - ([$chat, $image, $speech, $audio, $rerank]) => $chat || $image || $speech || $audio || $rerank + [chatStreaming, imageGenerating, speechGenerating, audioTranscribing, rerankLoading, videoGenerating], + ([$chat, $image, $speech, $audio, $rerank, $video]) => $chat || $image || $speech || $audio || $rerank || $video ); export const playgroundStores = { @@ -17,4 +18,5 @@ export const playgroundStores = { speechGenerating, audioTranscribing, rerankLoading, + videoGenerating, }; From 8a4c8bf31e25d35fdc985c7346ad7edca590a748 Mon Sep 17 00:00:00 2001 From: Dennis Kruyt Date: Thu, 6 Aug 2026 17:00:52 +0200 Subject: [PATCH 2/5] ui-svelte: broaden video playground params for backend extension fields Add a negative prompt field and an "Advanced parameters" JSON escape hatch to the Video playground tab, so backend-specific fields (e.g. vLLM-omni's width/height/num_frames/seed/extra_params) can be sent without hardcoding a widget per vendor field. - add negativePrompt/advanced to VideoGenerationParams - flatten params.advanced onto the multipart form in buildVideoFormData, JSON-encoding object values (e.g. extra_params) and overriding same-named basic fields; "model" is never overridable - add a Negative Prompt input and a collapsible Advanced JSON textarea to VideoInterface.svelte, parsed and validated on submit - extend videoApi.test.ts to cover negative_prompt/advanced flattening, extra_params JSON-encoding, and override precedence Verified backend routing needs no changes: internal/shared/http.go's multipart handling only touches the "model" field and passes every other field (and the input_reference file) through untouched. --- .../playground/VideoInterface.svelte | 76 ++++++++++++++++++- ui-svelte/src/lib/types.ts | 5 ++ ui-svelte/src/lib/videoApi.test.ts | 36 +++++++++ ui-svelte/src/lib/videoApi.ts | 16 ++++ 4 files changed, 131 insertions(+), 2 deletions(-) diff --git a/ui-svelte/src/components/playground/VideoInterface.svelte b/ui-svelte/src/components/playground/VideoInterface.svelte index 65ee118a0..c19f69ebc 100644 --- a/ui-svelte/src/components/playground/VideoInterface.svelte +++ b/ui-svelte/src/components/playground/VideoInterface.svelte @@ -10,8 +10,10 @@ import type { VideoJob } from "../../lib/types"; import { Button } from "$lib/components/ui/button/index.js"; import { Input } from "$lib/components/ui/input/index.js"; + import { Textarea } from "$lib/components/ui/textarea/index.js"; import * as Select from "$lib/components/ui/select/index.js"; - import { Download, X, Video as VideoIcon } from "@lucide/svelte"; + import * as Collapsible from "$lib/components/ui/collapsible/index.js"; + import { Download, X, Video as VideoIcon, ChevronRight } from "@lucide/svelte"; import { formatFileSize } from "../../lib/format"; const iface = createPlaygroundInterface("playground-video-model", playgroundStores.videoGenerating); @@ -22,9 +24,12 @@ const secondsStore = persistentStore("playground-video-seconds", 5); const fpsStore = persistentStore("playground-video-fps", 24); const modeStore = persistentStore<"async" | "sync">("playground-video-mode", "async"); + const negativePromptStore = persistentStore("playground-video-negative-prompt", ""); + const advancedStore = persistentStore("playground-video-advanced", ""); let prompt = $state(""); let isGenerating = $derived($busyStore); + let advancedOpen = $state(false); let referenceFile = $state(null); let isDragging = $state(false); let fileInput = $state(null); @@ -82,13 +87,42 @@ if (fileInput) fileInput.value = ""; } + // Parses the Advanced JSON box into a plain object, or returns an error + // message. An empty box is valid (no advanced overrides). + function parseAdvanced(): { value?: Record; error?: string } { + const raw = $advancedStore.trim(); + if (!raw) return { value: undefined }; + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch { + return { error: "Advanced parameters must be valid JSON." }; + } + if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) { + return { error: "Advanced parameters must be a JSON object, e.g. {\"width\": 832}." }; + } + return { value: parsed as Record }; + } + async function generate() { const trimmedPrompt = prompt.trim(); if (!trimmedPrompt || !$selectedModelStore || isGenerating) return; + const { value: advanced, error: advancedError } = parseAdvanced(); + if (advancedError) { + $error = advancedError; + return; + } + jobStatus = null; await iface.run(async (signal) => { - const params = { size: $sizeStore, seconds: $secondsStore, fps: $fpsStore }; + const params = { + size: $sizeStore, + seconds: $secondsStore, + fps: $fpsStore, + negativePrompt: $negativePromptStore.trim() || undefined, + advanced, + }; const videoBlob = $modeStore === "sync" @@ -280,6 +314,44 @@ {/if} + + + + + (advancedOpen = v)} class="shrink-0 mb-4"> + + + Advanced parameters + + +
+