ui-svelte,internal/server: add video generation API support (/v1/videos)

README.md

@@ -26,6 +26,11 @@ Built in Go for performance and simplicity, llama-swap has zero dependencies and
   - `v1/audio/voices`
   - `v1/images/generations`
   - `v1/images/edits`
+  - `v1/videos` - create a video generation job ([vLLM-omni video API](https://docs.vllm.ai/projects/vllm-omni/en/latest/serving/videos_api/))
+  - `v1/videos/sync` - create a video generation job and wait for the result
+  - `v1/videos/{video_id}`, `v1/videos/{video_id}/content` - poll status / download a video job
+    - the model isn't in the request, so these require `?model={model_id}`, same as `/props`
+  - `DELETE v1/videos/{video_id}` - delete a video job - also requires `?model={model_id}`
 - ✅ Anthropic API supported endpoints:
   - `v1/messages`
   - `v1/messages/count_tokens`

config-schema.json

@@ -561,7 +561,8 @@
                                     "enum": [
                                         "text",
                                         "audio",
-                                        "image"
+                                        "image",
+                                        "video"
                                     ]
                                 },
                                 "description": "List of input modalities understood by the model."
@@ -576,7 +577,8 @@
                                     "enum": [
                                         "text",
                                         "audio",
-                                        "image"
+                                        "image",
+                                        "video"
                                     ]
                                 },
                                 "description": "List of output modalities generated by the model."

config.example.yaml

@@ -417,14 +417,14 @@ models:
     capabilities:
       # in: list of modalities understood by the model
       # - default: []
-      # - valid: text, audio, image
+      # - valid: text, audio, image, video
       in:
         - text
         - audio
         - image
       # out: list of modalities generated by the model
       # - default: []
-      # - valid: text, audio, image
+      # - valid: text, audio, image, video
       out:
         - text
         - audio
@@ -473,6 +473,29 @@ models:
     # - processes have 5 seconds to shutdown until forceful termination is attempted
     cmdStop: docker stop ${MODEL_ID}
 
+  # Video generation example (vLLM-omni):
+  # any backend that implements vLLM-omni's OpenAI-compatible video API
+  # (POST /v1/videos, /v1/videos/sync) works with no special configuration -
+  # llama-swap only reads the "model" field from the request and passes
+  # every other field straight through, so backend-specific fields (e.g.
+  # width/height/num_frames/seed/extra_params) are handled automatically.
+  # See: https://docs.vllm.ai/projects/vllm-omni/en/latest/serving/videos_api/
+  "wan2.2-ti2v-5b":
+    description: "Wan2.2-TI2V-5B (Wan-AI/Wan2.2-TI2V-5B-Diffusers) - text/image-to-video via vLLM-omni"
+    capabilities:
+      in: [text, image]
+      out: [video]
+    proxy: "http://127.0.0.1:${PORT}"
+    checkEndpoint: /health
+    cmd: |
+      docker run --name ${MODEL_ID} --rm --gpus all --ipc=host
+      -p 127.0.0.1:${PORT}:8000 -v /models:/models:ro
+      vllm/vllm-omni:latest
+      vllm serve /models/Wan-AI/Wan2.2-TI2V-5B-Diffusers
+      --omni --host 0.0.0.0 --port 8000 --trust-remote-code
+      --served-model-name wan2.2-ti2v-5b
+    cmdStop: docker stop ${MODEL_ID}
+
 # hooks: a dictionary of event triggers and actions
 # - optional, default: empty dictionary
 # - the only supported hook is on_startup

internal/config/model_config.go

@@ -15,6 +15,7 @@ var validModalities = map[string]struct{}{
 	"text":  {},
 	"audio": {},
 	"image": {},
+	"video": {},
 }
 
 // ModelCapConfig defines what modalities and features a model supports.
@@ -38,12 +39,12 @@ func (c ModelCapConfig) Empty() bool {
 func (c ModelCapConfig) Validate() error {
 	for _, m := range c.In {
 		if _, ok := validModalities[m]; !ok {
-			return fmt.Errorf("capabilities.in: invalid modality %q, must be one of: text, audio, image", m)
+			return fmt.Errorf("capabilities.in: invalid modality %q, must be one of: text, audio, image, video", m)
 		}
 	}
 	for _, m := range c.Out {
 		if _, ok := validModalities[m]; !ok {
-			return fmt.Errorf("capabilities.out: invalid modality %q, must be one of: text, audio, image", m)
+			return fmt.Errorf("capabilities.out: invalid modality %q, must be one of: text, audio, image, video", m)
 		}
 	}
 	if c.Context < 0 {

internal/config/model_config_test.go

@@ -264,6 +264,27 @@ models:
 		assert.True(t, mc.Capabilities.Reranker)
 	})
 
+	t.Run("video fields", func(t *testing.T) {
+		content := `
+models:
+  model1:
+    cmd: path/to/cmd --port ${PORT}
+    capabilities:
+      in:
+        - text
+        - video
+      out:
+        - video
+`
+		config, err := LoadConfigFromReader(strings.NewReader(content))
+		assert.NoError(t, err)
+
+		mc := config.Models["model1"]
+		assert.False(t, mc.Capabilities.Empty())
+		assert.Equal(t, []string{"text", "video"}, mc.Capabilities.In)
+		assert.Equal(t, []string{"video"}, mc.Capabilities.Out)
+	})
+
 	t.Run("reranker false is empty", func(t *testing.T) {
 		content := `
 models:
@@ -297,19 +318,24 @@ func TestConfig_ModelCapabilities_Validate(t *testing.T) {
 	})
 
 	t.Run("invalid_in_modality", func(t *testing.T) {
-		caps := ModelCapConfig{In: []string{"video"}}
+		caps := ModelCapConfig{In: []string{"haptic"}}
 		err := caps.Validate()
 		assert.Error(t, err)
 		assert.Contains(t, err.Error(), "capabilities.in")
-		assert.Contains(t, err.Error(), "video")
+		assert.Contains(t, err.Error(), "haptic")
 	})
 
 	t.Run("invalid_out_modality", func(t *testing.T) {
-		caps := ModelCapConfig{Out: []string{"video"}}
+		caps := ModelCapConfig{Out: []string{"haptic"}}
 		err := caps.Validate()
 		assert.Error(t, err)
 		assert.Contains(t, err.Error(), "capabilities.out")
-		assert.Contains(t, err.Error(), "video")
+		assert.Contains(t, err.Error(), "haptic")
+	})
+
+	t.Run("video_modality_valid", func(t *testing.T) {
+		caps := ModelCapConfig{In: []string{"video"}, Out: []string{"video"}}
+		assert.NoError(t, caps.Validate())
 	})
 
 	t.Run("negative_context", func(t *testing.T) {
@@ -327,11 +353,11 @@ models:
     capabilities:
       in:
         - text
-        - video
+        - haptic
 `
 		_, err := LoadConfigFromReader(strings.NewReader(content))
 		assert.Error(t, err)
-		assert.Contains(t, err.Error(), "video")
+		assert.Contains(t, err.Error(), "haptic")
 	})
 }
 

internal/server/api.go

@@ -72,6 +72,9 @@ func renderCapabilities(caps config.ModelCapConfig) (arch map[string]any, capsMa
 		if contains(caps.In, "image") {
 			capsMap["vision"] = true
 		}
+		if contains(caps.In, "video") {
+			capsMap["video_understanding"] = true
+		}
 	}
 	if hasIn && hasOut {
 		if contains(caps.In, "audio") && contains(caps.Out, "text") {
@@ -86,6 +89,15 @@ func renderCapabilities(caps config.ModelCapConfig) (arch map[string]any, capsMa
 		if contains(caps.In, "image") && contains(caps.Out, "image") {
 			capsMap["image_to_image"] = true
 		}
+		if contains(caps.In, "text") && contains(caps.Out, "video") {
+			capsMap["video_generation"] = true
+		}
+		if contains(caps.In, "image") && contains(caps.Out, "video") {
+			capsMap["image_to_video"] = true
+		}
+		if contains(caps.In, "video") && contains(caps.Out, "video") {
+			capsMap["video_to_video"] = true
+		}
 	}
 
 	if caps.Tools {

internal/server/api_test.go

@@ -667,6 +667,42 @@ func TestServer_HandleListModels_Capabilities(t *testing.T) {
 		}
 	})
 
+	t.Run("video_understanding", func(t *testing.T) {
+		m := getModel(t, newServer(config.ModelConfig{
+			Capabilities: config.ModelCapConfig{In: []string{"video"}},
+		}))
+		if m.Capabilities == nil || m.Capabilities["video_understanding"] != true {
+			t.Error("expected video_understanding: true")
+		}
+	})
+
+	t.Run("video_generation", func(t *testing.T) {
+		m := getModel(t, newServer(config.ModelConfig{
+			Capabilities: config.ModelCapConfig{In: []string{"text"}, Out: []string{"video"}},
+		}))
+		if m.Capabilities == nil || m.Capabilities["video_generation"] != true {
+			t.Error("expected video_generation: true")
+		}
+	})
+
+	t.Run("image_to_video", func(t *testing.T) {
+		m := getModel(t, newServer(config.ModelConfig{
+			Capabilities: config.ModelCapConfig{In: []string{"image"}, Out: []string{"video"}},
+		}))
+		if m.Capabilities == nil || m.Capabilities["image_to_video"] != true {
+			t.Error("expected image_to_video: true")
+		}
+	})
+
+	t.Run("video_to_video", func(t *testing.T) {
+		m := getModel(t, newServer(config.ModelConfig{
+			Capabilities: config.ModelCapConfig{In: []string{"video"}, Out: []string{"video"}},
+		}))
+		if m.Capabilities == nil || m.Capabilities["video_to_video"] != true {
+			t.Error("expected video_to_video: true")
+		}
+	})
+
 	t.Run("empty_skip", func(t *testing.T) {
 		m := getModel(t, newServer(config.ModelConfig{}))
 		if m.Architecture != nil {

internal/server/inflight.go

@@ -433,6 +433,12 @@ func isModelDispatchedRequest(method, path string) bool {
 				return true
 			}
 		}
+	case http.MethodDelete:
+		for _, p := range modelDeleteRoutes {
+			if p == path {
+				return true
+			}
+		}
 	}
 	return false
 }

internal/server/profiles_test.go

@@ -125,6 +125,37 @@ func TestServer_ProfileMiddleware_JSONAndFilters(t *testing.T) {
 	assert.Equal(t, http.StatusNotFound, w.Code)
 }
 
+// DELETE /v1/videos/{video_id} carries its model in the query, so a profile pin
+// has to rewrite the query parameter rather than attach a form body.
+func TestServer_ProfileMiddleware_RewriteDeleteQueryModel(t *testing.T) {
+	cfg := profileTestConfig(t)
+	local := newStubRouter([]string{"real", "hidden"}, "")
+	var gotQueryModel string
+	var gotModelID string
+	var gotContentLength int64
+	var gotBody []byte
+	local.serveHTTP = func(w http.ResponseWriter, r *http.Request) {
+		gotQueryModel = r.URL.Query().Get("model")
+		gotContentLength = r.ContentLength
+		gotBody, _ = io.ReadAll(r.Body)
+		data, _ := shared.ReadContext(r.Context())
+		gotModelID = data.ModelID
+		w.WriteHeader(http.StatusOK)
+	}
+	s := profileTestServer(t, cfg, local)
+	_, err := s.setActiveProfile("coding")
+	require.NoError(t, err)
+
+	w := httptest.NewRecorder()
+	s.ServeHTTP(w, httptest.NewRequest(http.MethodDelete, "/v1/videos/video-123?model=public", nil))
+	require.Equal(t, http.StatusOK, w.Code, w.Body.String())
+
+	assert.Equal(t, "variant", gotQueryModel)
+	assert.Equal(t, "real", gotModelID)
+	assert.Zero(t, gotContentLength)
+	assert.Empty(t, gotBody)
+}
+
 func TestServer_Profile_UpstreamPreservesBody(t *testing.T) {
 	cfg := profileTestConfig(t)
 	local := newStubRouter([]string{"real", "hidden"}, "")

internal/server/selector_test.go

@@ -224,6 +224,36 @@ func TestServer_SelectorMiddleware_RewritesBeforeFiltersAndRecordsActivity(t *te
 	assert.Equal(t, "public", entries[0].Metadata["selector"])
 }
 
+// DELETE /v1/videos/{video_id} identifies its backend with a ?model= query
+// parameter, so the selector rewrite must land in the query and leave the
+// bodyless request alone.
+func TestServer_SelectorMiddleware_RewriteDeleteQueryModel(t *testing.T) {
+	cfg := selectorTestConfig(t)
+	local := newStubRouter([]string{"a", "b", "c"}, "")
+	var received shared.ReqContextData
+	var gotQueryModel string
+	var gotContentLength int64
+	var gotBody []byte
+	local.serveHTTP = func(w http.ResponseWriter, r *http.Request) {
+		received, _ = shared.ReadContext(r.Context())
+		gotQueryModel = r.URL.Query().Get("model")
+		gotContentLength = r.ContentLength
+		gotBody, _ = io.ReadAll(r.Body)
+		w.WriteHeader(http.StatusOK)
+	}
+	s := selectorTestServer(t, cfg, local)
+
+	w := httptest.NewRecorder()
+	s.ServeHTTP(w, httptest.NewRequest(http.MethodDelete, "/v1/videos/video-123?model=public", nil))
+	require.Equal(t, http.StatusOK, w.Code, w.Body.String())
+
+	assert.Equal(t, "variant", gotQueryModel)
+	assert.Equal(t, "variant", received.Model)
+	assert.Equal(t, "a", received.ModelID)
+	assert.Zero(t, gotContentLength)
+	assert.Empty(t, gotBody)
+}
+
 func TestServer_SelectorMiddleware_ProfileRunsFirst(t *testing.T) {
 	cfg := selectorTestConfig(t)
 	local := newStubRouter([]string{"a", "b", "c"}, "")

internal/server/server.go

@@ -117,6 +117,8 @@ var modelPostJSONRoutes = []string{
 var modelPostFormRoutes = []string{
 	"/v1/audio/transcriptions",
 	"/v1/images/edits",
+	"/v1/videos",
+	"/v1/videos/sync",
 }
 
 // modelGetRoutes are model-dispatched GET endpoints (the model arrives as a
@@ -125,6 +127,16 @@ var modelGetRoutes = []string{
 	"/v1/audio/voices",
 	"/sdapi/v1/loras",
 	"/props",
+	"/v1/videos",
+	"/v1/videos/{video_id}",
+	"/v1/videos/{video_id}/content",
+}
+
+// modelDeleteRoutes are model-dispatched DELETE endpoints. Like modelGetRoutes,
+// the model arrives as a query parameter since the resource id alone (e.g. a
+// video job id) doesn't identify which backend created it.
+var modelDeleteRoutes = []string{
+	"/v1/videos/{video_id}",
 }
 
 // isMetricsRecordPath reports whether path is one of the model-dispatched
@@ -145,6 +157,11 @@ func isMetricsRecordPath(path string) bool {
 			return true
 		}
 	}
+	for _, p := range modelDeleteRoutes {
+		if p == path {
+			return true
+		}
+	}
 	return false
 }
 
@@ -264,6 +281,9 @@ func (s *Server) routes() {
 	for _, path := range modelGetRoutes {
 		mux.Handle("GET "+path, modelChain.Then(dispatch))
 	}
+	for _, path := range modelDeleteRoutes {
+		mux.Handle("DELETE "+path, modelChain.Then(dispatch))
+	}
 
 	// llama-swap API + custom endpoints.
 	mux.Handle("GET /v1/models", apiChain.ThenFunc(s.handleListModels))