fix(vllm): bound output limit instead of reserving whole context window

src/plugin/commands.ts

@@ -122,7 +122,7 @@ Supported plugin options under provider.<id>.options.modelsDiscovery:
 - modelInfoFormat="models.dev": enrich from the public models.dev index; modelInfoEndpoint optionally overrides the complete models.json URL
 - modelInfoFormat="bifrost": read Bifrost's documented inline /v1/models limits, modalities, and base pricing without another request
 - modelInfoFormat="litellm": enrich from a LiteLLM-compatible /v1/model/info endpoint; modelInfoEndpoint optionally overrides the path
-- modelInfoFormat="vllm": for vLLM-compatible providers whose raw /v1/models entries include a positive numeric max_model_len; without another request, sets limit.context and limit.output for matching discovered models
+- modelInfoFormat="vllm": for vLLM-compatible providers whose raw /v1/models entries include a positive numeric max_model_len; without another request, sets limit.context and bounds limit.output to the plugin's default output limit for matching discovered models
 - modelInfoFormat="lmstudio": discover callable models through the normal /v1/models endpoint, then enrich exact model-id matches from LM Studio's /api/v1/models inventory
 - modelInfoFormat="llama-swap": read llama-swap's inline /v1/models context, modalities, and function-calling metadata without another request
 - modelInfoFormat="omniroute": read OmniRoute's documented inline /v1/models limits, modalities, and capabilities without another request

src/utils/model-info/vllm.ts

@@ -1,4 +1,5 @@
 import type { ModelEnricher } from '../../core/model-enrichment'
+import { createModelLimits } from '../../core/model-types'
 
 function hasUsableNumber(value: unknown): value is number {
   return typeof value === 'number' && Number.isFinite(value) && value > 0
@@ -9,10 +10,14 @@ export function createVLLMEnricher(_data: unknown): ModelEnricher {
     enrich(model) {
       const maxModelLen = model.max_model_len
       if (hasUsableNumber(maxModelLen)) {
-        return { limit: {
-          context: maxModelLen,
-          output: maxModelLen,
-        } }
+        // vLLM reports only max_model_len, which describes the whole window
+        // shared by input and output. Treat it as the context limit and let
+        // createModelLimits bound the output limit to the default token
+        // limit, matching the LM Studio enricher, so hosts do not reserve
+        // the entire context for output tokens and reject requests that
+        // exceed it after token accounting.
+        const limits = createModelLimits(maxModelLen)
+        if (limits) return { limit: limits }
       }
       return {}
     },

test/core/adapter-parity.test.ts

@@ -116,11 +116,11 @@ describe('end-to-end V1 and V2 adapter projection parity', () => {
     // smartModelName: false preserves exact raw ID as name
     expect(v1.name).toBe('qwen/qwen-2.5-72b-instruct')
     expect(v2.name).toBe('qwen/qwen-2.5-72b-instruct')
-    // vLLM max_model_len sets context and bounded output
+    // vLLM max_model_len sets context and output bounded by the default token limit
     expect(v1.limit?.context).toBe(32768)
-    expect(v1.limit?.output).toBe(32768)
+    expect(v1.limit?.output).toBe(32000)
     expect(v2.limit.context).toBe(32768)
-    expect(v2.limit.output).toBe(32768)
+    expect(v2.limit.output).toBe(32000)
   })
 
   it('guarantees identical modalities, reasoning, and tools for fully-enriched models', () => {

test/core/model-parity.test.ts

@@ -138,7 +138,7 @@ describe('V1/V2 shared-core parity', () => {
     const draft = drafts[0]!
     const mapped = mapToDiscoveredV2Model(draft, options)
 
-    expect(draft.limit).toEqual({ context: 32_768, output: 32_768 })
+    expect(draft.limit).toEqual({ context: 32_768, output: 32_000 })
     expect(mapped.limit).toEqual(draft.limit)
   })
 

test/plugin.test.ts

@@ -616,7 +616,7 @@ describe('ModelDiscovery Plugin', () => {
       expect(config.provider.test.models['chat-model']).toEqual({
         id: 'chat-model',
         name: 'Chat Model',
-        limit: { context: 32768, output: 32768 },
+        limit: { context: 32768, output: 32000 },
         modalities: { input: ['text'], output: ['text'] },
         variants: { custom: { reasoningEffort: 'high' } },
       })
@@ -660,7 +660,7 @@ describe('ModelDiscovery Plugin', () => {
       expect(state?.models).toEqual({
         'chat-model': expect.objectContaining({
           id: 'chat-model',
-          limit: { context: 32768, output: 32768 },
+          limit: { context: 32768, output: 32000 },
         }),
       })
       expect(state?.models['embedding-model']).toBeUndefined()
@@ -680,7 +680,7 @@ describe('ModelDiscovery Plugin', () => {
       mockFetch.mockClear()
       await pluginHooks.config(secondConfig)
 
-      expect(secondConfig.provider.cached.models['chat-model'].limit).toEqual({ context: 32768, output: 32768 })
+      expect(secondConfig.provider.cached.models['chat-model'].limit).toEqual({ context: 32768, output: 32000 })
       expect(mockFetch).not.toHaveBeenCalled()
     })
 

test/v2/model-mapper.test.ts

@@ -58,7 +58,7 @@ describe("V2 model-mapper", () => {
     const enricher = createModelEnricher(ModelInfoFormat.VLLM, null)
     const mapped = mapToDiscoveredV2Model(discover({ id: "qwen-2.5-72b", max_model_len: 32_768 }, true, enricher), options)
     expect(mapped.limit.context).toBe(32_768)
-    expect(mapped.limit.output).toBe(32_768)
+    expect(mapped.limit.output).toBe(32_000)
   })
 
   it("enriches limits and modalities from litellm model info endpoint", () => {

test/vllm-model-info.test.ts

@@ -17,6 +17,21 @@ describe('vLLM model info enricher', () => {
     })
   })
 
+  it('bounds output to the default token limit for large max_model_len', () => {
+    const rawModel = {
+      id: 'test-model',
+      object: 'model',
+      created: 1,
+      owned_by: 'vllm',
+      max_model_len: 250000,
+    }
+
+    expect(createVLLMEnricher(null).enrich(rawModel, { filterNonChat: true }).limit).toEqual({
+      context: 250000,
+      output: 32000,
+    })
+  })
+
   it('does not set limit when max_model_len is missing', () => {
     expect(createVLLMEnricher(null).enrich({ id: 'test-model', object: 'model', created: 1, owned_by: 'llama.cpp' }, { filterNonChat: true }).limit).toBeUndefined()
   })