From ad91766db3879d932bfd34557169508b211213be Mon Sep 17 00:00:00 2001 From: Adele Date: Thu, 15 Oct 2026 13:24:39 -0700 Subject: [PATCH] fix(vllm): bound output limit instead of reserving whole context window --- src/plugin/commands.ts | 2 +- src/utils/model-info/vllm.ts | 13 +++++++++---- test/core/adapter-parity.test.ts | 6 +++--- test/core/model-parity.test.ts | 2 +- test/plugin.test.ts | 6 +++--- test/v2/model-mapper.test.ts | 2 +- test/vllm-model-info.test.ts | 15 +++++++++++++++ 7 files changed, 33 insertions(+), 13 deletions(-) diff --git a/src/plugin/commands.ts b/src/plugin/commands.ts index 8d7680b8..2b16f642 100644 --- a/src/plugin/commands.ts +++ b/src/plugin/commands.ts @@ -122,7 +122,7 @@ Supported plugin options under provider..options.modelsDiscovery: - modelInfoFormat="models.dev": enrich from the public models.dev index; modelInfoEndpoint optionally overrides the complete models.json URL - modelInfoFormat="bifrost": read Bifrost's documented inline /v1/models limits, modalities, and base pricing without another request - modelInfoFormat="litellm": enrich from a LiteLLM-compatible /v1/model/info endpoint; modelInfoEndpoint optionally overrides the path -- modelInfoFormat="vllm": for vLLM-compatible providers whose raw /v1/models entries include a positive numeric max_model_len; without another request, sets limit.context and limit.output for matching discovered models +- modelInfoFormat="vllm": for vLLM-compatible providers whose raw /v1/models entries include a positive numeric max_model_len; without another request, sets limit.context and bounds limit.output to the plugin's default output limit for matching discovered models - modelInfoFormat="lmstudio": discover callable models through the normal /v1/models endpoint, then enrich exact model-id matches from LM Studio's /api/v1/models inventory - modelInfoFormat="llama-swap": read llama-swap's inline /v1/models context, modalities, and function-calling metadata without another request - modelInfoFormat="omniroute": read OmniRoute's documented inline /v1/models limits, modalities, and capabilities without another request diff --git a/src/utils/model-info/vllm.ts b/src/utils/model-info/vllm.ts index d8a20409..3df3bbae 100644 --- a/src/utils/model-info/vllm.ts +++ b/src/utils/model-info/vllm.ts @@ -1,4 +1,5 @@ import type { ModelEnricher } from '../../core/model-enrichment' +import { createModelLimits } from '../../core/model-types' function hasUsableNumber(value: unknown): value is number { return typeof value === 'number' && Number.isFinite(value) && value > 0 @@ -9,10 +10,14 @@ export function createVLLMEnricher(_data: unknown): ModelEnricher { enrich(model) { const maxModelLen = model.max_model_len if (hasUsableNumber(maxModelLen)) { - return { limit: { - context: maxModelLen, - output: maxModelLen, - } } + // vLLM reports only max_model_len, which describes the whole window + // shared by input and output. Treat it as the context limit and let + // createModelLimits bound the output limit to the default token + // limit, matching the LM Studio enricher, so hosts do not reserve + // the entire context for output tokens and reject requests that + // exceed it after token accounting. + const limits = createModelLimits(maxModelLen) + if (limits) return { limit: limits } } return {} }, diff --git a/test/core/adapter-parity.test.ts b/test/core/adapter-parity.test.ts index 6c34f41a..b239f8f3 100644 --- a/test/core/adapter-parity.test.ts +++ b/test/core/adapter-parity.test.ts @@ -116,11 +116,11 @@ describe('end-to-end V1 and V2 adapter projection parity', () => { // smartModelName: false preserves exact raw ID as name expect(v1.name).toBe('qwen/qwen-2.5-72b-instruct') expect(v2.name).toBe('qwen/qwen-2.5-72b-instruct') - // vLLM max_model_len sets context and bounded output + // vLLM max_model_len sets context and output bounded by the default token limit expect(v1.limit?.context).toBe(32768) - expect(v1.limit?.output).toBe(32768) + expect(v1.limit?.output).toBe(32000) expect(v2.limit.context).toBe(32768) - expect(v2.limit.output).toBe(32768) + expect(v2.limit.output).toBe(32000) }) it('guarantees identical modalities, reasoning, and tools for fully-enriched models', () => { diff --git a/test/core/model-parity.test.ts b/test/core/model-parity.test.ts index eadd8d26..3691a77b 100644 --- a/test/core/model-parity.test.ts +++ b/test/core/model-parity.test.ts @@ -138,7 +138,7 @@ describe('V1/V2 shared-core parity', () => { const draft = drafts[0]! const mapped = mapToDiscoveredV2Model(draft, options) - expect(draft.limit).toEqual({ context: 32_768, output: 32_768 }) + expect(draft.limit).toEqual({ context: 32_768, output: 32_000 }) expect(mapped.limit).toEqual(draft.limit) }) diff --git a/test/plugin.test.ts b/test/plugin.test.ts index 51aa1857..cb0303c6 100644 --- a/test/plugin.test.ts +++ b/test/plugin.test.ts @@ -616,7 +616,7 @@ describe('ModelDiscovery Plugin', () => { expect(config.provider.test.models['chat-model']).toEqual({ id: 'chat-model', name: 'Chat Model', - limit: { context: 32768, output: 32768 }, + limit: { context: 32768, output: 32000 }, modalities: { input: ['text'], output: ['text'] }, variants: { custom: { reasoningEffort: 'high' } }, }) @@ -660,7 +660,7 @@ describe('ModelDiscovery Plugin', () => { expect(state?.models).toEqual({ 'chat-model': expect.objectContaining({ id: 'chat-model', - limit: { context: 32768, output: 32768 }, + limit: { context: 32768, output: 32000 }, }), }) expect(state?.models['embedding-model']).toBeUndefined() @@ -680,7 +680,7 @@ describe('ModelDiscovery Plugin', () => { mockFetch.mockClear() await pluginHooks.config(secondConfig) - expect(secondConfig.provider.cached.models['chat-model'].limit).toEqual({ context: 32768, output: 32768 }) + expect(secondConfig.provider.cached.models['chat-model'].limit).toEqual({ context: 32768, output: 32000 }) expect(mockFetch).not.toHaveBeenCalled() }) diff --git a/test/v2/model-mapper.test.ts b/test/v2/model-mapper.test.ts index e661fa31..efe80145 100644 --- a/test/v2/model-mapper.test.ts +++ b/test/v2/model-mapper.test.ts @@ -58,7 +58,7 @@ describe("V2 model-mapper", () => { const enricher = createModelEnricher(ModelInfoFormat.VLLM, null) const mapped = mapToDiscoveredV2Model(discover({ id: "qwen-2.5-72b", max_model_len: 32_768 }, true, enricher), options) expect(mapped.limit.context).toBe(32_768) - expect(mapped.limit.output).toBe(32_768) + expect(mapped.limit.output).toBe(32_000) }) it("enriches limits and modalities from litellm model info endpoint", () => { diff --git a/test/vllm-model-info.test.ts b/test/vllm-model-info.test.ts index 17ea8ca2..c0b45ef3 100644 --- a/test/vllm-model-info.test.ts +++ b/test/vllm-model-info.test.ts @@ -17,6 +17,21 @@ describe('vLLM model info enricher', () => { }) }) + it('bounds output to the default token limit for large max_model_len', () => { + const rawModel = { + id: 'test-model', + object: 'model', + created: 1, + owned_by: 'vllm', + max_model_len: 250000, + } + + expect(createVLLMEnricher(null).enrich(rawModel, { filterNonChat: true }).limit).toEqual({ + context: 250000, + output: 32000, + }) + }) + it('does not set limit when max_model_len is missing', () => { expect(createVLLMEnricher(null).enrich({ id: 'test-model', object: 'model', created: 1, owned_by: 'llama.cpp' }, { filterNonChat: true }).limit).toBeUndefined() })