diff --git a/changelog.d/fixes/10284-reasoning-probe-truncated-200.md b/changelog.d/fixes/10284-reasoning-probe-truncated-200.md new file mode 100644 index 00000000000..c3ddd311d27 --- /dev/null +++ b/changelog.d/fixes/10284-reasoning-probe-truncated-200.md @@ -0,0 +1 @@ +- **fix(sse):** tiny-budget reasoning probes (e.g. Claude Code's `/model` check sends `max_tokens: 1`) are answered with a valid truncated 200 instead of relaying the upstream 5xx "empty response content" — which previously also marked the connection unavailable and poisoned fallback/cooldown bookkeeping for a request that is only a probe ([#10281](https://github.com/diegosouzapw/OmniRoute/issues/10281)) — thanks @harkaranbrar7 diff --git a/open-sse/handlers/chatCore.ts b/open-sse/handlers/chatCore.ts index 35e579a45d1..72ae89292ee 100644 --- a/open-sse/handlers/chatCore.ts +++ b/open-sse/handlers/chatCore.ts @@ -159,7 +159,13 @@ import { buildCapabilityMismatchMessage, } from "@/shared/constants/capabilities/capabilityFilter.ts"; import { isFeatureFlagEnabled } from "@/shared/utils/featureFlags.ts"; -import { toPositiveInteger } from "../services/reasoningTokenBuffer.ts"; +import { + REASONING_BUFFER_MIN_TRIGGER, + buildReasoningProbeTruncatedResponse, + isEmptyContentUpstreamFailure, + isTinyBudgetReasoningProbe, + toPositiveInteger, +} from "../services/reasoningTokenBuffer.ts"; import { normalizeThinkingForModel } from "@/shared/constants/modelSpecs.ts"; import { buildErrorBody, @@ -247,7 +253,10 @@ import { normalizeOpenAIToolFinishReasons, restoreNonStreamingToolNames, } from "./chatCore/passthroughToolNames.ts"; -import { createDisabledCompressionConfig, resolveCompressionSettings } from "./chatCore/compressionSettings.ts"; +import { + createDisabledCompressionConfig, + resolveCompressionSettings, +} from "./chatCore/compressionSettings.ts"; import type { EnforceDecision } from "@/lib/quota/types"; import { isCompressionExcluded } from "../services/compression/exclusions.ts"; import { @@ -1822,7 +1831,11 @@ export async function handleChatCore({ // engines (Caveman/RTK). Codex Desktop / Responses clients need this path even // when those engines are off, otherwise multi-turn image sessions hard-reject // at the budget check below (#8560). - if (reactiveContextCompactionEnabled && !nativeCodexPassthrough && estimatedTokens > threshold) { + if ( + reactiveContextCompactionEnabled && + !nativeCodexPassthrough && + estimatedTokens > threshold + ) { log?.info?.( "CONTEXT", `Proactive compression triggered: ${estimatedTokens} tokens > ${threshold} threshold (${contextLimit} limit)` @@ -1892,7 +1905,12 @@ export async function handleChatCore({ // Last-resort compaction against the concrete input budget (not the 70% threshold). // Covers cases where the proactive pass was skipped or still left the request oversized (#8560). - if (reactiveContextCompactionEnabled && !nativeCodexPassthrough && finalEstimatedInputTokens >= finalContextLimit && body) { + if ( + reactiveContextCompactionEnabled && + !nativeCodexPassthrough && + finalEstimatedInputTokens >= finalContextLimit && + body + ) { const lastResortTarget = Math.max(1, finalContextLimit - toolsReserve - 1); const lastResortAdapter = adaptBodyForCompression(body as Record); const lastResortResult = compressContext(lastResortAdapter.body, { @@ -3706,6 +3724,33 @@ export async function handleChatCore({ if (signatureRecovery.succeeded) break providerFailure; + // #10281 — tiny-budget reasoning probes (e.g. Claude Code's `/model` check + // sends `max_tokens: 1`): the model burns the whole budget on thinking, and + // some upstreams (e.g. api.cline.bot for deepseek-v4-flash) answer the empty + // outcome with a 5xx ("empty response content") instead of a truncated 200. + // Answer such probes with a valid truncated response rather than relaying the + // upstream failure — which would also mark the connection unavailable and + // poison fallback/cooldown bookkeeping for a request that is only a probe. + if ( + !stream && + isTinyBudgetReasoningProbe({ model: currentModel, body: finalBody || translatedBody }) && + isEmptyContentUpstreamFailure(statusCode, message) + ) { + providerResponse = buildReasoningProbeTruncatedResponse({ + model: currentModel, + maxTokens: toPositiveInteger( + (finalBody || translatedBody)?.max_tokens ?? + (finalBody || translatedBody)?.max_completion_tokens + ), + requestId: skillRequestId, + }); + log?.warn?.( + "PROBE", + `Reasoning probe (max_tokens < ${REASONING_BUFFER_MIN_TRIGGER}) answered with truncated 200 — upstream reported "${message}"` + ); + break providerFailure; + } + // T06/T10/T36: classify provider errors and persist terminal account states. let errorType = classifyProviderError(statusCode, message, provider); if (statusCode === 429 && isModelScope()) { @@ -4327,7 +4372,11 @@ export async function handleChatCore({ } : responseBody ); - sanitizeUsagePayloadForRequest(responseBody, finalBody || translatedBody || body, responsePayloadFormat); + sanitizeUsagePayloadForRequest( + responseBody, + finalBody || translatedBody || body, + responsePayloadFormat + ); effectiveServiceTier = resolveReportedServiceTier(responseBody) ?? effectiveServiceTier; // Notify success - caller can clear error status if needed if (onRequestSuccess) { @@ -4472,9 +4521,14 @@ export async function handleChatCore({ // #8331: keep the client-visible metering fields real everywhere except Claude-Code-compatible // providers, where Claude Code's own context accounting relies on the buffered number — see // clientUsageBuffer.ts module docstring. - applyClientUsageBuffer(translatedResponse, finalBody || translatedBody || body, clientResponseFormat, { - preserveContextBudgetInVisibleUsage: isClaudeCodeCompatible, - }); + applyClientUsageBuffer( + translatedResponse, + finalBody || translatedBody || body, + clientResponseFormat, + { + preserveContextBudgetInVisibleUsage: isClaudeCodeCompatible, + } + ); if (memoryOwnerId && memorySettings?.enabled && memorySettings.maxTokens > 0) { const requestMemoryText = extractMemoryTextFromRequestBody(body as Record); diff --git a/open-sse/services/reasoningTokenBuffer.ts b/open-sse/services/reasoningTokenBuffer.ts index 4c5ce880594..fc4ac7157f1 100644 --- a/open-sse/services/reasoningTokenBuffer.ts +++ b/open-sse/services/reasoningTokenBuffer.ts @@ -54,3 +54,74 @@ export function resolveReasoningBufferedMaxTokens( // silent cost increase the client did not authorize. return current; } + +/** + * A tiny-budget reasoning probe is a request with an explicit `max_tokens` + * below REASONING_BUFFER_MIN_TRIGGER targeting a reasoning-capable model — e.g. + * Claude Code's `/model` capability check sends `max_tokens: 1`. Reasoning + * models burn the whole probe on thinking, so the upstream produces no visible + * content; some upstreams (e.g. api.cline.bot for deepseek-v4-flash) answer the + * non-streaming probe with an HTTP 5xx (`"empty response content"`) instead of + * a truncated 200. See #10281. + */ +export function isTinyBudgetReasoningProbe(opts: { model: string; body: unknown }): boolean { + const body = (opts.body ?? {}) as Record; + const maxTokens = toPositiveInteger(body.max_tokens ?? body.max_completion_tokens); + if (maxTokens === null || maxTokens >= REASONING_BUFFER_MIN_TRIGGER) return false; + const capabilities = getResolvedModelCapabilities(opts.model); + return capabilities.supportsThinking === true; +} + +/** + * Upstream failure markers that describe the "model reasoned but produced no + * visible content" outcome (e.g. `{"error":{"message":"empty response content"}}`). + */ +const EMPTY_CONTENT_FAILURE_RE = + /empty(\s+response)?\s+content|no\s+(usable\s+)?content|reasoning\s+consumed/i; + +/** + * True when the upstream failure is a 5xx describing the empty-content outcome + * of a reasoning probe rather than a genuine provider outage. Combined with + * `isTinyBudgetReasoningProbe`, false positives are not practical (a real 5xx + * carrying these markers on a tiny-budget reasoning request is this exact case). + */ +export function isEmptyContentUpstreamFailure(statusCode: number, message: string): boolean { + if (!Number.isFinite(statusCode) || statusCode < 500 || statusCode >= 600) return false; + return EMPTY_CONTENT_FAILURE_RE.test(String(message || "")); +} + +/** + * Build a valid truncated OpenAI chat.completion response (200, empty content, + * `finish_reason: "length"`) used to answer a tiny-budget reasoning probe whose + * upstream answered the empty outcome with a 5xx. Mirrors the semantics OmniRoute + * already grants to `finish_reason: "length"` empty 200s (errorClassifier.ts). + */ +export function buildReasoningProbeTruncatedResponse(opts: { + model: string; + maxTokens: number | null; + requestId: string; +}): Response { + const maxTokens = opts.maxTokens ?? 1; + const body = { + id: `chatcmpl-${opts.requestId}`, + object: "chat.completion", + created: Math.floor(Date.now() / 1000), + model: opts.model, + choices: [ + { + index: 0, + message: { role: "assistant", content: "" }, + finish_reason: "length", + }, + ], + usage: { + prompt_tokens: 0, + completion_tokens: maxTokens, + total_tokens: maxTokens, + }, + }; + return new Response(JSON.stringify(body), { + status: 200, + headers: { "content-type": "application/json" }, + }); +} diff --git a/tests/unit/reasoning-probe-truncated-response-10281.test.ts b/tests/unit/reasoning-probe-truncated-response-10281.test.ts new file mode 100644 index 00000000000..b69a452defa --- /dev/null +++ b/tests/unit/reasoning-probe-truncated-response-10281.test.ts @@ -0,0 +1,171 @@ +/** + * #10281 — tiny-budget reasoning probes (e.g. Claude Code's `/model` capability + * check sends `max_tokens: 1`) must be answered with a valid truncated 200 when + * the upstream answers the reasoning-only outcome with a 5xx ("empty response + * content") instead of a truncated 200 — rather than relaying the upstream + * failure, which also poisons connection cooldown/health bookkeeping. + * + * Covers the pure helpers in open-sse/services/reasoningTokenBuffer.ts: + * - isTinyBudgetReasoningProbe — probe detection + * - isEmptyContentUpstreamFailure — empty-content 5xx detection + * - buildReasoningProbeTruncatedResponse — synthetic truncated 200 + * plus the invariant that the synthetic body is NOT flagged as empty content by + * errorClassifier.isEmptyContentResponse (finish_reason "length" is legitimate). + */ +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-reasoning-probe-")); +process.env.DATA_DIR = TEST_DATA_DIR; + +const core = await import("../../src/lib/db/core.ts"); +const { saveModelsDevCapabilities, clearModelsDevCapabilities } = + await import("../../src/lib/modelsDevSync.ts"); +const { + REASONING_BUFFER_MIN_TRIGGER, + buildReasoningProbeTruncatedResponse, + isEmptyContentUpstreamFailure, + isTinyBudgetReasoningProbe, +} = await import("../../open-sse/services/reasoningTokenBuffer.ts"); +const { isEmptyContentResponse } = await import("../../open-sse/services/errorClassifier.ts"); + +function capabilityEntry(limitContext: unknown, overrides: Record = {}) { + return { + tool_call: true, + reasoning: false, + attachment: false, + structured_output: true, + temperature: true, + modalities_input: JSON.stringify(["text"]), + modalities_output: JSON.stringify(["text"]), + knowledge_cutoff: null, + release_date: null, + last_updated: null, + status: null, + family: null, + open_weights: false, + limit_context: limitContext, + limit_input: limitContext, + limit_output: 4096, + interleaved_field: null, + ...overrides, + }; +} + +test.before(() => { + saveModelsDevCapabilities({ + zhipu: { + // A thinking-capable model: probe detection + buffer logic both engage. + "glm-5.2": capabilityEntry(200000, { reasoning: true, limit_output: 65536 }), + // A non-reasoning sibling: probes are not special-cased. + "glm-5.2-flash": capabilityEntry(200000, { reasoning: false, limit_output: 4096 }), + }, + }); +}); + +test.after(() => { + clearModelsDevCapabilities(); + core.resetDbInstance(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); +}); + +test("#10281 isTinyBudgetReasoningProbe detects tiny explicit budgets on reasoning models", () => { + const thinking = "zhipu/glm-5.2"; + // Claude Code's `/model` probe (max_tokens: 1) is a tiny-budget reasoning probe. + assert.equal( + isTinyBudgetReasoningProbe({ model: thinking, body: { max_tokens: 1 } }), + true, + "max_tokens=1 on a reasoning model is a probe" + ); + // Just below the trigger threshold is still a probe. + assert.equal( + isTinyBudgetReasoningProbe({ + model: thinking, + body: { max_tokens: REASONING_BUFFER_MIN_TRIGGER - 1 }, + }), + true, + "budgets below REASONING_BUFFER_MIN_TRIGGER are probes" + ); + // At/above the threshold it is a genuine budget, not a probe. + assert.equal( + isTinyBudgetReasoningProbe({ + model: thinking, + body: { max_tokens: REASONING_BUFFER_MIN_TRIGGER }, + }), + false, + "budgets at REASONING_BUFFER_MIN_TRIGGER are not probes" + ); + assert.equal( + isTinyBudgetReasoningProbe({ model: thinking, body: { max_tokens: 512 } }), + false, + "genuine budgets are not probes" + ); + // OpenAI Responses format field is honoured. + assert.equal( + isTinyBudgetReasoningProbe({ model: thinking, body: { max_completion_tokens: 1 } }), + true, + "max_completion_tokens=1 is a probe" + ); + // Missing / non-positive budgets are not probes. + assert.equal( + isTinyBudgetReasoningProbe({ model: thinking, body: {} }), + false, + "no budget is not a probe" + ); + assert.equal( + isTinyBudgetReasoningProbe({ model: thinking, body: { max_tokens: 0 } }), + false, + "non-positive budget is not a probe" + ); + // Non-reasoning models never probe-special-case. + assert.equal( + isTinyBudgetReasoningProbe({ model: "zhipu/glm-5.2-flash", body: { max_tokens: 1 } }), + false, + "non-reasoning models are not probes" + ); +}); + +test("#10281 isEmptyContentUpstreamFailure matches empty-content 5xx markers", () => { + assert.equal(isEmptyContentUpstreamFailure(500, "empty response content"), true); + assert.equal(isEmptyContentUpstreamFailure(500, "No content was produced"), true); + assert.equal(isEmptyContentUpstreamFailure(502, "empty response content"), true); + assert.equal( + isEmptyContentUpstreamFailure(500, "empty response body"), + false, + "generic empty-body 5xx is not a reasoning outcome" + ); + assert.equal(isEmptyContentUpstreamFailure(500, "server_error"), false); + assert.equal(isEmptyContentUpstreamFailure(503, "upstream timeout"), false); + assert.equal( + isEmptyContentUpstreamFailure(429, "empty response content"), + false, + "non-5xx is not an empty-content failure" + ); + assert.equal(isEmptyContentUpstreamFailure(200, "empty response content"), false); +}); + +test("#10281 buildReasoningProbeTruncatedResponse yields a valid truncated 200", async () => { + const res = buildReasoningProbeTruncatedResponse({ + model: "zhipu/glm-5.2", + maxTokens: 1, + requestId: "test-request-id", + }); + assert.equal(res.status, 200); + assert.match(res.headers.get("content-type") || "", /application\/json/); + + const body = (await res.json()) as Record; + const choice = (body.choices as Array>)[0]; + assert.equal(body.object, "chat.completion"); + assert.equal(body.model, "zhipu/glm-5.2"); + assert.equal(choice.finish_reason, "length"); + assert.equal((choice.message as Record).content, ""); + assert.equal((body.usage as Record).completion_tokens, 1); + + // The synthetic body must pass the empty-content guard (finish_reason "length" + // is a legitimate truncated completion — see errorClassifier.ts) so the + // non-stream success path does not re-flag it as a fake-success failure. + assert.equal(isEmptyContentResponse(body), false, "truncated probe response is a legitimate 200"); +});