diff --git a/docs/PROMPT_CACHE_RETENTION.md b/docs/PROMPT_CACHE_RETENTION.md new file mode 100644 index 0000000..07f61cc --- /dev/null +++ b/docs/PROMPT_CACHE_RETENTION.md @@ -0,0 +1,76 @@ +# GPT-5.6 prompt-cache retention evidence + +Status: implemented for direct OpenAI GPT-5.6 request builders. +Date: 2026-06-06. + +## Decision + +For direct OpenAI GPT-5.6 profiles, the TypeScript SDK sends `prompt_cache_retention: "24h"` by default on both Responses and Chat Completions request bodies. Profiles can set `promptCacheRetention: "disabled"` to omit the field, or `promptCacheRetention: "24h"` to persist the choice explicitly. + +The SDK only applies this to known-safe direct OpenAI GPT-5.6 routes: + +- `providerId: "openai"` +- model IDs exactly `gpt-5.6` / `openai/gpt-5.6` or bounded variants such as `gpt-5.6-*` / `gpt-5.6.*` +- default OpenAI API base URL or `https://api.openai.com/...` + +It deliberately omits the field for LiteLLM/OpenRouter aliases, provider-branded custom/internal proxy URLs, and ChatGPT subscription/Codex endpoints until those routes have direct evidence. `promptCacheKey` is persisted on profiles and sent only on the same known-safe routes. + +## Official documentation snapshot + +Source: + +Relevant official-doc points captured during implementation: + +- Prompt caching is automatic for eligible recent models, but GPT-5.6 and later report cache writes through `cache_write_tokens`. +- `prompt_cache_key` helps route requests with shared long prefixes to the same cache; OpenAI recommends stable keys for shared prefixes. +- For GPT-5.6 and later, the public guide describes `prompt_cache_options.ttl`, currently supporting only `30m`. +- The same guide says legacy `prompt_cache_retention` is deprecated for GPT-5.6 and later and remains the retention policy field for earlier models. + +The live API currently accepts `prompt_cache_retention: "24h"` for GPT-5.6 despite that deprecation wording, so this SDK change is intentionally gated to direct OpenAI GPT-5.6 evidence rather than generalized to proxies. + +## Live probe evidence + +Environment: + +- OpenAI API key available. +- Model: `gpt-5.6`. +- All probes used small output limits and `reasoning.effort` / `reasoning_effort` set to `none` where accepted. + +### Field acceptance + +| API | Request field | Result | Notes | +| --- | --- | --- | --- | +| Responses | `prompt_cache_retention: "24h"` | 200 | Response echoed `prompt_cache_retention: "24h"`; short prompt had zero cache writes. | +| Chat Completions | `prompt_cache_retention: "24h"` | 200 | Response did not echo the field, but usage included cache detail fields. | +| Responses | `prompt_cache_options: { ttl: "24h" }` | 400 | Error: `Invalid value: '24h'. Supported values are: '30m'.` | +| Chat Completions | `prompt_cache_options: { ttl: "24h" }` | 400 | Same invalid-value error. | +| Responses | `prompt_cache_options: { ttl: "30m" }` | 200 | Response still echoed `prompt_cache_retention: "24h"` in this probe. | +| Chat Completions | `prompt_cache_options: { ttl: "30m" }` | 200 | Accepted. | + +### 5,412-token repeated prefix cache proof + +Responses API with identical input text and a stable `prompt_cache_key`: + +| Run | Input tokens | Cache write tokens | Cached tokens | Response field | +| --- | ---: | ---: | ---: | --- | +| 1 | 5,412 | 5,409 | 0 | `prompt_cache_retention: "24h"` | +| 2 | 5,412 | 0 | 5,409 | `prompt_cache_retention: "24h"` | +| 3 | 5,412 | 0 | 5,409 | `prompt_cache_retention: "24h"` | + +Chat Completions with identical input text and the same stable key pattern: + +| Run | Prompt tokens | Cache write tokens | Cached tokens | +| --- | ---: | ---: | ---: | +| 1 | 5,412 | 5,409 | 0 | +| 2 | 5,412 | 0 | 5,409 | +| 3 | 5,412 | 0 | 5,409 | + +A varying suffix inside one raw Responses `input` string wrote the cache again on run 2 instead of producing a cache hit. The implementation therefore only exposes the retention/key request fields; callers still need stable rendered prefixes or explicit future breakpoint support to reliably reuse cache across changing turns. + +## Implementation notes + +- `promptCacheRetention` is part of `llmProfileSchema` so persisted profiles round-trip the option. +- `null` means SDK default. For direct GPT-5.6 OpenAI requests, that default is `24h`. +- `disabled` omits `prompt_cache_retention`; automatic provider prompt caching may still occur. +- `promptCacheKey` is optional and only sent where the retention route is known-safe. +- Unsupported models/routes strip both prompt-cache fields from OpenAI-compatible request bodies instead of guessing. diff --git a/docs/README.md b/docs/README.md index f9149e3..5c81cc8 100644 --- a/docs/README.md +++ b/docs/README.md @@ -7,6 +7,8 @@ This directory captures the working documentation for `@smolpaws/openhands-agent - [`ARCHITECTURE.md`](ARCHITECTURE.md) — main component architecture and data flow. - [`TRANSPILE_PLAN.md`](TRANSPILE_PLAN.md) — upstream target, parity principles, accepted deviations, and remaining roadmap. - [`REASONING_CAPABILITIES.md`](REASONING_CAPABILITIES.md) — provider/model-specific reasoning and thinking controls investigation plus proposed API shape. +- [`PROMPT_CACHE_RETENTION.md`](PROMPT_CACHE_RETENTION.md) — GPT-5.6 prompt-cache retention evidence, live probes, and TypeScript SDK implementation decision. + - [`RELEASE_0.3.1.md`](RELEASE_0.3.1.md) — current 0.3.1 release notes. - [`RELEASE_0.3.0.md`](RELEASE_0.3.0.md) — 0.3.0 event-log persistence release notes. - [`RELEASE_0.2.0.md`](RELEASE_0.2.0.md) — 0.2.0 parity release notes. diff --git a/docs/TRANSPILE_PLAN.md b/docs/TRANSPILE_PLAN.md index cc0ece4..7ee3eb9 100644 --- a/docs/TRANSPILE_PLAN.md +++ b/docs/TRANSPILE_PLAN.md @@ -66,6 +66,7 @@ Follow-up scope after 0.2.0: - **Examples parity expansion.** Continue adding relevant TypeScript examples for persistence, async/send-message-while-running, condenser workflows, observability wiring, and testing helpers where the TS surface supports them. - **Remote runtime hardening.** Keep `RemoteConversation`/`RemoteWorkspace` aligned with the agent-server API used by smolpaws and add integration coverage when the local dev stack exposes stable endpoints. - **Provider-native reasoning controls.** Replace the cross-provider `reasoningEffort` abstraction with provider/model-specific capability discovery and serializable reasoning config. See [`REASONING_CAPABILITIES.md`](REASONING_CAPABILITIES.md) for official-doc evidence, OpenAI/Gemini live probes, the proposed TypeScript API shape, and remaining Anthropic/LiteLLM live-test gaps. +- **GPT-5.6 prompt-cache retention.** Direct OpenAI GPT-5.6 request builders now default to `prompt_cache_retention: "24h"` with profile-persisted override/key fields and route gates. See [`PROMPT_CACHE_RETENTION.md`](PROMPT_CACHE_RETENTION.md) for official-doc tension, live probe evidence, and unsupported-route decisions. - **Observability depth.** The current wrapper is intentionally small; add real OpenTelemetry/Laminar integration only when a downstream product needs it. - **Extensions metadata.** Keep source-resolution and installation metadata useful without reviving plugin/marketplace runtime behavior. - **Plugin** and **marketplace** remain intentionally skipped. Do not create beads for them unless explicitly requested. diff --git a/src/conversation/__tests__/event-log-parity.test.ts b/src/conversation/__tests__/event-log-parity.test.ts index f2c50ac..318292f 100644 --- a/src/conversation/__tests__/event-log-parity.test.ts +++ b/src/conversation/__tests__/event-log-parity.test.ts @@ -321,6 +321,8 @@ class FakeLLM implements LLMClient { timeoutSeconds: null, reasoningEffort: null, reasoningSummary: null, + promptCacheRetention: null, + promptCacheKey: null, headers: {}, useProfileKeyOverride: false, }; diff --git a/src/conversation/__tests__/event-log.test.ts b/src/conversation/__tests__/event-log.test.ts index 73a61a2..90e7278 100644 --- a/src/conversation/__tests__/event-log.test.ts +++ b/src/conversation/__tests__/event-log.test.ts @@ -243,6 +243,8 @@ class FakeLLM implements LLMClient { timeoutSeconds: null, reasoningEffort: null, reasoningSummary: null, + promptCacheRetention: null, + promptCacheKey: null, headers: {}, useProfileKeyOverride: false, }; diff --git a/src/conversation/__tests__/local-conversation.test.ts b/src/conversation/__tests__/local-conversation.test.ts index 2cd4c07..a41f6a0 100644 --- a/src/conversation/__tests__/local-conversation.test.ts +++ b/src/conversation/__tests__/local-conversation.test.ts @@ -74,7 +74,7 @@ describe('LocalConversation', () => { }); class FakeLLM implements LLMClient { - readonly profile: LLMProfile = { profileId: 'fake', providerId: 'fake', model: 'fake', baseUrl: null, openAiApiMode: 'chat_completions', temperature: null, topP: null, topK: null, maxInputTokens: null, maxOutputTokens: null, timeoutSeconds: null, reasoningEffort: null, reasoningSummary: null, headers: {}, useProfileKeyOverride: false }; + readonly profile: LLMProfile = { profileId: 'fake', providerId: 'fake', model: 'fake', baseUrl: null, openAiApiMode: 'chat_completions', temperature: null, topP: null, topK: null, maxInputTokens: null, maxOutputTokens: null, timeoutSeconds: null, reasoningEffort: null, reasoningSummary: null, promptCacheRetention: null, promptCacheKey: null, headers: {}, useProfileKeyOverride: false }; readonly requests: readonly Message[][] = []; diff --git a/src/llm/__tests__/openai-client.test.ts b/src/llm/__tests__/openai-client.test.ts index a20dc69..e2d7433 100644 --- a/src/llm/__tests__/openai-client.test.ts +++ b/src/llm/__tests__/openai-client.test.ts @@ -211,6 +211,85 @@ describe('OpenAI chat message serialization parity', () => { expect(body).toMatchObject({ reasoning: { effort: 'low' } }); }); + it('persists prompt-cache profile options through the LLM profile schema', () => { + const profile = llmProfileSchema.parse({ + profileId: 'cache-profile', + providerId: 'openai', + model: 'gpt-5.6', + promptCacheRetention: '24h', + promptCacheKey: 'stable-prefix-v1', + }); + + expect(profile.promptCacheRetention).toBe('24h'); + expect(profile.promptCacheKey).toBe('stable-prefix-v1'); + expect(() => llmProfileSchema.parse({ profileId: 'bad', providerId: 'openai', model: 'gpt-5.6', promptCacheRetention: '30m' })).toThrow(); + }); + + it('adds default 24h prompt-cache retention for direct OpenAI GPT-5.6 Responses requests', () => { + const profile = llmProfileSchema.parse({ + profileId: 'responses-cache', + providerId: 'openai', + model: 'gpt-5.6', + openAiApiMode: 'responses', + promptCacheKey: 'conversation-cache-key', + }); + + const body = buildOpenAIResponsesBody(profile, [{ role: 'user', content: [textContent('cache me')] }]); + + expect(body).toMatchObject({ + prompt_cache_retention: '24h', + prompt_cache_key: 'conversation-cache-key', + }); + }); + + it('adds default 24h prompt-cache retention for direct OpenAI GPT-5.6 Chat requests', () => { + const profile = llmProfileSchema.parse({ + profileId: 'chat-cache', + providerId: 'openai', + model: 'gpt-5.6', + promptCacheKey: 'conversation-cache-key', + }); + + const body = buildChatCompletionsBody(profile, [{ role: 'user', content: [textContent('cache me')] }]); + + expect(body).toMatchObject({ + prompt_cache_retention: '24h', + prompt_cache_key: 'conversation-cache-key', + }); + }); + + it('omits prompt-cache retention for unsupported routes and explicit disablement', () => { + const gpt51 = buildOpenAIResponsesBody( + llmProfileSchema.parse({ profileId: 'gpt51', providerId: 'openai', model: 'gpt-5.1', openAiApiMode: 'responses', promptCacheRetention: '24h', promptCacheKey: 'ignored-key' }), + [{ role: 'user', content: [textContent('cache me')] }], + ); + const litellmAlias = buildChatCompletionsBody( + llmProfileSchema.parse({ profileId: 'proxy', providerId: 'litellm_proxy', model: 'openai/gpt-5.6', baseUrl: 'https://llm-proxy.example.test', promptCacheRetention: '24h', promptCacheKey: 'ignored-key' }), + [{ role: 'user', content: [textContent('cache me')] }], + ); + const subscriptionEndpoint = buildOpenAIResponsesBody( + llmProfileSchema.parse({ profileId: 'subscription', providerId: 'openai', model: 'gpt-5.6-codex', baseUrl: 'https://chatgpt.com/backend-api/codex', openAiApiMode: 'responses', promptCacheRetention: '24h', promptCacheKey: 'ignored-key' }), + [{ role: 'user', content: [textContent('cache me')] }], + ); + const openAINamedProxy = buildChatCompletionsBody( + llmProfileSchema.parse({ profileId: 'openai-proxy', providerId: 'openai', model: 'gpt-5.6', baseUrl: 'https://openai-proxy.example.test/v1', promptCacheRetention: '24h', promptCacheKey: 'ignored-key' }), + [{ role: 'user', content: [textContent('cache me')] }], + ); + const futureSimilarModel = buildChatCompletionsBody( + llmProfileSchema.parse({ profileId: 'future', providerId: 'openai', model: 'gpt-5.60', promptCacheRetention: '24h', promptCacheKey: 'ignored-key' }), + [{ role: 'user', content: [textContent('cache me')] }], + ); + const disabled = buildOpenAIResponsesBody( + llmProfileSchema.parse({ profileId: 'disabled', providerId: 'openai', model: 'gpt-5.6', openAiApiMode: 'responses', promptCacheRetention: 'disabled' }), + [{ role: 'user', content: [textContent('cache me')] }], + ); + + for (const body of [gpt51, litellmAlias, subscriptionEndpoint, openAINamedProxy, futureSimilarModel, disabled]) { + expect(body).not.toHaveProperty('prompt_cache_retention'); + expect(body).not.toHaveProperty('prompt_cache_key'); + } + }); + it('replays Responses reasoning items with encrypted content in stateless mode', () => { const profile = llmProfileSchema.parse({ profileId: 'responses', diff --git a/src/llm/index.ts b/src/llm/index.ts index 384e465..5b45bdb 100644 --- a/src/llm/index.ts +++ b/src/llm/index.ts @@ -11,6 +11,7 @@ export const llmProviderIdSchema = z.string().min(1).regex(/^[A-Za-z0-9._-]+$/u) export const openAiApiModeSchema = z.union([z.literal('chat_completions'), z.literal('responses')]); export const reasoningEffortSchema = z.union([z.literal('low'), z.literal('medium'), z.literal('high')]); export const reasoningSummarySchema = z.union([z.literal('auto'), z.literal('concise'), z.literal('detailed')]); +export const promptCacheRetentionSchema = z.union([z.literal('24h'), z.literal('disabled')]); export const llmProfileSchema = z .object({ @@ -27,6 +28,8 @@ export const llmProfileSchema = z timeoutSeconds: z.number().positive().nullable().default(null), reasoningEffort: reasoningEffortSchema.nullable().default(null), reasoningSummary: reasoningSummarySchema.nullable().default(null), + promptCacheRetention: promptCacheRetentionSchema.nullable().default(null), + promptCacheKey: z.string().min(1).nullable().default(null), headers: z.record(z.string(), z.string()).default({}), useProfileKeyOverride: z.boolean().default(false), }) @@ -36,6 +39,7 @@ export type LLMProfile = z.infer; export type OpenAiApiMode = z.infer; export type ReasoningEffort = z.infer; export type ReasoningSummary = z.infer; +export type PromptCacheRetention = z.infer; export function resolveLlmProfileApiKeyRef(profile: LLMProfile, store: SecretStore): Promise { return resolveLlmApiKeyRef( diff --git a/src/llm/openai.ts b/src/llm/openai.ts index 9edc083..b045121 100644 --- a/src/llm/openai.ts +++ b/src/llm/openai.ts @@ -11,7 +11,7 @@ import { type Message, type MessageToolCall, } from './index.js'; -import { normalizeGenerationParamsForModel } from './provider-quirks.js'; +import { normalizeGenerationParamsForModel, resolveOpenAIPromptCacheKey, resolveOpenAIPromptCacheRetention } from './provider-quirks.js'; export { llmCompletionResponseSchema, llmUsageSchema } from './client.js'; export type { FetchLike, FetchResponseLike, LLMClient, LLMCompletionResponse, LLMUsage } from './client.js'; @@ -124,6 +124,18 @@ export async function createOpenAIResponsesClientFromProfile( return new OpenAIResponsesClient(profile, apiKey, options.fetch ?? defaultFetch); } +function applyOpenAIPromptCacheOptions(body: Record, profile: LLMProfile): void { + const retention = resolveOpenAIPromptCacheRetention(profile); + if (retention !== undefined) { + body.prompt_cache_retention = retention; + } + + const cacheKey = resolveOpenAIPromptCacheKey(profile); + if (cacheKey !== undefined) { + body.prompt_cache_key = cacheKey; + } +} + export function buildChatCompletionsBody(profile: LLMProfile, messages: readonly Message[]): Record { const normalizedProfile = normalizeGenerationParamsForModel(profile); const body: Record = { @@ -145,6 +157,7 @@ export function buildChatCompletionsBody(profile: LLMProfile, messages: readonly if (normalizedProfile.reasoningEffort !== null) { body.reasoning_effort = normalizedProfile.reasoningEffort; } + applyOpenAIPromptCacheOptions(body, normalizedProfile); return body; } @@ -176,6 +189,7 @@ export function buildOpenAIResponsesBody(profile: LLMProfile, messages: readonly ...(normalizedProfile.reasoningSummary === null ? {} : { summary: normalizedProfile.reasoningSummary }), }; } + applyOpenAIPromptCacheOptions(body, normalizedProfile); return body; } diff --git a/src/llm/provider-quirks.ts b/src/llm/provider-quirks.ts index 3a7c536..2e60f5a 100644 --- a/src/llm/provider-quirks.ts +++ b/src/llm/provider-quirks.ts @@ -1,4 +1,4 @@ -import type { LLMProfile, ReasoningEffort } from './index.js'; +import type { LLMProfile, PromptCacheRetention, ReasoningEffort } from './index.js'; export const ANTHROPIC_THINKING_MIN_BUDGET = 1024; export const ANTHROPIC_THINKING_MAX_BUDGET = 128000; @@ -24,6 +24,39 @@ export function isGpt5Model(model: string | null | undefined): boolean { return model?.trim().toLowerCase().includes('gpt-5') === true; } +export function isGpt56Model(model: string | null | undefined): boolean { + const normalized = model?.trim().toLowerCase().replace(/^openai\//u, '') ?? ''; + return /^gpt-5\.6(?:[-.]|$)/u.test(normalized); +} + +export function isOpenAISubscriptionEndpoint(profile: LLMProfile): boolean { + const baseUrl = profile.baseUrl?.trim().toLowerCase() ?? ''; + return baseUrl.includes('chatgpt.com/backend-api/codex'); +} + +export function supportsOpenAIPromptCacheRetention(profile: LLMProfile): boolean { + if (profile.providerId !== 'openai' || isOpenAISubscriptionEndpoint(profile) || !isGpt56Model(profile.model)) { + return false; + } + const baseUrl = profile.baseUrl?.trim().toLowerCase(); + return baseUrl === undefined || baseUrl === '' || baseUrl.startsWith('https://api.openai.com/'); +} + +export function resolveOpenAIPromptCacheRetention(profile: LLMProfile): PromptCacheRetention | undefined { + if (!supportsOpenAIPromptCacheRetention(profile) || profile.promptCacheRetention === 'disabled') { + return undefined; + } + return profile.promptCacheRetention ?? '24h'; +} + +export function resolveOpenAIPromptCacheKey(profile: LLMProfile): string | undefined { + if (!supportsOpenAIPromptCacheRetention(profile)) { + return undefined; + } + return profile.promptCacheKey ?? undefined; +} + + export function hasExtendedThinking(profile: LLMProfile): boolean { return profile.reasoningEffort !== null; } diff --git a/src/settings/__tests__/settings.test.ts b/src/settings/__tests__/settings.test.ts index 45ea296..3ccafe0 100644 --- a/src/settings/__tests__/settings.test.ts +++ b/src/settings/__tests__/settings.test.ts @@ -101,6 +101,8 @@ describe('AgentSettings', () => { maxOutputTokens: 2000, reasoningEffort: 'high', reasoningSummary: 'auto', + promptCacheRetention: '24h', + promptCacheKey: 'conversation-cache-key', inputCostPerToken: 0.1, outputCostPerToken: 0.2, encrypted_reasoning: 'kept', @@ -120,6 +122,8 @@ describe('AgentSettings', () => { 'maxOutputTokens', 'reasoningEffort', 'reasoningSummary', + 'promptCacheRetention', + 'promptCacheKey', 'inputCostPerToken', 'outputCostPerToken', ]); diff --git a/src/settings/index.ts b/src/settings/index.ts index 11dd1a5..52436b7 100644 --- a/src/settings/index.ts +++ b/src/settings/index.ts @@ -19,6 +19,8 @@ export const RAW_LLM_FIELDS_IGNORED_WHEN_PROFILE_SELECTED = [ 'maxOutputTokens', 'reasoningEffort', 'reasoningSummary', + 'promptCacheRetention', + 'promptCacheKey', 'inputCostPerToken', 'outputCostPerToken', ] as const; @@ -111,6 +113,8 @@ export function clearRawLlmFieldsWhenProfileSelected