diff --git a/packages/claude-agents/src/__tests__/handler.test.ts b/packages/claude-agents/src/__tests__/handler.test.ts index b228fe4a..aacd18eb 100644 --- a/packages/claude-agents/src/__tests__/handler.test.ts +++ b/packages/claude-agents/src/__tests__/handler.test.ts @@ -56,6 +56,7 @@ vi.mock('@launchdarkly/ai-server', async (importOriginal) => { import { type Message, NATIVE_TOOL_KEY, NativeTool } from '@launchdarkly/ai-server'; import { buildPrompt, buildToolMCP, createClaudeAgentsHandler, partitionTools } from '../handler.js'; +import { expectNoNeverForwardedValue, NEVER_FORWARDED_PARAMETERS } from './never-forwarded.js'; // ─── Helpers ────────────────────────────────────────────────────────────────── @@ -189,6 +190,179 @@ describe('createClaudeAgentsHandler', () => { expect(result.output).toBe('final answer'); }); + it('forwards model.parameters.maxTurns to query options', async () => { + mockQuery.mockImplementation(makeResultMessage()); + const config = { ...baseConfig, model: { ...baseConfig.model, parameters: { maxTurns: 3 } } }; + await createClaudeAgentsHandler()(config as any, 'q'); + const { options } = mockQuery.mock.calls[0][0]; + expect(options.maxTurns).toBe(3); + }); + + // The LaunchDarkly UI writes model.parameters in snake_case (the wire name every provider + // client expects), but the Claude Agent SDK's query() Options only reads camelCase — so a + // config saved as `max_turns` must reach query() as `maxTurns`, not vanish silently. + it('converts a snake_case max_turns to maxTurns for query options', async () => { + mockQuery.mockImplementation(makeResultMessage()); + const config = { ...baseConfig, model: { ...baseConfig.model, parameters: { max_turns: 7 } } }; + await createClaudeAgentsHandler()(config as any, 'q'); + const { options } = mockQuery.mock.calls[0][0]; + expect(options.maxTurns).toBe(7); + expect(options.max_turns).toBeUndefined(); + }); + + it('converts snake_case model.parameters keys for query options (maxThinkingTokens etc.)', async () => { + mockQuery.mockImplementation(makeResultMessage()); + const config = { + ...baseConfig, + model: { + ...baseConfig.model, + parameters: { max_thinking_tokens: 2048, max_budget_usd: 1.5, fallback_model: 'claude-haiku' }, + }, + }; + await createClaudeAgentsHandler()(config as any, 'q'); + const { options } = mockQuery.mock.calls[0][0]; + expect(options.maxThinkingTokens).toBe(2048); + expect(options.maxBudgetUsd).toBe(1.5); + expect(options.fallbackModel).toBe('claude-haiku'); + }); + + it('forwards recognized model.parameters keys to query options', async () => { + mockQuery.mockImplementation(makeResultMessage()); + const config = { + ...baseConfig, + model: { + ...baseConfig.model, + parameters: { maxThinkingTokens: 2048, maxBudgetUsd: 1.5, effort: 'high', fallbackModel: 'claude-haiku' }, + }, + }; + await createClaudeAgentsHandler()(config as any, 'q'); + const { options } = mockQuery.mock.calls[0][0]; + expect(options.maxThinkingTokens).toBe(2048); + expect(options.maxBudgetUsd).toBe(1.5); + expect(options.effort).toBe('high'); + expect(options.fallbackModel).toBe('claude-haiku'); + }); + + it('drops model.parameters keys that are not on the query options allowlist', async () => { + mockQuery.mockImplementation(makeResultMessage()); + const config = { + ...baseConfig, + model: { ...baseConfig.model, parameters: { temperature: 0.7, someUnknownKey: 100, maxTurns: 2 } }, + }; + await createClaudeAgentsHandler()(config as any, 'q'); + const { options } = mockQuery.mock.calls[0][0]; + expect(options.temperature).toBeUndefined(); + expect(options.someUnknownKey).toBeUndefined(); + expect(options.maxTurns).toBe(2); + }); + + it('never forwards credentials, endpoints, remote tools or host-process settings to query options', async () => { + mockQuery.mockImplementation(makeResultMessage()); + const config = { ...baseConfig, model: { ...baseConfig.model, parameters: NEVER_FORWARDED_PARAMETERS } }; + await createClaudeAgentsHandler()(config as any, 'q'); + const { options } = mockQuery.mock.calls[0][0]; + // mcpServers and hooks are set by the handler itself; the deep check proves neither carries + // the config's value. + expectNoNeverForwardedValue(options, ['mcpServers', 'hooks']); + }); + + it('never forwards them on the streaming path either', async () => { + mockQuery.mockImplementation(async function* () { + yield { type: 'result', subtype: 'success', result: 'done', usage: {} }; + }); + const config = { ...baseConfig, model: { ...baseConfig.model, parameters: NEVER_FORWARDED_PARAMETERS } }; + await collectStream(createClaudeAgentsHandler().stream?.(config as any, 'q', {}, {})); + const { options } = mockQuery.mock.calls[0][0]; + expectNoNeverForwardedValue(options, ['mcpServers', 'hooks']); + }); + + it('forwards every allowed key under the Claude Agent SDK name and shape', async () => { + mockQuery.mockImplementation(makeResultMessage()); + const schema = { type: 'object', properties: { answer_text: { type: 'string' } } }; + const config = { + ...baseConfig, + model: { + ...baseConfig.model, + parameters: { + max_turns: 4, + max_thinking_tokens: 2048, + thinking: { type: 'enabled', budget_tokens: 1024, display: 'summarized', extra_key: 'dropped' }, + effort: 'high', + max_budget_usd: 1.5, + fallback_model: 'claude-haiku', + output_format: { type: 'json_schema', schema, extra_key: 'dropped' }, + betas: ['context-1m-2025-08-07'], + }, + }, + }; + await createClaudeAgentsHandler()(config as any, 'q'); + const { options } = mockQuery.mock.calls[0][0]; + expect(options).toMatchObject({ + maxTurns: 4, + maxThinkingTokens: 2048, + effort: 'high', + maxBudgetUsd: 1.5, + fallbackModel: 'claude-haiku', + betas: ['context-1m-2025-08-07'], + }); + // The SDK's ThinkingEnabled reads budgetTokens, not the Messages API's budget_tokens. + expect(options.thinking).toEqual({ type: 'enabled', budgetTokens: 1024, display: 'summarized' }); + // The JSON schema itself is passed through as written. + expect(options.outputFormat).toEqual({ type: 'json_schema', schema }); + }); + + it('accepts a camelCase thinking.budgetTokens and drops a thinking value without a type', async () => { + mockQuery.mockImplementation(makeResultMessage()); + const run = async (thinking: unknown) => { + mockQuery.mockClear(); + const config = { ...baseConfig, model: { ...baseConfig.model, parameters: { thinking } } }; + await createClaudeAgentsHandler()(config as any, 'q'); + return mockQuery.mock.calls[0][0].options.thinking; + }; + expect(await run({ type: 'enabled', budgetTokens: 512 })).toEqual({ type: 'enabled', budgetTokens: 512 }); + expect(await run({ type: 'adaptive' })).toEqual({ type: 'adaptive' }); + expect(await run({ budget_tokens: 512 })).toBeUndefined(); + expect(await run('enabled')).toBeUndefined(); + }); + + it('does not let model.parameters override model, tools, allowedTools, mcpServers, hooks, or systemPrompt', async () => { + mockQuery.mockImplementation(makeResultMessage()); + const config = { + ...baseConfig, + model: { + ...baseConfig.model, + parameters: { + model: 'smuggled-model', + tools: ['smuggled-tool'], + allowedTools: ['smuggled-allowed'], + mcpServers: { smuggled: true }, + hooks: { smuggled: true }, + systemPrompt: 'smuggled prompt', + }, + }, + }; + await createClaudeAgentsHandler()(config as any, 'q'); + const { options } = mockQuery.mock.calls[0][0]; + expect(options.model).toBe('claude-opus-4-5'); + expect(options.systemPrompt).toBe('You are helpful.'); + expect(options.tools).not.toEqual(['smuggled-tool']); + expect(options.allowedTools).not.toEqual(['smuggled-allowed']); + expect(options.mcpServers).not.toEqual({ smuggled: true }); + expect(options.hooks).not.toEqual({ smuggled: true }); + }); + + it('does not set maxTurns or other forwarded keys when model.parameters is absent', async () => { + mockQuery.mockImplementation(makeResultMessage()); + await createClaudeAgentsHandler()(baseConfig as any, 'q'); + const { options } = mockQuery.mock.calls[0][0]; + expect(options.maxTurns).toBeUndefined(); + expect(options.maxThinkingTokens).toBeUndefined(); + expect(options.maxBudgetUsd).toBeUndefined(); + expect(options.effort).toBeUndefined(); + expect(options.fallbackModel).toBeUndefined(); + expect(options.thinking).toBeUndefined(); + }); + // ── 1.3 Tool wiring ───────────────────────────────────────────────────────── it('builds MCP server when config.tools has user-defined tools', async () => { @@ -492,6 +666,44 @@ describe('createClaudeAgentsHandler', () => { expect(done.output).toBe('Final answer'); }); + it('forwards model.parameters.maxTurns alongside includePartialMessages on the streaming path', async () => { + mockQuery.mockImplementation(async function* () { + yield { type: 'result', subtype: 'success', result: 'done', usage: {} }; + }); + const config = { ...baseConfig, model: { ...baseConfig.model, parameters: { maxTurns: 5 } } }; + const handler = createClaudeAgentsHandler(); + await collectStream(handler.stream?.(config as any, 'q', {}, {})); + const { options } = mockQuery.mock.calls[0][0]; + expect(options.maxTurns).toBe(5); + expect(options.includePartialMessages).toBe(true); + }); + + it('converts a snake_case max_turns to maxTurns on the streaming path', async () => { + mockQuery.mockImplementation(async function* () { + yield { type: 'result', subtype: 'success', result: 'done', usage: {} }; + }); + const config = { ...baseConfig, model: { ...baseConfig.model, parameters: { max_turns: 9 } } }; + const handler = createClaudeAgentsHandler(); + await collectStream(handler.stream?.(config as any, 'q', {}, {})); + const { options } = mockQuery.mock.calls[0][0]; + expect(options.maxTurns).toBe(9); + expect(options.max_turns).toBeUndefined(); + }); + + it('does not let model.parameters override includePartialMessages on the streaming path', async () => { + mockQuery.mockImplementation(async function* () { + yield { type: 'result', subtype: 'success', result: 'done', usage: {} }; + }); + const config = { + ...baseConfig, + model: { ...baseConfig.model, parameters: { includePartialMessages: false } }, + }; + const handler = createClaudeAgentsHandler(); + await collectStream(handler.stream?.(config as any, 'q', {}, {})); + const { options } = mockQuery.mock.calls[0][0]; + expect(options.includePartialMessages).toBe(true); + }); + it('sets span attributes and puts content on attributes when enabled', async () => { mockQuery.mockImplementation(async function* () { yield { type: 'result', subtype: 'success', result: 'ok', usage: { input_tokens: 2, output_tokens: 1 } }; diff --git a/packages/claude-agents/src/__tests__/native-graph.test.ts b/packages/claude-agents/src/__tests__/native-graph.test.ts index 1faca814..71606c82 100644 --- a/packages/claude-agents/src/__tests__/native-graph.test.ts +++ b/packages/claude-agents/src/__tests__/native-graph.test.ts @@ -47,6 +47,7 @@ vi.mock('@launchdarkly/ai-server', async (importOriginal) => { import { NativeTool } from '@launchdarkly/ai-server'; import { toClaudeAgents } from '../native-graph.js'; +import { expectNoNeverForwardedValue, NEVER_FORWARDED_PARAMETERS } from './never-forwarded.js'; // ─── Fixtures ───────────────────────────────────────────────────────────────── @@ -110,6 +111,45 @@ describe('toClaudeAgents', () => { } }); + it("forwards each node's allowlisted model.parameters, including maxTurns, to that node's query()", async () => { + const def = makeGraphDef(); + (def.root!.config.model as any).parameters = { + max_turns: 3, + thinking: { type: 'enabled', budget_tokens: 2048 }, + effort: 'low', + }; + (def.getNode('leaf')!.config.model as any).parameters = { max_turns: 6, max_budget_usd: 0.5 }; + await toClaudeAgents(Promise.resolve(def)).invoke('hi'); + + const rootOptions = mockQuery.mock.calls[0][0].options; + expect(rootOptions).toMatchObject({ + maxTurns: 3, + thinking: { type: 'enabled', budgetTokens: 2048 }, + effort: 'low', + model: 'claude-opus-4-5', + }); + + // The leaf runs when the root calls its sub-agent tool; call that tool's handler directly. + const leafTool = mockTool.mock.calls.find((call) => call[0] === 'leaf'); + await (leafTool![3] as (args: { input: string }) => Promise)({ input: 'sub task' }); + const leafOptions = mockQuery.mock.calls[1][0].options; + expect(leafOptions).toMatchObject({ maxTurns: 6, maxBudgetUsd: 0.5 }); + expect(leafOptions).not.toHaveProperty('max_turns'); + }); + + it('never forwards credentials, endpoints, remote tools or host-process settings from any node', async () => { + const def = makeGraphDef(); + (def.root!.config.model as any).parameters = NEVER_FORWARDED_PARAMETERS; + (def.getNode('leaf')!.config.model as any).parameters = NEVER_FORWARDED_PARAMETERS; + await toClaudeAgents(Promise.resolve(def)).invoke('hi'); + const leafTool = mockTool.mock.calls.find((call) => call[0] === 'leaf'); + await (leafTool![3] as (args: { input: string }) => Promise)({ input: 'sub task' }); + expect(mockQuery).toHaveBeenCalledTimes(2); + for (const call of mockQuery.mock.calls) { + expectNoNeverForwardedValue(call[0].options, ['mcpServers', 'hooks']); + } + }); + it('throws when the graph is disabled', async () => { const def = makeGraphDef({ enabled: false }); await expect( diff --git a/packages/claude-agents/src/__tests__/never-forwarded.ts b/packages/claude-agents/src/__tests__/never-forwarded.ts new file mode 100644 index 00000000..300d6e51 --- /dev/null +++ b/packages/claude-agents/src/__tests__/never-forwarded.ts @@ -0,0 +1,163 @@ +import { expect } from 'vitest'; + +/** + * The value every never-forwarded key carries in `NEVER_FORWARDED_PARAMETERS`. A handler that + * passes any of those keys on, under any name or nested anywhere, leaves this string somewhere in + * the options it builds, which is what `expectNoNeverForwardedValue` looks for. + */ +export const NEVER_FORWARDED = 'NEVER_FORWARDED'; + +/** + * Keys that must never reach a provider, framework or host-process option from + * `model.parameters`, in both the snake_case the LaunchDarkly UI writes and the camelCase the + * frameworks read: credentials, endpoints and connection settings, request injection, remote + * tools, and Claude Agents host-process settings. + */ +export const NEVER_FORWARDED_KEYS = [ + // credentials + 'api_key', + 'apiKey', + 'openai_api_key', + 'openAIApiKey', + 'anthropic_api_key', + 'anthropicApiKey', + 'bedrock_api_key', + 'bedrockApiKey', + 'bedrock_api_secret', + 'bedrockApiSecret', + 'credentials', + 'auth_token', + 'authToken', + 'access_token', + 'accessToken', + 'aws_access_key_id', + 'awsAccessKeyId', + 'aws_secret_access_key', + 'awsSecretAccessKey', + 'aws_session_token', + 'awsSessionToken', + 'organization', + // endpoints and connection + 'base_url', + 'baseURL', + 'baseUrl', + 'anthropic_api_url', + 'anthropicApiUrl', + 'endpoint', + 'endpoint_host', + 'endpointHost', + 'endpoint_url', + 'endpointUrl', + 'region', + 'region_name', + 'regionName', + 'configuration', + 'client_options', + 'clientOptions', + 'default_headers', + 'defaultHeaders', + 'timeout', + 'max_retries', + 'maxRetries', + 'fetch', + 'http_client', + 'httpClient', + 'http_agent', + 'httpAgent', + 'proxy', + 'proxies', + 'client', + // request injection + 'headers', + 'extra_headers', + 'extraHeaders', + 'extra_body', + 'extraBody', + 'extra_query', + 'extraQuery', + 'model_kwargs', + 'modelKwargs', + 'invocation_kwargs', + 'invocationKwargs', + 'additional_model_request_fields', + 'additionalModelRequestFields', + 'provider_data', + 'providerData', + 'provider_options', + 'providerOptions', + // remote tools + 'mcp_servers', + 'mcpServers', + // Claude Agents host-process settings + 'path_to_claude_code_executable', + 'pathToClaudeCodeExecutable', + 'cli_path', + 'cliPath', + 'executable', + 'executable_args', + 'executableArgs', + 'extra_args', + 'extraArgs', + 'env', + 'cwd', + 'additional_directories', + 'additionalDirectories', + 'add_dirs', + 'addDirs', + 'permission_mode', + 'permissionMode', + 'allow_dangerously_skip_permissions', + 'allowDangerouslySkipPermissions', + 'settings', + 'setting_sources', + 'settingSources', + 'managed_settings', + 'managedSettings', + 'plugins', + 'sandbox', + 'resume', + 'resume_session_at', + 'resumeSessionAt', + 'session_id', + 'sessionId', + 'fork_session', + 'forkSession', + 'continue', + 'hooks', + 'can_use_tool', + 'canUseTool', + 'stderr', + 'abort_controller', + 'abortController', + 'spawn_claude_code_process', + 'spawnClaudeCodeProcess', +] as const; + +/** A `model.parameters` bag that sets every key in `NEVER_FORWARDED_KEYS` to `NEVER_FORWARDED`. */ +export const NEVER_FORWARDED_PARAMETERS: Record = Object.fromEntries( + NEVER_FORWARDED_KEYS.map((key) => [key, NEVER_FORWARDED]), +); + +function findNeverForwarded(value: unknown, path: string, seen: Set): string | undefined { + if (value === NEVER_FORWARDED) return path || '(root)'; + if (!value || typeof value !== 'object' || seen.has(value)) return undefined; + seen.add(value); + for (const [key, child] of Object.entries(value)) { + const found = findNeverForwarded(child, path ? `${path}.${key}` : key, seen); + if (found) return found; + } + return undefined; +} + +/** + * Asserts that no `NEVER_FORWARDED` value appears anywhere in `options`, and that none of the + * `NEVER_FORWARDED_KEYS` the handler does not own itself is set on it. `handlerOwned` lists the + * keys the handler legitimately sets from its own state (Claude Agents sets `mcpServers` and + * `hooks`, for example); those must still not carry the config's value. + */ +export function expectNoNeverForwardedValue(options: unknown, handlerOwned: ReadonlyArray = []): void { + expect(findNeverForwarded(options, '', new Set())).toBeUndefined(); + const record = options as Record; + const leaked = NEVER_FORWARDED_KEYS.filter((key) => !handlerOwned.includes(key) && record[key] !== undefined); + expect(leaked).toEqual([]); +} diff --git a/packages/claude-agents/src/handler.ts b/packages/claude-agents/src/handler.ts index ebf30c1d..9c1f538a 100644 --- a/packages/claude-agents/src/handler.ts +++ b/packages/claude-agents/src/handler.ts @@ -34,6 +34,7 @@ import { } from '@launchdarkly/ai-server'; import { type Context, context, type Span, SpanStatusCode, trace } from '@opentelemetry/api'; import { z } from 'zod'; +import { buildModelParameterQueryOptions } from './model-parameters.js'; const TOOL_MCP_NAME = 'tool-mcp'; const MCP_TOOL_PREFIX = `mcp__${TOOL_MCP_NAME}__`; @@ -892,19 +893,29 @@ function buildQueryOptions( toolMCP: any, // biome-ignore lint/suspicious/noExplicitAny: Claude SDK hooks config type is not publicly exported hooks: any, - extra?: Record, + /** + * The allowlisted subset of `model.parameters`, from `buildModelParameterQueryOptions`. + * It comes from the AI Config, so it is spread FIRST and every handler-owned key below + * overrides it. Passed explicitly rather than folded into `internalOptions` so the two + * trust levels cannot be confused at a call site. + */ + modelParameters: Record, + /** Options this handler sets for itself, e.g. `includePartialMessages` on the streaming + * path. Trusted, so these are spread LAST and win over everything. */ + internalOptions?: Record, ) { const allAllowedTools = [...mcpAllowedTools, ...nativeToolNames]; return { prompt, options: { + ...modelParameters, model: config.model.name, tools: nativeToolNames.length > 0 ? nativeToolNames : [], allowedTools: allAllowedTools.length > 0 ? allAllowedTools : undefined, mcpServers: toolMCP ? { [TOOL_MCP_NAME]: toolMCP } : undefined, hooks, - ...(systemPrompt ? { systemPrompt } : {}), - ...extra, + systemPrompt, + ...internalOptions, }, }; } @@ -981,6 +992,7 @@ export function createClaudeAgentsHandler({ captureContent = false }: ContentCap mcpAllowedTools, toolMCP, toolTelemetry?.hooks, + buildModelParameterQueryOptions(config.model.parameters), ), )) { recordConversationId(span, message); @@ -1107,9 +1119,8 @@ export function createClaudeAgentsHandler({ captureContent = false }: ContentCap mcpAllowedTools, toolMCP, toolTelemetry?.hooks, - { - includePartialMessages: true, - }, + buildModelParameterQueryOptions(config.model.parameters), + { includePartialMessages: true }, ), )) { recordConversationId(span, message); diff --git a/packages/claude-agents/src/model-parameters.ts b/packages/claude-agents/src/model-parameters.ts new file mode 100644 index 00000000..3378b159 --- /dev/null +++ b/packages/claude-agents/src/model-parameters.ts @@ -0,0 +1,84 @@ +import type { Options } from '@anthropic-ai/claude-agent-sdk'; +import { + type AiConfigRep, + camelizeModelParameters, + normalizeModelParameters, + pickForwardedModelParameters, +} from '@launchdarkly/ai-server'; + +/** + * `query()` options this package takes from `config.model.parameters`, after camelizing the + * bag (`max_turns` becomes `maxTurns`). Every other key is dropped. + * + * This is an allowlist because `model.parameters` comes from the AI Config, not from the + * application, and most `query()` options configure the host process rather than the model: + * which executable runs (`pathToClaudeCodeExecutable`, `executable`, `executableArgs`, + * `extraArgs`), its environment and working directory (`env`, `cwd`, `additionalDirectories`), + * what it may do without asking (`permissionMode`, `allowDangerouslySkipPermissions`, `sandbox`, + * `settings`, `settingSources`, `plugins`, `hooks`, `canUseTool`), which remote tools it connects + * to (`mcpServers`), and which session it resumes (`resume`, `sessionId`, `forkSession`, + * `continue`). None of those can come from a config. + */ +const FORWARDED_QUERY_OPTION_KEYS = [ + 'betas', + 'effort', + 'fallbackModel', + 'maxBudgetUsd', + 'maxThinkingTokens', + 'maxTurns', + 'outputFormat', + 'thinking', +] as const satisfies ReadonlyArray; + +/** + * `thinking` as the Claude Agent SDK reads it. The LaunchDarkly UI writes the Messages API shape, + * `{ type: 'enabled', budget_tokens: 1024 }`, but the SDK's `ThinkingEnabled` reads + * `budgetTokens`, so a snake_case budget would be ignored. Only `type`, the budget, and `display` + * are kept; anything that is not an object with a string `type` is dropped. + */ +function toThinkingOption(value: unknown): Options['thinking'] | undefined { + if (!value || typeof value !== 'object' || Array.isArray(value)) return undefined; + const source = value as Record; + if (typeof source.type !== 'string') return undefined; + const thinking: Record = { type: source.type }; + const budgetTokens = source.budget_tokens ?? source.budgetTokens; + if (typeof budgetTokens === 'number') thinking.budgetTokens = budgetTokens; + if (typeof source.display === 'string') thinking.display = source.display; + return thinking as Options['thinking']; +} + +/** + * `outputFormat` as the Claude Agent SDK reads it: `{ type: 'json_schema', schema }`, with the + * schema passed through as written. Anything else is dropped. + */ +function toOutputFormatOption(value: unknown): Options['outputFormat'] | undefined { + if (!value || typeof value !== 'object' || Array.isArray(value)) return undefined; + const source = value as Record; + if (source.type !== 'json_schema' || !source.schema || typeof source.schema !== 'object') return undefined; + return { type: 'json_schema', schema: source.schema as Record }; +} + +/** + * `config.model.parameters` as `query()` options: camelized, narrowed to + * `FORWARDED_QUERY_OPTION_KEYS`, with `thinking` and `outputFormat` rebuilt in the shapes the SDK + * reads. A config that sets none of them produces `{}`. + */ +export function buildModelParameterQueryOptions( + parameters: AiConfigRep['model']['parameters'] | unknown, +): Partial { + const options = pickForwardedModelParameters( + camelizeModelParameters(normalizeModelParameters(parameters)), + FORWARDED_QUERY_OPTION_KEYS, + ); + if (options.thinking !== undefined) { + const thinking = toThinkingOption(options.thinking); + if (thinking) options.thinking = thinking; + else delete options.thinking; + } + if (options.outputFormat !== undefined) { + const outputFormat = toOutputFormatOption(options.outputFormat); + if (outputFormat) options.outputFormat = outputFormat; + else delete options.outputFormat; + } + return options as Partial; +} diff --git a/packages/claude-agents/src/native-graph.ts b/packages/claude-agents/src/native-graph.ts index d6b7d87f..2c8fc412 100644 --- a/packages/claude-agents/src/native-graph.ts +++ b/packages/claude-agents/src/native-graph.ts @@ -15,6 +15,7 @@ import { import { SpanStatusCode, trace } from '@opentelemetry/api'; import { z } from 'zod'; import { buildPrompt, buildQueryPrompt, buildToolMCP, partitionTools } from './handler.js'; +import { buildModelParameterQueryOptions } from './model-parameters.js'; const TOOL_MCP_NAME = 'tool-mcp'; const SUBAGENT_MCP_NAME = 'subagents'; @@ -110,6 +111,9 @@ const runQuery = async ( for await (const message of query({ prompt: queryPrompt, options: { + // The node's allowlisted model.parameters (maxTurns, thinking, effort, ...), spread first so + // every handler-owned option below wins. + ...buildModelParameterQueryOptions(node.config.model.parameters), model: node.config.model.name, tools: nativeToolNames.length > 0 ? nativeToolNames : [], allowedTools: allAllowedTools.length > 0 ? allAllowedTools : undefined, diff --git a/packages/claude-messages/src/__tests__/handler.test.ts b/packages/claude-messages/src/__tests__/handler.test.ts index 7cc6f863..81710490 100644 --- a/packages/claude-messages/src/__tests__/handler.test.ts +++ b/packages/claude-messages/src/__tests__/handler.test.ts @@ -58,6 +58,7 @@ vi.mock('@launchdarkly/ai-server', async (importOriginal) => { }); import { createClaudeMessagesHandler } from '../handler.js'; +import { expectNoNeverForwardedValue, NEVER_FORWARDED_PARAMETERS } from './never-forwarded.js'; // ─── Fixtures ───────────────────────────────────────────────────────────────── @@ -1349,3 +1350,249 @@ describe('createClaudeMessagesHandler — MAX_STEPS cap (§1.10)', () => { ); }); }); + +describe('createClaudeMessagesHandler — model.parameters forwarding', () => { + beforeEach(() => { + vi.clearAllMocks(); + mockChildSpans.length = 0; + }); + + function makeStreamMock(events: any[], finalMsg: any) { + const asyncIterable = (async function* () { + for (const e of events) yield e; + })(); + return { + [Symbol.asyncIterator]: () => asyncIterable[Symbol.asyncIterator](), + finalMessage: vi.fn().mockResolvedValue(finalMsg), + }; + } + + it('forwards recognized keys (temperature, top_p, top_k, stop_sequences, thinking) to messages.create', async () => { + mockMessagesCreate.mockResolvedValue(mockFinalResponse()); + const cfg = { + ...baseConfig, + model: { + ...baseConfig.model, + parameters: { + temperature: 0.5, + top_p: 0.9, + top_k: 40, + stop_sequences: ['STOP'], + thinking: { type: 'enabled', budget_tokens: 2048 }, + }, + }, + }; + const handler = createClaudeMessagesHandler(); + await handler(cfg as any, 'hi'); + const call = mockMessagesCreate.mock.calls[0][0]; + expect(call.temperature).toBe(0.5); + expect(call.top_p).toBe(0.9); + expect(call.top_k).toBe(40); + expect(call.stop_sequences).toEqual(['STOP']); + expect(call.thinking).toEqual({ type: 'enabled', budget_tokens: 2048 }); + }); + + it('still defaults max_tokens to 1024 when model.parameters is set but omits it', async () => { + mockMessagesCreate.mockResolvedValue(mockFinalResponse()); + const cfg = { ...baseConfig, model: { ...baseConfig.model, parameters: { temperature: 0.3 } } }; + const handler = createClaudeMessagesHandler(); + await handler(cfg as any, 'hi'); + expect(mockMessagesCreate.mock.calls[0][0].max_tokens).toBe(1024); + }); + + it('forwards a configured max_tokens unchanged (existing behaviour)', async () => { + mockMessagesCreate.mockResolvedValue(mockFinalResponse()); + const cfg = { ...baseConfig, model: { ...baseConfig.model, parameters: { max_tokens: 2048 } } }; + const handler = createClaudeMessagesHandler(); + await handler(cfg as any, 'hi'); + expect(mockMessagesCreate.mock.calls[0][0].max_tokens).toBe(2048); + }); + + // This handler wraps the raw @anthropic-ai/sdk client, which reads snake_case keys itself, so + // (unlike the four framework handlers) the UI's snake_case must reach messages.create untouched + // rather than being camelized. + it('forwards snake_case model.parameters keys unchanged, not camelized', async () => { + mockMessagesCreate.mockResolvedValue(mockFinalResponse()); + const cfg = { + ...baseConfig, + model: { ...baseConfig.model, parameters: { max_tokens: 300, top_p: 0.8, stop_sequences: ['STOP'] } }, + }; + const handler = createClaudeMessagesHandler(); + await handler(cfg as any, 'hi'); + const call = mockMessagesCreate.mock.calls[0][0]; + expect(call.max_tokens).toBe(300); + expect(call.top_p).toBe(0.8); + expect(call.stop_sequences).toEqual(['STOP']); + expect(call.maxTokens).toBeUndefined(); + expect(call.topP).toBeUndefined(); + }); + + it('does not let model.parameters override model, messages, tools, or system', async () => { + mockMessagesCreate.mockResolvedValue(mockFinalResponse()); + const cfg = { + ...baseConfig, + model: { + ...baseConfig.model, + parameters: { + model: 'evil-model', + messages: ['evil'], + tools: ['evil'], + system: 'evil system', + stream: true, + }, + }, + }; + const handler = createClaudeMessagesHandler(); + await handler(cfg as any, 'hi'); + const call = mockMessagesCreate.mock.calls[0][0]; + expect(call.model).toBe(baseConfig.model.name); + expect(call.system).toBe('You are helpful.'); + expect(Array.isArray(call.messages)).toBe(true); + expect(call.messages.at(-1).content).toBe('hi'); + expect(call.tools).toBeUndefined(); + }); + + it('never forwards credentials, endpoints, request overrides or remote tools from model.parameters', async () => { + mockMessagesCreate.mockResolvedValue(mockFinalResponse()); + const cfg = { ...baseConfig, model: { ...baseConfig.model, parameters: NEVER_FORWARDED_PARAMETERS } }; + await createClaudeMessagesHandler()(cfg as any, 'hi'); + expect(mockMessagesCreate).toHaveBeenCalledTimes(1); + for (const arg of mockMessagesCreate.mock.calls[0]) { + expectNoNeverForwardedValue(arg); + } + }); + + it('drops an unrecognized model.parameters key rather than forwarding it', async () => { + // Unlike the four framework handlers, this handler wraps the raw Anthropic SDK client: an + // unsupported key reaching messages.create is a wire-level 400, not a harmlessly ignored extra. + mockMessagesCreate.mockResolvedValue(mockFinalResponse()); + const cfg = { ...baseConfig, model: { ...baseConfig.model, parameters: { made_up_key: 'nope' } } }; + const handler = createClaudeMessagesHandler(); + await handler(cfg as any, 'hi'); + expect(mockMessagesCreate.mock.calls[0][0]).not.toHaveProperty('made_up_key'); + }); + + it('drops a UI-offered key the Messages API top level does not accept', async () => { + // The LaunchDarkly UI offers top-level `effort`; the Messages API only takes it nested under + // `output_config`. Without the rename in buildModelParameterOptions this would 400. + mockMessagesCreate.mockResolvedValue(mockFinalResponse()); + const cfg = { ...baseConfig, model: { ...baseConfig.model, parameters: { effort: 'high' } } }; + const handler = createClaudeMessagesHandler(); + await handler(cfg as any, 'hi'); + const call = mockMessagesCreate.mock.calls[0][0]; + expect(call).not.toHaveProperty('effort'); + expect(call.output_config).toEqual({ effort: 'high' }); + }); + + it('an explicit output_config.effort in the config wins over the top-level effort rename', async () => { + mockMessagesCreate.mockResolvedValue(mockFinalResponse()); + const cfg = { + ...baseConfig, + model: { ...baseConfig.model, parameters: { effort: 'high', output_config: { effort: 'low' } } }, + }; + const handler = createClaudeMessagesHandler(); + await handler(cfg as any, 'hi'); + expect(mockMessagesCreate.mock.calls[0][0].output_config).toEqual({ effort: 'low' }); + }); + + it('merges a renamed effort into an output_config that also sets other fields', async () => { + mockMessagesCreate.mockResolvedValue(mockFinalResponse()); + const cfg = { + ...baseConfig, + model: { ...baseConfig.model, parameters: { effort: 'high', output_config: { format: { type: 'text' } } } }, + }; + const handler = createClaudeMessagesHandler(); + await handler(cfg as any, 'hi'); + expect(mockMessagesCreate.mock.calls[0][0].output_config).toEqual({ effort: 'high', format: { type: 'text' } }); + }); + + it('leaves the call unchanged from today when model.parameters is absent', async () => { + mockMessagesCreate.mockResolvedValue(mockFinalResponse()); + const handler = createClaudeMessagesHandler(); + await handler(baseConfig as any, 'hi'); + const call = mockMessagesCreate.mock.calls[0][0]; + expect(call).toEqual({ + model: baseConfig.model.name, + max_tokens: 1024, + system: 'You are helpful.', + messages: [{ role: 'user', content: 'hi' }], + }); + }); + + it('leaves the call unchanged from today when model.parameters is an empty object', async () => { + mockMessagesCreate.mockResolvedValue(mockFinalResponse()); + const cfg = { ...baseConfig, model: { ...baseConfig.model, parameters: {} } }; + const handler = createClaudeMessagesHandler(); + await handler(cfg as any, 'hi'); + const call = mockMessagesCreate.mock.calls[0][0]; + expect(call).toEqual({ + model: baseConfig.model.name, + max_tokens: 1024, + system: 'You are helpful.', + messages: [{ role: 'user', content: 'hi' }], + }); + }); + + // ── streaming path ────────────────────────────────────────────────────────── + + it('forwards recognized model.parameters to messages.stream', async () => { + const finalMsg = { + usage: { input_tokens: 3, output_tokens: 7 }, + stop_reason: 'end_turn', + content: [{ type: 'text', text: 'Hi' }], + }; + mockMessagesStream.mockReturnValue(makeStreamMock([], finalMsg)); + const cfg = { ...baseConfig, model: { ...baseConfig.model, parameters: { temperature: 0.7, top_k: 10 } } }; + const handler = createClaudeMessagesHandler(); + const gen = handler.stream?.(cfg as any, 'q', {}, {}); + for await (const _e of gen) { + // drain + } + const call = mockMessagesStream.mock.calls[0][0]; + expect(call.temperature).toBe(0.7); + expect(call.top_k).toBe(10); + }); + + it('streaming path still defaults max_tokens to 1024 and drops an unrecognized key', async () => { + const finalMsg = { + usage: { input_tokens: 3, output_tokens: 7 }, + stop_reason: 'end_turn', + content: [{ type: 'text', text: 'Hi' }], + }; + mockMessagesStream.mockReturnValue(makeStreamMock([], finalMsg)); + const cfg = { ...baseConfig, model: { ...baseConfig.model, parameters: { made_up_key: 'nope' } } }; + const handler = createClaudeMessagesHandler(); + const gen = handler.stream?.(cfg as any, 'q', {}, {}); + for await (const _e of gen) { + // drain + } + const call = mockMessagesStream.mock.calls[0][0]; + expect(call.max_tokens).toBe(1024); + expect(call).not.toHaveProperty('made_up_key'); + }); + + it('streaming path does not let model.parameters override model/messages/system', async () => { + const finalMsg = { + usage: { input_tokens: 3, output_tokens: 7 }, + stop_reason: 'end_turn', + content: [{ type: 'text', text: 'Hi' }], + }; + mockMessagesStream.mockReturnValue(makeStreamMock([], finalMsg)); + const cfg = { + ...baseConfig, + model: { + ...baseConfig.model, + parameters: { model: 'evil-model', system: 'evil system', messages: ['evil'] }, + }, + }; + const handler = createClaudeMessagesHandler(); + const gen = handler.stream?.(cfg as any, 'q', {}, {}); + for await (const _e of gen) { + // drain + } + const call = mockMessagesStream.mock.calls[0][0]; + expect(call.model).toBe(baseConfig.model.name); + expect(call.system).toBe('You are helpful.'); + expect(call.messages.at(-1).content).toBe('q'); + }); +}); diff --git a/packages/claude-messages/src/__tests__/never-forwarded.ts b/packages/claude-messages/src/__tests__/never-forwarded.ts new file mode 100644 index 00000000..300d6e51 --- /dev/null +++ b/packages/claude-messages/src/__tests__/never-forwarded.ts @@ -0,0 +1,163 @@ +import { expect } from 'vitest'; + +/** + * The value every never-forwarded key carries in `NEVER_FORWARDED_PARAMETERS`. A handler that + * passes any of those keys on, under any name or nested anywhere, leaves this string somewhere in + * the options it builds, which is what `expectNoNeverForwardedValue` looks for. + */ +export const NEVER_FORWARDED = 'NEVER_FORWARDED'; + +/** + * Keys that must never reach a provider, framework or host-process option from + * `model.parameters`, in both the snake_case the LaunchDarkly UI writes and the camelCase the + * frameworks read: credentials, endpoints and connection settings, request injection, remote + * tools, and Claude Agents host-process settings. + */ +export const NEVER_FORWARDED_KEYS = [ + // credentials + 'api_key', + 'apiKey', + 'openai_api_key', + 'openAIApiKey', + 'anthropic_api_key', + 'anthropicApiKey', + 'bedrock_api_key', + 'bedrockApiKey', + 'bedrock_api_secret', + 'bedrockApiSecret', + 'credentials', + 'auth_token', + 'authToken', + 'access_token', + 'accessToken', + 'aws_access_key_id', + 'awsAccessKeyId', + 'aws_secret_access_key', + 'awsSecretAccessKey', + 'aws_session_token', + 'awsSessionToken', + 'organization', + // endpoints and connection + 'base_url', + 'baseURL', + 'baseUrl', + 'anthropic_api_url', + 'anthropicApiUrl', + 'endpoint', + 'endpoint_host', + 'endpointHost', + 'endpoint_url', + 'endpointUrl', + 'region', + 'region_name', + 'regionName', + 'configuration', + 'client_options', + 'clientOptions', + 'default_headers', + 'defaultHeaders', + 'timeout', + 'max_retries', + 'maxRetries', + 'fetch', + 'http_client', + 'httpClient', + 'http_agent', + 'httpAgent', + 'proxy', + 'proxies', + 'client', + // request injection + 'headers', + 'extra_headers', + 'extraHeaders', + 'extra_body', + 'extraBody', + 'extra_query', + 'extraQuery', + 'model_kwargs', + 'modelKwargs', + 'invocation_kwargs', + 'invocationKwargs', + 'additional_model_request_fields', + 'additionalModelRequestFields', + 'provider_data', + 'providerData', + 'provider_options', + 'providerOptions', + // remote tools + 'mcp_servers', + 'mcpServers', + // Claude Agents host-process settings + 'path_to_claude_code_executable', + 'pathToClaudeCodeExecutable', + 'cli_path', + 'cliPath', + 'executable', + 'executable_args', + 'executableArgs', + 'extra_args', + 'extraArgs', + 'env', + 'cwd', + 'additional_directories', + 'additionalDirectories', + 'add_dirs', + 'addDirs', + 'permission_mode', + 'permissionMode', + 'allow_dangerously_skip_permissions', + 'allowDangerouslySkipPermissions', + 'settings', + 'setting_sources', + 'settingSources', + 'managed_settings', + 'managedSettings', + 'plugins', + 'sandbox', + 'resume', + 'resume_session_at', + 'resumeSessionAt', + 'session_id', + 'sessionId', + 'fork_session', + 'forkSession', + 'continue', + 'hooks', + 'can_use_tool', + 'canUseTool', + 'stderr', + 'abort_controller', + 'abortController', + 'spawn_claude_code_process', + 'spawnClaudeCodeProcess', +] as const; + +/** A `model.parameters` bag that sets every key in `NEVER_FORWARDED_KEYS` to `NEVER_FORWARDED`. */ +export const NEVER_FORWARDED_PARAMETERS: Record = Object.fromEntries( + NEVER_FORWARDED_KEYS.map((key) => [key, NEVER_FORWARDED]), +); + +function findNeverForwarded(value: unknown, path: string, seen: Set): string | undefined { + if (value === NEVER_FORWARDED) return path || '(root)'; + if (!value || typeof value !== 'object' || seen.has(value)) return undefined; + seen.add(value); + for (const [key, child] of Object.entries(value)) { + const found = findNeverForwarded(child, path ? `${path}.${key}` : key, seen); + if (found) return found; + } + return undefined; +} + +/** + * Asserts that no `NEVER_FORWARDED` value appears anywhere in `options`, and that none of the + * `NEVER_FORWARDED_KEYS` the handler does not own itself is set on it. `handlerOwned` lists the + * keys the handler legitimately sets from its own state (Claude Agents sets `mcpServers` and + * `hooks`, for example); those must still not carry the config's value. + */ +export function expectNoNeverForwardedValue(options: unknown, handlerOwned: ReadonlyArray = []): void { + expect(findNeverForwarded(options, '', new Set())).toBeUndefined(); + const record = options as Record; + const leaked = NEVER_FORWARDED_KEYS.filter((key) => !handlerOwned.includes(key) && record[key] !== undefined); + expect(leaked).toEqual([]); +} diff --git a/packages/claude-messages/src/__tests__/wire.test.ts b/packages/claude-messages/src/__tests__/wire.test.ts new file mode 100644 index 00000000..04e73620 --- /dev/null +++ b/packages/claude-messages/src/__tests__/wire.test.ts @@ -0,0 +1,79 @@ +// ─── Wire-level tests ─────────────────────────────────────────────────────── +// +// Unlike handler.test.ts, this file does NOT mock `@anthropic-ai/sdk`. It lets the real +// Anthropic SDK build a request and intercepts the outgoing HTTP call via a stubbed global +// `fetch`, so these tests prove the JSON body actually sent to the Messages API — not just +// what reaches a mocked `messages.create`. + +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +function mockFetchResponse(body: unknown) { + return new Response(JSON.stringify(body), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); +} + +function finalMessageBody() { + return { + id: 'msg_1', + type: 'message', + role: 'assistant', + model: 'claude-3-5-sonnet-20241022', + content: [{ type: 'text', text: 'hi' }], + stop_reason: 'end_turn', + stop_sequence: null, + usage: { input_tokens: 1, output_tokens: 1 }, + }; +} + +describe('createClaudeMessagesHandler — wire-level request body', () => { + let capturedBody: Record | undefined; + let fetchMock: ReturnType; + + beforeEach(async () => { + vi.resetModules(); + process.env.ANTHROPIC_API_KEY = 'test-key'; + capturedBody = undefined; + fetchMock = vi.fn(async (_url: unknown, init: RequestInit) => { + capturedBody = JSON.parse(init.body as string); + return mockFetchResponse(finalMessageBody()); + }); + vi.stubGlobal('fetch', fetchMock); + }); + + afterEach(() => { + vi.unstubAllGlobals(); + }); + + it('moves a UI-offered top-level effort into output_config.effort and drops a key the Messages API top level does not accept', async () => { + const { createClaudeMessagesHandler } = await import('../handler.js'); + const config = { + model: { name: 'claude-3-5-sonnet-20241022', parameters: { effort: 'high' } }, + provider: { name: 'Anthropic' }, + instructions: 'You are helpful.', + }; + + const handler = createClaudeMessagesHandler(); + await handler(config as any, 'hi'); + + expect(fetchMock).toHaveBeenCalled(); + expect(capturedBody).not.toHaveProperty('effort'); + expect(capturedBody?.output_config).toEqual({ effort: 'high' }); + }); + + it('forwards an accepted key (temperature) and drops an unrecognized one at the wire', async () => { + const { createClaudeMessagesHandler } = await import('../handler.js'); + const config = { + model: { name: 'claude-3-5-sonnet-20241022', parameters: { temperature: 0.3, made_up_key: 'nope' } }, + provider: { name: 'Anthropic' }, + instructions: 'You are helpful.', + }; + + const handler = createClaudeMessagesHandler(); + await handler(config as any, 'hi'); + + expect(capturedBody?.temperature).toBe(0.3); + expect(capturedBody).not.toHaveProperty('made_up_key'); + }); +}); diff --git a/packages/claude-messages/src/handler.ts b/packages/claude-messages/src/handler.ts index e9d41b84..2d94735d 100644 --- a/packages/claude-messages/src/handler.ts +++ b/packages/claude-messages/src/handler.ts @@ -1,4 +1,5 @@ import Anthropic from '@anthropic-ai/sdk'; +import type { MessageCreateParamsBase } from '@anthropic-ai/sdk/resources/messages/messages'; import { type AiConfigRep, addCachedTokensToInput, @@ -13,8 +14,10 @@ import { type Message, type MessageContent, type NativeTool, + normalizeModelParameters, type ProviderHandler, parseTemplate, + pickForwardedModelParameters, type SpanMessage, type SpanMessagePart, setInputContentAttributes, @@ -338,6 +341,70 @@ const buildMessages = ( return { messages, system }; }; +/** + * `MessageCreateParamsBase` keys this handler forwards verbatim from `config.model.parameters` + * (after the `effort` rename below is folded into `output_config`), beyond `max_tokens`, which is + * handled separately because it carries a default. + * + * The rule for this list: exclude a key only if setting it would BREAK the handler; forward + * everything else the API accepts, even settings with no obvious generation effect — those are + * forwarded because a config that sets one still gets a working call. + * + * Handler-owned (this handler sets these itself, from the config and the call shape, so a + * `model.parameters` value must not be able to override what it already decided): `model`, + * `messages`, `system`, `tools`, `max_tokens`. + * + * Excluded (would break the handler): `stream` — this handler selects streaming by choosing + * between `messages.create()` and `messages.stream()`, not by setting a field on the request + * body, so a config value here would fight the method actually invoked rather than configure + * anything. + */ +const FORWARDED_MODEL_PARAMETER_KEYS = [ + 'cache_control', + 'container', + 'inference_geo', + 'metadata', + 'output_config', + 'service_tier', + 'stop_sequences', + 'temperature', + 'thinking', + 'tool_choice', + 'top_k', + 'top_p', +] as const; + +type AnthropicHandlerOwnedKeys = 'max_tokens' | 'messages' | 'model' | 'system' | 'tools'; +type AnthropicExcludedKeys = 'stream'; +// If a key of MessageCreateParamsBase is added to the SDK and not classified above as forwarded, +// handler-owned, or excluded, this type resolves to something other than `never` and the +// assignment below fails to compile, naming the unclassified key. +type AnthropicUndecidedModelParameterKeys = Exclude< + keyof MessageCreateParamsBase, + AnthropicHandlerOwnedKeys | AnthropicExcludedKeys | (typeof FORWARDED_MODEL_PARAMETER_KEYS)[number] +>; +const _anthropicModelParameterKeysExhaustive: Record = {} as Record< + never, + never +>; + +/** + * Picks the subset of `config.model.parameters` that maps onto `MessageCreateParamsBase`, after + * moving a top-level `effort` — the shape the LaunchDarkly UI writes — into `output_config.effort`, + * the shape the Messages API actually accepts. An `output_config.effort` already present in the + * config wins over the renamed value, since it is what the customer set explicitly for that field. + * A config that sets nothing here produces `{}`, so the provider call sees exactly what it always + * has. + */ +function buildModelParameterOptions(parameters: AiConfigRep['model']['parameters']): Record { + const normalized = normalizeModelParameters(parameters); + const { effort, output_config, ...rest } = normalized; + const mergedOutputConfig = + effort !== undefined ? { effort, ...(output_config as Record | undefined) } : output_config; + const withEffortMoved = mergedOutputConfig !== undefined ? { ...rest, output_config: mergedOutputConfig } : rest; + return pickForwardedModelParameters(withEffortMoved, FORWARDED_MODEL_PARAMETER_KEYS); +} + const MAX_STEPS = 10; export function createClaudeMessagesHandler({ captureContent = false }: ContentCaptureOptions = {}): ProviderHandler { @@ -377,11 +444,12 @@ export function createClaudeMessagesHandler({ captureContent = false }: ContentC let response: Anthropic.Message; try { response = await anthropic.messages.create({ + ...buildModelParameterOptions(config.model.parameters), model: config.model.name, max_tokens: maxTokens, - ...(system ? { system } : {}), + system, messages: conversation, - ...(tools.length > 0 ? { tools } : {}), + tools: tools.length > 0 ? tools : undefined, }); } catch (err) { failSpan(modelSpan, err); @@ -539,11 +607,12 @@ export function createClaudeMessagesHandler({ captureContent = false }: ContentC let finalMsg: Anthropic.Message; try { const stream = anthropic.messages.stream({ + ...buildModelParameterOptions(config.model.parameters), model: config.model.name, max_tokens: maxTokens, - ...(system ? { system } : {}), + system, messages: conversation, - ...(tools.length > 0 ? { tools } : {}), + tools: tools.length > 0 ? tools : undefined, }); // Yield text deltas for this turn diff --git a/packages/client/src/__tests__/utils.test.ts b/packages/client/src/__tests__/utils.test.ts index 902e7d73..704a4f0f 100644 --- a/packages/client/src/__tests__/utils.test.ts +++ b/packages/client/src/__tests__/utils.test.ts @@ -1,11 +1,14 @@ import { describe, expect, it, vi } from 'vitest'; import { + camelizeModelParameters, createHandler, normalizeMode, + normalizeModelParameters, numberOrZero, parseJSONWithPossibleFences, parseTemplate, parseUsage, + pickForwardedModelParameters, setLdSpanAttributes, setUsageSpanAttributes, } from '../utils.js'; @@ -621,3 +624,126 @@ describe('setUsageSpanAttributes', () => { }); }); }); + +// ─── normalizeModelParameters ────────────────────────────────────────────────── + +describe('normalizeModelParameters', () => { + it('returns {} for undefined', () => { + expect(normalizeModelParameters(undefined)).toEqual({}); + }); + + it('returns {} for null', () => { + expect(normalizeModelParameters(null)).toEqual({}); + }); + + it('returns {} for a non-object value', () => { + expect(normalizeModelParameters('not an object')).toEqual({}); + }); + + it('returns the object unchanged when it is a usable object', () => { + const parameters = { temperature: 0.7, thinking: { type: 'enabled' } }; + expect(normalizeModelParameters(parameters)).toBe(parameters); + }); + + it('returns an empty array unchanged (arrays are objects)', () => { + expect(normalizeModelParameters([])).toEqual([]); + }); +}); + +// ─── pickForwardedModelParameters ───────────────────────────────────────────── + +describe('pickForwardedModelParameters', () => { + it('returns {} when parameters is undefined', () => { + expect(pickForwardedModelParameters(undefined, ['temperature'])).toEqual({}); + }); + + it('returns {} when parameters sets none of the allowed keys', () => { + expect(pickForwardedModelParameters({ made_up_key: 'nope' }, ['temperature'])).toEqual({}); + }); + + it('picks only the keys present in both parameters and the allowlist', () => { + const parameters = { temperature: 0.5, top_p: 0.9, made_up_key: 'nope' }; + expect(pickForwardedModelParameters(parameters, ['temperature', 'top_p'])).toEqual({ + temperature: 0.5, + top_p: 0.9, + }); + }); + + it('omits an allowed key whose value is undefined', () => { + const parameters = { temperature: undefined, top_p: 0.9 }; + expect(pickForwardedModelParameters(parameters, ['temperature', 'top_p'])).toEqual({ top_p: 0.9 }); + }); +}); + +// ─── camelizeModelParameters ─────────────────────────────────────────────────── + +describe('camelizeModelParameters', () => { + it('converts snake_case keys generated from the key itself', () => { + expect( + camelizeModelParameters({ + max_turns: 3, + top_p: 0.9, + max_output_tokens: 512, + frequency_penalty: 0.2, + }), + ).toEqual({ + maxTurns: 3, + topP: 0.9, + maxOutputTokens: 512, + frequencyPenalty: 0.2, + }); + }); + + it('leaves keys with no underscore unchanged', () => { + expect(camelizeModelParameters({ temperature: 0.7, store: true, maxTurns: 3 })).toEqual({ + temperature: 0.7, + store: true, + maxTurns: 3, + }); + }); + + it('converts only top-level keys, leaving nested values untouched', () => { + const input = { max_turns: 3, thinking: { type: 'enabled', budget_tokens: 1024 } }; + expect(camelizeModelParameters(input)).toEqual({ + maxTurns: 3, + thinking: { type: 'enabled', budget_tokens: 1024 }, + }); + }); + + it('keeps the nested object reference unchanged (byte for byte)', () => { + const thinking = { type: 'enabled', budget_tokens: 1024 }; + const result = camelizeModelParameters({ thinking }); + expect(result.thinking).toBe(thinking); + }); + + it('prefers the snake_case spelling when both spellings are set, snake_case first', () => { + expect(camelizeModelParameters({ max_turns: 5, maxTurns: 1 })).toEqual({ maxTurns: 5 }); + }); + + it('prefers the snake_case spelling when both spellings are set, camelCase first', () => { + expect(camelizeModelParameters({ maxTurns: 1, max_turns: 5 })).toEqual({ maxTurns: 5 }); + }); + + it('leaves a key with a leading underscore unchanged', () => { + expect(camelizeModelParameters({ _foo: 1 })).toEqual({ _foo: 1 }); + }); + + it('leaves a key with a trailing underscore unchanged', () => { + expect(camelizeModelParameters({ foo_: 1 })).toEqual({ foo_: 1 }); + }); + + it('leaves a key with a doubled underscore unchanged', () => { + expect(camelizeModelParameters({ foo__bar: 1 })).toEqual({ foo__bar: 1 }); + }); + + it('returns a new object and never mutates the input', () => { + const input = { max_turns: 3 }; + const result = camelizeModelParameters(input); + expect(result).not.toBe(input); + expect(input).toEqual({ max_turns: 3 }); + }); + + it('returns {} for an empty bag', () => { + expect(camelizeModelParameters({})).toEqual({}); + }); +}); diff --git a/packages/client/src/index.ts b/packages/client/src/index.ts index f354bbc2..f490f51f 100644 --- a/packages/client/src/index.ts +++ b/packages/client/src/index.ts @@ -85,14 +85,17 @@ export { GraphTopologySchema, NATIVE_TOOL_KEY, NativeTool } from './types.js'; export type { RunUsage, SpanUsage } from './utils.js'; export { addCachedTokensToInput, + camelizeModelParameters, collapseMessagesToInstructions, createHandler, createRunUsage, endSpanOnce, langChainSpanUsage, + normalizeModelParameters, omitModelStamps, parseJSONWithPossibleFences, parseTemplate, + pickForwardedModelParameters, setLdSpanAttributes, setModelIdentityAttributes, setUsageSpanAttributes, diff --git a/packages/client/src/utils.ts b/packages/client/src/utils.ts index f7f3d90b..e64eb4f3 100644 --- a/packages/client/src/utils.ts +++ b/packages/client/src/utils.ts @@ -537,6 +537,97 @@ export function parseJSONWithPossibleFences(rawText: string): T | null { return null; } +/** + * Returns `parameters` unchanged when it is a usable object, `{}` otherwise. + * + * `model.parameters` is optional on `AiConfigRep`, and a config's own `model` field is technically + * reachable as `unknown` at handler call sites, so both must be checked before the bag is read. + * This only guards the shape: every handler then narrows the result to its own allowlist with + * `pickForwardedModelParameters`, because `model.parameters` comes from the AI Config and must + * never be able to set credentials, endpoints, headers or host-process settings. + */ +export function normalizeModelParameters(parameters: unknown): Record { + return parameters && typeof parameters === 'object' ? (parameters as Record) : {}; +} + +/** + * Picks the subset of `parameters` whose keys appear in `keys`, unchanged otherwise: a config that + * sets nothing in `keys` produces `{}`, so the provider call sees exactly what it always has. + * + * Every handler forwards `model.parameters` through this, against an explicit per-handler + * allowlist kept next to the call it maps onto (checked at compile time against the provider or + * framework SDK's own option types where the SDK exports them). Unknown keys are dropped. The rule + * for those lists: model and run settings only. Credentials, base URLs and other endpoint or + * connection settings, timeouts and retries, request header, body and query overrides, remote + * tool servers, and host-process settings are never on one, in any spelling. + */ +export function pickForwardedModelParameters( + parameters: Record | undefined, + keys: ReadonlyArray, +): Record { + const picked: Record = {}; + if (!parameters) return picked; + for (const key of keys) { + if (parameters[key] !== undefined) picked[key] = parameters[key]; + } + return picked; +} + +/** + * Converts one snake_case key to camelCase, or returns `undefined` when the key should pass + * through unchanged: a key with no underscore (already camelCase, or a single word like + * `temperature`), or one with a leading, trailing, or doubled underscore, where guessing the + * intended split would be wrong more often than leaving it alone. + */ +function camelizeKey(key: string): string | undefined { + if (!key.includes('_')) return undefined; + const parts = key.split('_'); + if (parts.some((part) => part.length === 0)) return undefined; + return ( + parts[0] + + parts + .slice(1) + .map((part) => part[0].toUpperCase() + part.slice(1)) + .join('') + ); +} + +/** + * Converts the TOP-LEVEL keys of a `model.parameters` bag from snake_case (the LaunchDarkly UI's + * convention, and the wire name every provider SDK expects) to camelCase, for the handlers whose + * underlying framework, as opposed to a raw provider client, only reads camelCase option names: + * `max_turns` becomes `maxTurns`, `top_p` becomes `topP`. Callers narrow the result to their + * allowlist afterwards, so camelizing never makes a key forwardable by itself. + * + * Only top-level keys convert. A nested value such as `thinking: { budget_tokens: 1024 }` is + * copied through untouched; a handler whose SDK reads a nested camelCase shape rebuilds that value + * itself. + * + * When a bag sets both spellings of one key (`max_turns` and `maxTurns`), the snake_case one + * wins — it is the convention the UI writes, so it is treated as authoritative — deterministically + * regardless of which key came first in the object. + * + * Returns a new object; never mutates the input. + */ +export function camelizeModelParameters(parameters: Record): Record { + const result: Record = {}; + const converted: Array<[string, unknown]> = []; + for (const [key, value] of Object.entries(parameters)) { + const camelKey = camelizeKey(key); + if (camelKey === undefined) { + result[key] = value; + } else { + converted.push([camelKey, value]); + } + } + // Applied after every unconverted key, so a snake_case key always overwrites a colliding + // camelCase key regardless of iteration order. + for (const [camelKey, value] of converted) { + result[camelKey] = value; + } + return result; +} + /** * Returns a copy of `trackData` without `modelKey` / `modelVersion`. Used when * overlaying a judge's `trackData` on its parent's so a judge without a pinned diff --git a/packages/langchain-agents/README.md b/packages/langchain-agents/README.md index 6ec6484b..f26163dd 100644 --- a/packages/langchain-agents/README.md +++ b/packages/langchain-agents/README.md @@ -48,10 +48,15 @@ be built after evaluation: ```ts const handler = createLangChainAgentsHandler((config) => new ChatAnthropic({ model: config.model.name, - ...config.model.parameters, + temperature: typeof config.model.parameters?.temperature === 'number' ? config.model.parameters.temperature : undefined, })); ``` +Pick out the settings you want by name. Don't spread `config.model.parameters` into the constructor: +it comes from the AI Config, and a spread would let it set constructor options such as `apiKey`, +`anthropicApiUrl` or `clientOptions`. If you pass no model, the handler builds one itself and +forwards only an allowlist of generation settings. + ### Convenience wrapper ```ts diff --git a/packages/langchain-agents/src/__tests__/handler.test.ts b/packages/langchain-agents/src/__tests__/handler.test.ts index c499bd7b..472f247b 100644 --- a/packages/langchain-agents/src/__tests__/handler.test.ts +++ b/packages/langchain-agents/src/__tests__/handler.test.ts @@ -84,6 +84,7 @@ vi.mock('@launchdarkly/ai-server', async (importOriginal) => { }); import { createLangChainAgentsHandler } from '../handler.js'; +import { expectNoNeverForwardedValue, NEVER_FORWARDED_PARAMETERS } from './never-forwarded.js'; // ─── Helpers ────────────────────────────────────────────────────────────────── @@ -1003,7 +1004,10 @@ describe('model source', () => { expect(mockCreateAgent).toHaveBeenCalledWith(expect.objectContaining({ model: llm })); }); - it('spreads model.parameters into the default OpenAI constructor', async () => { + // @langchain/openai's ChatOpenAI constructor only reads camelCase (`maxTokens`), so the + // snake_case bag the UI writes is camelized, then narrowed to ChatOpenAI's allowlist: `tools` + // is not a constructor field, so it is dropped. + it('passes allowlisted model.parameters to the default OpenAI constructor, camelized, and drops the rest', async () => { const constructed = { tag: 'default-openai' }; MockChatOpenAI.mockImplementation(function MockChatOpenAI() { return constructed; @@ -1015,13 +1019,46 @@ describe('model source', () => { await createLangChainAgentsHandler()(cfg as any, 'q'); expect(MockChatOpenAI).toHaveBeenCalledWith({ temperature: 0.2, - max_tokens: 512, - tools: ['openai-tool'], + maxTokens: 512, model: 'gpt-4o', }); expect(mockCreateAgent).toHaveBeenCalledWith(expect.objectContaining({ model: constructed })); }); + it('camelCase model.parameters keys still pass through unchanged into the default constructor', async () => { + const constructed = { tag: 'default-openai-camel' }; + MockChatOpenAI.mockImplementation(function MockChatOpenAI() { + return constructed; + }); + const cfg = { + model: { name: 'gpt-4o', parameters: { temperature: 0.2, maxTokens: 512 } }, + provider: { name: 'LangChain' }, + instructions: 'Be helpful.', + }; + await createLangChainAgentsHandler()(cfg as any, 'q'); + expect(MockChatOpenAI).toHaveBeenCalledWith({ temperature: 0.2, maxTokens: 512, model: 'gpt-4o' }); + }); + + it.each([ + ['OpenAI', 'gpt-4o', MockChatOpenAI], + ['Anthropic', 'claude-sonnet-4-5', MockChatAnthropic], + ['Bedrock', 'anthropic.claude-sonnet-4-5', MockChatBedrockConverse], + ] as const)('never passes credentials or connection settings to the default %s constructor', async (provider, name, ctor) => { + ctor.mockClear(); + ctor.mockImplementation(function MockChatModel() { + return {}; + }); + const cfg = { + provider: { name: provider }, + model: { name, parameters: { ...NEVER_FORWARDED_PARAMETERS, temperature: 0.3 } }, + instructions: 'Be helpful.', + }; + await createLangChainAgentsHandler()(cfg as any, 'q'); + const args = ctor.mock.calls.at(-1)![0] as Record; + expect(args.temperature).toBe(0.3); + expectNoNeverForwardedValue(args); + }); + it('spreads model.parameters into the default Anthropic constructor', async () => { const constructed = { tag: 'default-anthropic' }; MockChatAnthropic.mockImplementation(function MockChatAnthropic() { diff --git a/packages/langchain-agents/src/__tests__/model-parameters.test.ts b/packages/langchain-agents/src/__tests__/model-parameters.test.ts new file mode 100644 index 00000000..883d206a --- /dev/null +++ b/packages/langchain-agents/src/__tests__/model-parameters.test.ts @@ -0,0 +1,133 @@ +import { describe, expect, it } from 'vitest'; +import { modelConstructorParameters } from '../model-parameters.js'; +import { expectNoNeverForwardedValue, NEVER_FORWARDED_PARAMETERS } from './never-forwarded.js'; + +describe('modelConstructorParameters', () => { + it.each(['openai', 'anthropic', 'bedrock'] as const)('drops every never-forwarded key for %s', (modelClass) => { + const args = modelConstructorParameters({ ...NEVER_FORWARDED_PARAMETERS, temperature: 0.5 }, modelClass); + expect(args).toEqual({ temperature: 0.5 }); + expectNoNeverForwardedValue(args); + }); + + it.each(['openai', 'anthropic', 'bedrock'] as const)('drops unknown keys and model for %s', (modelClass) => { + expect( + modelConstructorParameters( + { model: 'smuggled', model_name: 'smuggled', tools: ['x'], unknown_key: 1, callbacks: [] }, + modelClass, + ), + ).toEqual({}); + }); + + it('forwards ChatOpenAI fields under their constructor names', () => { + expect( + modelConstructorParameters( + { + temperature: 0.2, + max_tokens: 100, + max_completion_tokens: 200, + top_p: 0.9, + frequency_penalty: 0.1, + presence_penalty: 0.2, + n: 2, + logit_bias: { '50256': -100 }, + logprobs: true, + top_logprobs: 3, + stop: ['END'], + stream_usage: true, + reasoning: { effort: 'low' }, + verbosity: 'low', + service_tier: 'flex', + prompt_cache_key: 'k', + use_responses_api: true, + user: 'u-1', + tags: ['t'], + metadata: { a: 1 }, + }, + 'openai', + ), + ).toEqual({ + temperature: 0.2, + maxTokens: 100, + maxCompletionTokens: 200, + topP: 0.9, + frequencyPenalty: 0.1, + presencePenalty: 0.2, + n: 2, + logitBias: { '50256': -100 }, + logprobs: true, + topLogprobs: 3, + stop: ['END'], + streamUsage: true, + reasoning: { effort: 'low' }, + verbosity: 'low', + // ChatOpenAI reads this one field in snake_case. + service_tier: 'flex', + promptCacheKey: 'k', + useResponsesApi: true, + user: 'u-1', + tags: ['t'], + metadata: { a: 1 }, + }); + }); + + it('forwards ChatAnthropic fields, keeps nested Messages API shapes, and folds effort into outputConfig', () => { + expect( + modelConstructorParameters( + { + temperature: 0.2, + max_tokens: 100, + top_k: 40, + top_p: 0.9, + stop_sequences: ['END'], + thinking: { type: 'enabled', budget_tokens: 1024 }, + context_management: { edits: [{ type: 'clear_tool_uses_20250919' }] }, + inference_geo: 'us', + betas: ['context-1m-2025-08-07'], + effort: 'low', + }, + 'anthropic', + ), + ).toEqual({ + temperature: 0.2, + maxTokens: 100, + topK: 40, + topP: 0.9, + stopSequences: ['END'], + // ChatAnthropic sends these to the Messages API as written, so they stay snake_case. + thinking: { type: 'enabled', budget_tokens: 1024 }, + contextManagement: { edits: [{ type: 'clear_tool_uses_20250919' }] }, + inferenceGeo: 'us', + betas: ['context-1m-2025-08-07'], + outputConfig: { effort: 'low' }, + }); + expect(modelConstructorParameters({ effort: 'low', output_config: { effort: 'high' } }, 'anthropic')).toEqual({ + outputConfig: { effort: 'high' }, + }); + }); + + it('forwards ChatBedrockConverse fields under their constructor names', () => { + expect( + modelConstructorParameters( + { + temperature: 0.2, + max_tokens: 100, + top_p: 0.9, + guardrail_config: { guardrailIdentifier: 'g', guardrailVersion: '1' }, + performance_config: { latency: 'optimized' }, + service_tier: 'priority', + supports_tool_choice_values: ['auto'], + application_inference_profile: 'arn:smuggled', + }, + 'bedrock', + ), + ).toEqual({ + temperature: 0.2, + maxTokens: 100, + topP: 0.9, + guardrailConfig: { guardrailIdentifier: 'g', guardrailVersion: '1' }, + performanceConfig: { latency: 'optimized' }, + serviceTier: 'priority', + supportsToolChoiceValues: ['auto'], + }); + }); +}); diff --git a/packages/langchain-agents/src/__tests__/native-graph.test.ts b/packages/langchain-agents/src/__tests__/native-graph.test.ts index a6270026..48d9d6d7 100644 --- a/packages/langchain-agents/src/__tests__/native-graph.test.ts +++ b/packages/langchain-agents/src/__tests__/native-graph.test.ts @@ -98,6 +98,7 @@ vi.mock('@launchdarkly/ai-server', async (importOriginal) => { }); import { toLangGraph } from '../native-graph.js'; +import { expectNoNeverForwardedValue, NEVER_FORWARDED_PARAMETERS } from './never-forwarded.js'; // ─── Test helpers ───────────────────────────────────────────────────────────── @@ -178,13 +179,35 @@ describe('toLangGraph', () => { expect(registeredNames).toContain('leaf'); }); - it('spreads model.parameters into the default ChatOpenAI constructor', async () => { + // ChatOpenAI only reads camelCase, so the graph's default model factory camelizes the + // snake_case bag the UI writes, then narrows it to ChatOpenAI's allowlist. + it('spreads model.parameters into the default ChatOpenAI constructor, camelized', async () => { const root = makeNode('root', '', []); root.config.model.parameters = { temperature: 0.2, max_tokens: 512 }; const def = makeGraphDef([root], {}, 'root'); MockChatOpenAI.mockClear(); await toLangGraph(Promise.resolve(def)).invoke('hi'); - expect(MockChatOpenAI).toHaveBeenCalledWith({ temperature: 0.2, max_tokens: 512, model: 'gpt-4o' }); + expect(MockChatOpenAI).toHaveBeenCalledWith({ temperature: 0.2, maxTokens: 512, model: 'gpt-4o' }); + }); + + it('leaves camelCase model.parameters keys unchanged for the default ChatOpenAI constructor', async () => { + const root = makeNode('root', '', []); + root.config.model.parameters = { temperature: 0.2, maxTokens: 512 }; + const def = makeGraphDef([root], {}, 'root'); + MockChatOpenAI.mockClear(); + await toLangGraph(Promise.resolve(def)).invoke('hi'); + expect(MockChatOpenAI).toHaveBeenCalledWith({ temperature: 0.2, maxTokens: 512, model: 'gpt-4o' }); + }); + + it('never passes credentials or connection settings to the default ChatOpenAI constructor', async () => { + const root = makeNode('root', '', []); + root.config.model.parameters = { ...NEVER_FORWARDED_PARAMETERS, temperature: 0.3 }; + const def = makeGraphDef([root], {}, 'root'); + MockChatOpenAI.mockClear(); + await toLangGraph(Promise.resolve(def)).invoke('hi'); + const args = MockChatOpenAI.mock.calls.at(-1)![0] as Record; + expect(args).toEqual({ temperature: 0.3, model: 'gpt-4o' }); + expectNoNeverForwardedValue(args); }); it('wires the root node from START', async () => { diff --git a/packages/langchain-agents/src/__tests__/never-forwarded.ts b/packages/langchain-agents/src/__tests__/never-forwarded.ts new file mode 100644 index 00000000..300d6e51 --- /dev/null +++ b/packages/langchain-agents/src/__tests__/never-forwarded.ts @@ -0,0 +1,163 @@ +import { expect } from 'vitest'; + +/** + * The value every never-forwarded key carries in `NEVER_FORWARDED_PARAMETERS`. A handler that + * passes any of those keys on, under any name or nested anywhere, leaves this string somewhere in + * the options it builds, which is what `expectNoNeverForwardedValue` looks for. + */ +export const NEVER_FORWARDED = 'NEVER_FORWARDED'; + +/** + * Keys that must never reach a provider, framework or host-process option from + * `model.parameters`, in both the snake_case the LaunchDarkly UI writes and the camelCase the + * frameworks read: credentials, endpoints and connection settings, request injection, remote + * tools, and Claude Agents host-process settings. + */ +export const NEVER_FORWARDED_KEYS = [ + // credentials + 'api_key', + 'apiKey', + 'openai_api_key', + 'openAIApiKey', + 'anthropic_api_key', + 'anthropicApiKey', + 'bedrock_api_key', + 'bedrockApiKey', + 'bedrock_api_secret', + 'bedrockApiSecret', + 'credentials', + 'auth_token', + 'authToken', + 'access_token', + 'accessToken', + 'aws_access_key_id', + 'awsAccessKeyId', + 'aws_secret_access_key', + 'awsSecretAccessKey', + 'aws_session_token', + 'awsSessionToken', + 'organization', + // endpoints and connection + 'base_url', + 'baseURL', + 'baseUrl', + 'anthropic_api_url', + 'anthropicApiUrl', + 'endpoint', + 'endpoint_host', + 'endpointHost', + 'endpoint_url', + 'endpointUrl', + 'region', + 'region_name', + 'regionName', + 'configuration', + 'client_options', + 'clientOptions', + 'default_headers', + 'defaultHeaders', + 'timeout', + 'max_retries', + 'maxRetries', + 'fetch', + 'http_client', + 'httpClient', + 'http_agent', + 'httpAgent', + 'proxy', + 'proxies', + 'client', + // request injection + 'headers', + 'extra_headers', + 'extraHeaders', + 'extra_body', + 'extraBody', + 'extra_query', + 'extraQuery', + 'model_kwargs', + 'modelKwargs', + 'invocation_kwargs', + 'invocationKwargs', + 'additional_model_request_fields', + 'additionalModelRequestFields', + 'provider_data', + 'providerData', + 'provider_options', + 'providerOptions', + // remote tools + 'mcp_servers', + 'mcpServers', + // Claude Agents host-process settings + 'path_to_claude_code_executable', + 'pathToClaudeCodeExecutable', + 'cli_path', + 'cliPath', + 'executable', + 'executable_args', + 'executableArgs', + 'extra_args', + 'extraArgs', + 'env', + 'cwd', + 'additional_directories', + 'additionalDirectories', + 'add_dirs', + 'addDirs', + 'permission_mode', + 'permissionMode', + 'allow_dangerously_skip_permissions', + 'allowDangerouslySkipPermissions', + 'settings', + 'setting_sources', + 'settingSources', + 'managed_settings', + 'managedSettings', + 'plugins', + 'sandbox', + 'resume', + 'resume_session_at', + 'resumeSessionAt', + 'session_id', + 'sessionId', + 'fork_session', + 'forkSession', + 'continue', + 'hooks', + 'can_use_tool', + 'canUseTool', + 'stderr', + 'abort_controller', + 'abortController', + 'spawn_claude_code_process', + 'spawnClaudeCodeProcess', +] as const; + +/** A `model.parameters` bag that sets every key in `NEVER_FORWARDED_KEYS` to `NEVER_FORWARDED`. */ +export const NEVER_FORWARDED_PARAMETERS: Record = Object.fromEntries( + NEVER_FORWARDED_KEYS.map((key) => [key, NEVER_FORWARDED]), +); + +function findNeverForwarded(value: unknown, path: string, seen: Set): string | undefined { + if (value === NEVER_FORWARDED) return path || '(root)'; + if (!value || typeof value !== 'object' || seen.has(value)) return undefined; + seen.add(value); + for (const [key, child] of Object.entries(value)) { + const found = findNeverForwarded(child, path ? `${path}.${key}` : key, seen); + if (found) return found; + } + return undefined; +} + +/** + * Asserts that no `NEVER_FORWARDED` value appears anywhere in `options`, and that none of the + * `NEVER_FORWARDED_KEYS` the handler does not own itself is set on it. `handlerOwned` lists the + * keys the handler legitimately sets from its own state (Claude Agents sets `mcpServers` and + * `hooks`, for example); those must still not carry the config's value. + */ +export function expectNoNeverForwardedValue(options: unknown, handlerOwned: ReadonlyArray = []): void { + expect(findNeverForwarded(options, '', new Set())).toBeUndefined(); + const record = options as Record; + const leaked = NEVER_FORWARDED_KEYS.filter((key) => !handlerOwned.includes(key) && record[key] !== undefined); + expect(leaked).toEqual([]); +} diff --git a/packages/langchain-agents/src/__tests__/wire.test.ts b/packages/langchain-agents/src/__tests__/wire.test.ts new file mode 100644 index 00000000..f5f80d52 --- /dev/null +++ b/packages/langchain-agents/src/__tests__/wire.test.ts @@ -0,0 +1,83 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { createLangChainAgentsHandler } from '../handler.js'; +import { NEVER_FORWARDED_PARAMETERS } from './never-forwarded.js'; + +// This file deliberately does NOT mock @langchain/openai. Every other test in this package mocks +// it out, so a handler-internal bug that maps a config key to the wrong ChatOpenAI constructor +// field would never reach a real request body. Constructing a real ChatOpenAI and intercepting +// its outgoing fetch call checks what actually goes on the wire. + +type Captured = { url: string; headers: Record; body: Record }; + +const captured: Captured[] = []; + +beforeEach(() => { + captured.length = 0; + // The customer's own credentials, from the environment, as an application would set them. + vi.stubEnv('OPENAI_API_KEY', 'CUSTOMER'); + vi.stubGlobal( + 'fetch', + vi.fn(async (url: unknown, init?: RequestInit) => { + captured.push({ + url: String(url instanceof Request ? url.url : url), + headers: Object.fromEntries(new Headers(init?.headers).entries()), + body: JSON.parse(String(init?.body)), + }); + // A 401 is not retried by either the openai client or LangChain's AsyncCaller. + return new Response(JSON.stringify({ error: { message: 'captured' } }), { + status: 401, + headers: { 'content-type': 'application/json' }, + }); + }), + ); +}); + +afterEach(() => { + vi.unstubAllEnvs(); + vi.unstubAllGlobals(); +}); + +const baseConfig = { + model: { name: 'gpt-4o', parameters: { top_p: 0.42, max_tokens: 321 } }, + provider: { name: 'LangChain' }, + instructions: 'You are helpful.', +}; + +describe('createLangChainAgentsHandler wire request (real ChatOpenAI, unmocked)', () => { + it('sends snake_case model.parameters onto the wire as top_p / max_tokens', async () => { + await expect(createLangChainAgentsHandler()(baseConfig as any, 'hi')).rejects.toThrow(); + expect(captured.length).toBeGreaterThan(0); + expect(captured[0].body.top_p).toBe(0.42); + expect(captured[0].body.max_tokens).toBe(321); + }, 10000); + + it('keeps the customer key, base URL and headers when a config sets credentials, endpoints and headers', async () => { + const config = { + ...baseConfig, + model: { + ...baseConfig.model, + parameters: { + ...baseConfig.model.parameters, + ...NEVER_FORWARDED_PARAMETERS, + api_key: 'ATTACKER', + base_url: 'https://attacker.example/v1', + configuration: { + apiKey: 'ATTACKER', + baseURL: 'https://attacker.example/v1', + defaultHeaders: { Authorization: 'Bearer ATTACKER' }, + }, + model_kwargs: { injected: 'NEVER_FORWARDED' }, + }, + }, + }; + await expect(createLangChainAgentsHandler()(config as any, 'hi')).rejects.toThrow(); + expect(captured.length).toBeGreaterThan(0); + for (const request of captured) { + expect(request.url.startsWith('https://api.openai.com/')).toBe(true); + expect(request.headers.authorization).toBe('Bearer CUSTOMER'); + expect(JSON.stringify(request)).not.toContain('NEVER_FORWARDED'); + expect(JSON.stringify(request)).not.toContain('ATTACKER'); + } + expect(captured[0].body.top_p).toBe(0.42); + }, 10000); +}); diff --git a/packages/langchain-agents/src/handler.ts b/packages/langchain-agents/src/handler.ts index 98650611..7953a87b 100644 --- a/packages/langchain-agents/src/handler.ts +++ b/packages/langchain-agents/src/handler.ts @@ -33,6 +33,7 @@ import { import { type Context, context, type Span, SpanStatusCode, trace } from '@opentelemetry/api'; import { createAgent } from 'langchain'; import { toLangChainMessages } from './messages.js'; +import { type LangChainModelClass, modelConstructorParameters } from './model-parameters.js'; const TRACER_NAME = '@launchdarkly/ai-langchain-agents'; @@ -259,12 +260,16 @@ export function buildSpanCallbacks( export type LangChainModelSource = BaseChatModel | ((config: AiConfigRep) => BaseChatModel | Promise); -function modelConstructorArgs(config: AiConfigRep, fallbackName: string): Record { - const parameters = { - ...(config.model?.parameters && typeof config.model.parameters === 'object' ? config.model.parameters : {}), +function modelConstructorArgs( + config: AiConfigRep, + fallbackName: string, + modelClass: LangChainModelClass, +): Record { + // Name from the config always wins; `model` is never on a forwarded-keys list anyway. + return { + ...modelConstructorParameters(config.model?.parameters, modelClass), + model: resolvedModelName(config, fallbackName), }; - if ((config.provider?.name ?? '').toLowerCase() === 'bedrock') delete parameters.tools; - return { ...parameters, model: resolvedModelName(config, fallbackName) }; } async function resolveBaseModel(aiConfig: AiConfigRep, llm?: LangChainModelSource): Promise { @@ -274,8 +279,8 @@ async function resolveBaseModel(aiConfig: AiConfigRep, llm?: LangChainModelSourc const provider = (invocation.provider?.name ?? '').toLowerCase(); if (provider === 'anthropic') { const { ChatAnthropic } = await import('@langchain/anthropic'); - // biome-ignore lint/suspicious/noExplicitAny: parameter bag is caller-owned and not remapped - return new ChatAnthropic(modelConstructorArgs(invocation, 'claude-3-5-sonnet-20241022') as any); + // biome-ignore lint/suspicious/noExplicitAny: constructor fields are narrowed per model class in modelConstructorParameters + return new ChatAnthropic(modelConstructorArgs(invocation, 'claude-3-5-sonnet-20241022', 'anthropic') as any); } if (provider === 'bedrock') { // biome-ignore lint/suspicious/noExplicitAny: @langchain/aws loaded via dynamic import with no static types @@ -285,11 +290,11 @@ async function resolveBaseModel(aiConfig: AiConfigRep, llm?: LangChainModelSourc } catch { throw new Error('Using Bedrock models requires @langchain/aws. Install it with: npm install @langchain/aws'); } - // biome-ignore lint/suspicious/noExplicitAny: parameter bag is caller-owned and not remapped - return new mod.ChatBedrockConverse(modelConstructorArgs(invocation, '') as any); + // biome-ignore lint/suspicious/noExplicitAny: constructor fields are narrowed per model class in modelConstructorParameters + return new mod.ChatBedrockConverse(modelConstructorArgs(invocation, '', 'bedrock') as any); } - // biome-ignore lint/suspicious/noExplicitAny: parameter bag is caller-owned and not remapped - return new ChatOpenAI(modelConstructorArgs(invocation, 'gpt-4o') as any); + // biome-ignore lint/suspicious/noExplicitAny: constructor fields are narrowed per model class in modelConstructorParameters + return new ChatOpenAI(modelConstructorArgs(invocation, 'gpt-4o', 'openai') as any); } const buildAgentTools = (configTools: Record, toolHandlers: Record) => diff --git a/packages/langchain-agents/src/model-parameters.ts b/packages/langchain-agents/src/model-parameters.ts new file mode 100644 index 00000000..9c56c87e --- /dev/null +++ b/packages/langchain-agents/src/model-parameters.ts @@ -0,0 +1,146 @@ +import type { ChatAnthropicInput } from '@langchain/anthropic'; +import type { ChatBedrockConverseInput } from '@langchain/aws'; +import type { ChatOpenAIFields } from '@langchain/openai'; +import { + type AiConfigRep, + camelizeModelParameters, + normalizeModelParameters, + pickForwardedModelParameters, +} from '@launchdarkly/ai-server'; + +/** + * Constructor fields every LangChain chat model takes from `BaseChatModelParams` that a config + * can set as JSON. `callbacks`, `callbackManager`, `cache` and `maxConcurrency` are wiring, and + * `maxRetries` is client retry policy, so none of them are forwarded. + */ +const BASE_CHAT_MODEL_KEYS = ['disableStreaming', 'metadata', 'outputVersion', 'tags', 'verbose'] as const; + +/** + * `ChatOpenAI` constructor fields this package takes from `config.model.parameters`, after + * camelizing the bag. Mirrors the Python SDK's `ChatOpenAI` list. + * + * Never forwarded: `apiKey`, `openAIApiKey`, `configuration` (client options, including the base + * URL, default headers, fetch and proxy settings), `timeout`, `maxRetries`, `modelKwargs` (merged + * into the request body as written), `prefixMessages` (prompt content), `model` / `modelName` + * (the handler sets the model), and the client internals `completions`, `responses`, + * `__includeRawResponse` and `supportsStrictToolCalling`. + */ +export const CHAT_OPENAI_FORWARDED_KEYS = [ + ...BASE_CHAT_MODEL_KEYS, + 'audio', + 'frequencyPenalty', + 'logitBias', + 'logprobs', + 'maxCompletionTokens', + 'maxTokens', + 'modalities', + 'n', + 'presencePenalty', + 'promptCacheKey', + 'promptCacheRetention', + 'reasoning', + 'service_tier', + 'stop', + 'stopSequences', + 'streamUsage', + 'streaming', + 'temperature', + 'topLogprobs', + 'topP', + 'useResponsesApi', + 'user', + 'verbosity', + 'zdrEnabled', +] as const satisfies ReadonlyArray; + +/** + * `ChatAnthropic` constructor fields this package takes from `config.model.parameters`, after + * camelizing the bag. Mirrors the Python SDK's `ChatAnthropic` list, minus `mcp_servers` (remote + * tools) and `model_kwargs` (request injection), neither of which a config may set. + * + * Never forwarded: `apiKey`, `anthropicApiKey`, `anthropicApiUrl`, `clientOptions` (including + * default headers and fetch), `createClient`, `invocationKwargs` (merged into the request body as + * written), and `model` / `modelName`. + * + * Nested values (`thinking`, `contextManagement`, `outputConfig`) are the Anthropic Messages API's + * own snake_case shapes, which `ChatAnthropic` sends as written, so they are not converted. A + * top-level `effort` (the name the UI writes) becomes `outputConfig.effort`; see + * `modelConstructorParameters`. + */ +export const CHAT_ANTHROPIC_FORWARDED_KEYS = [ + ...BASE_CHAT_MODEL_KEYS, + 'betas', + 'contextManagement', + 'inferenceGeo', + 'maxTokens', + 'outputConfig', + 'stopSequences', + 'streamUsage', + 'streaming', + 'temperature', + 'thinking', + 'topK', + 'topP', +] as const satisfies ReadonlyArray; + +/** + * `ChatBedrockConverse` constructor fields this package takes from `config.model.parameters`, + * after camelizing the bag. Mirrors the Python SDK's `ChatBedrockConverse` list, minus + * `region_name`, which is connection configuration. + * + * Never forwarded: `additionalModelRequestFields` (merged into the request body as written), + * `region`, `credentials`, `bedrockApiKey`, `bedrockApiSecret`, + * `bedrockApiSessionToken`, the AWS credential-provider settings (`profile`, `roleArn`, ...), + * `endpointHost`, `client`, `clientOptions`, `defaultHeaders`, `model`, and + * `applicationInferenceProfile` (which replaces the model the handler resolved). + */ +export const CHAT_BEDROCK_CONVERSE_FORWARDED_KEYS = [ + ...BASE_CHAT_MODEL_KEYS, + 'guardrailConfig', + 'maxTokens', + 'performanceConfig', + 'serviceTier', + 'streamUsage', + 'streaming', + 'supportsToolChoiceValues', + 'temperature', + 'topP', +] as const satisfies ReadonlyArray; + +export type LangChainModelClass = 'openai' | 'anthropic' | 'bedrock'; + +const FORWARDED_KEYS: Record> = { + openai: CHAT_OPENAI_FORWARDED_KEYS, + anthropic: CHAT_ANTHROPIC_FORWARDED_KEYS, + bedrock: CHAT_BEDROCK_CONVERSE_FORWARDED_KEYS, +}; + +/** + * `config.model.parameters` as constructor fields for one LangChain chat model class: camelized + * (`top_p` becomes `topP`; nested values are left alone, and snake_case wins when both spellings + * are set), then narrowed to that class's allowlist. Unknown keys are dropped. + * + * Renames before the allowlist is applied: + * - `ChatOpenAI` reads its service tier as `service_tier`, so a camelized `serviceTier` is + * renamed back. + * - `ChatAnthropic` has no `effort` field; a top-level `effort` becomes `outputConfig.effort`. + * An explicit `output_config.effort` wins. + */ +export function modelConstructorParameters( + parameters: AiConfigRep['model']['parameters'] | unknown, + modelClass: LangChainModelClass, +): Record { + const camelized = camelizeModelParameters(normalizeModelParameters(parameters)); + if (modelClass === 'openai' && camelized.serviceTier !== undefined) { + camelized.service_tier = camelized.serviceTier; + delete camelized.serviceTier; + } + if (modelClass === 'anthropic' && camelized.effort !== undefined) { + const outputConfig = + camelized.outputConfig && typeof camelized.outputConfig === 'object' && !Array.isArray(camelized.outputConfig) + ? (camelized.outputConfig as Record) + : {}; + camelized.outputConfig = { effort: camelized.effort, ...outputConfig }; + } + return pickForwardedModelParameters(camelized, FORWARDED_KEYS[modelClass]); +} diff --git a/packages/langchain-agents/src/native-graph.ts b/packages/langchain-agents/src/native-graph.ts index 029ca26e..e77818f8 100644 --- a/packages/langchain-agents/src/native-graph.ts +++ b/packages/langchain-agents/src/native-graph.ts @@ -20,6 +20,7 @@ import { import { SpanStatusCode, trace } from '@opentelemetry/api'; import { z } from 'zod'; import { toLangChainMessages } from './messages.js'; +import { modelConstructorParameters } from './model-parameters.js'; // ─── helpers ───────────────────────────────────────────────────────────────── @@ -144,9 +145,7 @@ export const toLangGraph = ( opts?.modelFactory ?? ((node) => new ChatOpenAI({ - ...(node.config.model.parameters && typeof node.config.model.parameters === 'object' - ? node.config.model.parameters - : {}), + ...modelConstructorParameters(node.config.model.parameters, 'openai'), model: node.config.model.name, })); const ldContext = opts?.context; diff --git a/packages/langchain-messages/README.md b/packages/langchain-messages/README.md index ee465125..1cd59999 100644 --- a/packages/langchain-messages/README.md +++ b/packages/langchain-messages/README.md @@ -45,10 +45,15 @@ be built after evaluation: ```ts const handler = createLangChainHandler((config) => new ChatAnthropic({ model: config.model.name, - ...config.model.parameters, + temperature: typeof config.model.parameters?.temperature === 'number' ? config.model.parameters.temperature : undefined, })); ``` +Pick out the settings you want by name. Don't spread `config.model.parameters` into the constructor: +it comes from the AI Config, and a spread would let it set constructor options such as `apiKey`, +`anthropicApiUrl` or `clientOptions`. If you pass no model, the handler builds one itself and +forwards only an allowlist of generation settings. + ### Convenience wrapper ```ts diff --git a/packages/langchain-messages/src/__tests__/handler.test.ts b/packages/langchain-messages/src/__tests__/handler.test.ts index de46ba46..7dc04431 100644 --- a/packages/langchain-messages/src/__tests__/handler.test.ts +++ b/packages/langchain-messages/src/__tests__/handler.test.ts @@ -74,6 +74,7 @@ import { ChatAnthropic } from '@langchain/anthropic'; import { ChatBedrockConverse } from '@langchain/aws'; import { ChatOpenAI } from '@langchain/openai'; import { createLangChainHandler } from '../handler.js'; +import { expectNoNeverForwardedValue, NEVER_FORWARDED_PARAMETERS } from './never-forwarded.js'; // ─── Helpers ────────────────────────────────────────────────────────────────── @@ -1403,7 +1404,10 @@ describe('model source', () => { expect(result.output).toBe('from-instance'); }); - it('spreads model.parameters into the default OpenAI constructor', async () => { + // @langchain/openai's ChatOpenAI constructor only reads camelCase (`maxTokens`), so the + // snake_case bag the UI writes is camelized, then narrowed to ChatOpenAI's allowlist: `tools` + // is not a constructor field, so it is dropped. + it('passes allowlisted model.parameters to the default OpenAI constructor, camelized, and drops the rest', async () => { vi.mocked(ChatOpenAI).mockImplementation(function MockChatOpenAI() { return makeMockLLM('default-openai') as any; }); @@ -1414,12 +1418,42 @@ describe('model source', () => { await createLangChainHandler()(cfg as any, 'q'); expect(ChatOpenAI).toHaveBeenCalledWith({ temperature: 0.2, - max_tokens: 512, - tools: ['openai-tool'], + maxTokens: 512, model: 'gpt-4o', }); }); + it('camelCase model.parameters keys still pass through unchanged into the default constructor', async () => { + vi.mocked(ChatOpenAI).mockImplementation(function MockChatOpenAI() { + return makeMockLLM('default-openai') as any; + }); + const cfg = { + ...baseConfig, + model: { name: 'gpt-4o', parameters: { temperature: 0.2, maxTokens: 512 } }, + }; + await createLangChainHandler()(cfg as any, 'q'); + expect(ChatOpenAI).toHaveBeenCalledWith({ temperature: 0.2, maxTokens: 512, model: 'gpt-4o' }); + }); + + it.each([ + ['OpenAI', 'gpt-4o', () => ChatOpenAI], + ['Anthropic', 'claude-sonnet-4-5', () => ChatAnthropic], + ['Bedrock', 'anthropic.claude-sonnet-4-5', () => ChatBedrockConverse], + ] as const)('never passes credentials or connection settings to the default %s constructor', async (provider, name, ctor) => { + vi.mocked(ctor()).mockImplementation(function MockChatModel() { + return makeMockLLM('default') as any; + } as any); + const cfg = { + ...baseConfig, + provider: { name: provider }, + model: { name, parameters: { ...NEVER_FORWARDED_PARAMETERS, temperature: 0.3 } }, + }; + await createLangChainHandler()(cfg as any, 'q'); + const args = vi.mocked(ctor()).mock.calls.at(-1)![0] as Record; + expect(args.temperature).toBe(0.3); + expectNoNeverForwardedValue(args); + }); + it('spreads model.parameters into the default Anthropic constructor', async () => { vi.mocked(ChatAnthropic).mockImplementation(function MockChatAnthropic() { return makeMockLLM('default-anthropic') as any; diff --git a/packages/langchain-messages/src/__tests__/model-parameters.test.ts b/packages/langchain-messages/src/__tests__/model-parameters.test.ts new file mode 100644 index 00000000..883d206a --- /dev/null +++ b/packages/langchain-messages/src/__tests__/model-parameters.test.ts @@ -0,0 +1,133 @@ +import { describe, expect, it } from 'vitest'; +import { modelConstructorParameters } from '../model-parameters.js'; +import { expectNoNeverForwardedValue, NEVER_FORWARDED_PARAMETERS } from './never-forwarded.js'; + +describe('modelConstructorParameters', () => { + it.each(['openai', 'anthropic', 'bedrock'] as const)('drops every never-forwarded key for %s', (modelClass) => { + const args = modelConstructorParameters({ ...NEVER_FORWARDED_PARAMETERS, temperature: 0.5 }, modelClass); + expect(args).toEqual({ temperature: 0.5 }); + expectNoNeverForwardedValue(args); + }); + + it.each(['openai', 'anthropic', 'bedrock'] as const)('drops unknown keys and model for %s', (modelClass) => { + expect( + modelConstructorParameters( + { model: 'smuggled', model_name: 'smuggled', tools: ['x'], unknown_key: 1, callbacks: [] }, + modelClass, + ), + ).toEqual({}); + }); + + it('forwards ChatOpenAI fields under their constructor names', () => { + expect( + modelConstructorParameters( + { + temperature: 0.2, + max_tokens: 100, + max_completion_tokens: 200, + top_p: 0.9, + frequency_penalty: 0.1, + presence_penalty: 0.2, + n: 2, + logit_bias: { '50256': -100 }, + logprobs: true, + top_logprobs: 3, + stop: ['END'], + stream_usage: true, + reasoning: { effort: 'low' }, + verbosity: 'low', + service_tier: 'flex', + prompt_cache_key: 'k', + use_responses_api: true, + user: 'u-1', + tags: ['t'], + metadata: { a: 1 }, + }, + 'openai', + ), + ).toEqual({ + temperature: 0.2, + maxTokens: 100, + maxCompletionTokens: 200, + topP: 0.9, + frequencyPenalty: 0.1, + presencePenalty: 0.2, + n: 2, + logitBias: { '50256': -100 }, + logprobs: true, + topLogprobs: 3, + stop: ['END'], + streamUsage: true, + reasoning: { effort: 'low' }, + verbosity: 'low', + // ChatOpenAI reads this one field in snake_case. + service_tier: 'flex', + promptCacheKey: 'k', + useResponsesApi: true, + user: 'u-1', + tags: ['t'], + metadata: { a: 1 }, + }); + }); + + it('forwards ChatAnthropic fields, keeps nested Messages API shapes, and folds effort into outputConfig', () => { + expect( + modelConstructorParameters( + { + temperature: 0.2, + max_tokens: 100, + top_k: 40, + top_p: 0.9, + stop_sequences: ['END'], + thinking: { type: 'enabled', budget_tokens: 1024 }, + context_management: { edits: [{ type: 'clear_tool_uses_20250919' }] }, + inference_geo: 'us', + betas: ['context-1m-2025-08-07'], + effort: 'low', + }, + 'anthropic', + ), + ).toEqual({ + temperature: 0.2, + maxTokens: 100, + topK: 40, + topP: 0.9, + stopSequences: ['END'], + // ChatAnthropic sends these to the Messages API as written, so they stay snake_case. + thinking: { type: 'enabled', budget_tokens: 1024 }, + contextManagement: { edits: [{ type: 'clear_tool_uses_20250919' }] }, + inferenceGeo: 'us', + betas: ['context-1m-2025-08-07'], + outputConfig: { effort: 'low' }, + }); + expect(modelConstructorParameters({ effort: 'low', output_config: { effort: 'high' } }, 'anthropic')).toEqual({ + outputConfig: { effort: 'high' }, + }); + }); + + it('forwards ChatBedrockConverse fields under their constructor names', () => { + expect( + modelConstructorParameters( + { + temperature: 0.2, + max_tokens: 100, + top_p: 0.9, + guardrail_config: { guardrailIdentifier: 'g', guardrailVersion: '1' }, + performance_config: { latency: 'optimized' }, + service_tier: 'priority', + supports_tool_choice_values: ['auto'], + application_inference_profile: 'arn:smuggled', + }, + 'bedrock', + ), + ).toEqual({ + temperature: 0.2, + maxTokens: 100, + topP: 0.9, + guardrailConfig: { guardrailIdentifier: 'g', guardrailVersion: '1' }, + performanceConfig: { latency: 'optimized' }, + serviceTier: 'priority', + supportsToolChoiceValues: ['auto'], + }); + }); +}); diff --git a/packages/langchain-messages/src/__tests__/never-forwarded.ts b/packages/langchain-messages/src/__tests__/never-forwarded.ts new file mode 100644 index 00000000..300d6e51 --- /dev/null +++ b/packages/langchain-messages/src/__tests__/never-forwarded.ts @@ -0,0 +1,163 @@ +import { expect } from 'vitest'; + +/** + * The value every never-forwarded key carries in `NEVER_FORWARDED_PARAMETERS`. A handler that + * passes any of those keys on, under any name or nested anywhere, leaves this string somewhere in + * the options it builds, which is what `expectNoNeverForwardedValue` looks for. + */ +export const NEVER_FORWARDED = 'NEVER_FORWARDED'; + +/** + * Keys that must never reach a provider, framework or host-process option from + * `model.parameters`, in both the snake_case the LaunchDarkly UI writes and the camelCase the + * frameworks read: credentials, endpoints and connection settings, request injection, remote + * tools, and Claude Agents host-process settings. + */ +export const NEVER_FORWARDED_KEYS = [ + // credentials + 'api_key', + 'apiKey', + 'openai_api_key', + 'openAIApiKey', + 'anthropic_api_key', + 'anthropicApiKey', + 'bedrock_api_key', + 'bedrockApiKey', + 'bedrock_api_secret', + 'bedrockApiSecret', + 'credentials', + 'auth_token', + 'authToken', + 'access_token', + 'accessToken', + 'aws_access_key_id', + 'awsAccessKeyId', + 'aws_secret_access_key', + 'awsSecretAccessKey', + 'aws_session_token', + 'awsSessionToken', + 'organization', + // endpoints and connection + 'base_url', + 'baseURL', + 'baseUrl', + 'anthropic_api_url', + 'anthropicApiUrl', + 'endpoint', + 'endpoint_host', + 'endpointHost', + 'endpoint_url', + 'endpointUrl', + 'region', + 'region_name', + 'regionName', + 'configuration', + 'client_options', + 'clientOptions', + 'default_headers', + 'defaultHeaders', + 'timeout', + 'max_retries', + 'maxRetries', + 'fetch', + 'http_client', + 'httpClient', + 'http_agent', + 'httpAgent', + 'proxy', + 'proxies', + 'client', + // request injection + 'headers', + 'extra_headers', + 'extraHeaders', + 'extra_body', + 'extraBody', + 'extra_query', + 'extraQuery', + 'model_kwargs', + 'modelKwargs', + 'invocation_kwargs', + 'invocationKwargs', + 'additional_model_request_fields', + 'additionalModelRequestFields', + 'provider_data', + 'providerData', + 'provider_options', + 'providerOptions', + // remote tools + 'mcp_servers', + 'mcpServers', + // Claude Agents host-process settings + 'path_to_claude_code_executable', + 'pathToClaudeCodeExecutable', + 'cli_path', + 'cliPath', + 'executable', + 'executable_args', + 'executableArgs', + 'extra_args', + 'extraArgs', + 'env', + 'cwd', + 'additional_directories', + 'additionalDirectories', + 'add_dirs', + 'addDirs', + 'permission_mode', + 'permissionMode', + 'allow_dangerously_skip_permissions', + 'allowDangerouslySkipPermissions', + 'settings', + 'setting_sources', + 'settingSources', + 'managed_settings', + 'managedSettings', + 'plugins', + 'sandbox', + 'resume', + 'resume_session_at', + 'resumeSessionAt', + 'session_id', + 'sessionId', + 'fork_session', + 'forkSession', + 'continue', + 'hooks', + 'can_use_tool', + 'canUseTool', + 'stderr', + 'abort_controller', + 'abortController', + 'spawn_claude_code_process', + 'spawnClaudeCodeProcess', +] as const; + +/** A `model.parameters` bag that sets every key in `NEVER_FORWARDED_KEYS` to `NEVER_FORWARDED`. */ +export const NEVER_FORWARDED_PARAMETERS: Record = Object.fromEntries( + NEVER_FORWARDED_KEYS.map((key) => [key, NEVER_FORWARDED]), +); + +function findNeverForwarded(value: unknown, path: string, seen: Set): string | undefined { + if (value === NEVER_FORWARDED) return path || '(root)'; + if (!value || typeof value !== 'object' || seen.has(value)) return undefined; + seen.add(value); + for (const [key, child] of Object.entries(value)) { + const found = findNeverForwarded(child, path ? `${path}.${key}` : key, seen); + if (found) return found; + } + return undefined; +} + +/** + * Asserts that no `NEVER_FORWARDED` value appears anywhere in `options`, and that none of the + * `NEVER_FORWARDED_KEYS` the handler does not own itself is set on it. `handlerOwned` lists the + * keys the handler legitimately sets from its own state (Claude Agents sets `mcpServers` and + * `hooks`, for example); those must still not carry the config's value. + */ +export function expectNoNeverForwardedValue(options: unknown, handlerOwned: ReadonlyArray = []): void { + expect(findNeverForwarded(options, '', new Set())).toBeUndefined(); + const record = options as Record; + const leaked = NEVER_FORWARDED_KEYS.filter((key) => !handlerOwned.includes(key) && record[key] !== undefined); + expect(leaked).toEqual([]); +} diff --git a/packages/langchain-messages/src/__tests__/wire.test.ts b/packages/langchain-messages/src/__tests__/wire.test.ts new file mode 100644 index 00000000..2d82e03f --- /dev/null +++ b/packages/langchain-messages/src/__tests__/wire.test.ts @@ -0,0 +1,83 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { createLangChainHandler } from '../handler.js'; +import { NEVER_FORWARDED_PARAMETERS } from './never-forwarded.js'; + +// This file deliberately does NOT mock @langchain/openai. Every other test in this package mocks +// it out, so a handler-internal bug that maps a config key to the wrong ChatOpenAI constructor +// field would never reach a real request body. Constructing a real ChatOpenAI and intercepting +// its outgoing fetch call checks what actually goes on the wire. + +type Captured = { url: string; headers: Record; body: Record }; + +const captured: Captured[] = []; + +beforeEach(() => { + captured.length = 0; + // The customer's own credentials, from the environment, as an application would set them. + vi.stubEnv('OPENAI_API_KEY', 'CUSTOMER'); + vi.stubGlobal( + 'fetch', + vi.fn(async (url: unknown, init?: RequestInit) => { + captured.push({ + url: String(url instanceof Request ? url.url : url), + headers: Object.fromEntries(new Headers(init?.headers).entries()), + body: JSON.parse(String(init?.body)), + }); + // A 401 is not retried by either the openai client or LangChain's AsyncCaller. + return new Response(JSON.stringify({ error: { message: 'captured' } }), { + status: 401, + headers: { 'content-type': 'application/json' }, + }); + }), + ); +}); + +afterEach(() => { + vi.unstubAllEnvs(); + vi.unstubAllGlobals(); +}); + +const baseConfig = { + model: { name: 'gpt-4o', parameters: { top_p: 0.42, max_tokens: 321 } }, + provider: { name: 'LangChain' }, + instructions: 'You are helpful.', +}; + +describe('createLangChainHandler wire request (real ChatOpenAI, unmocked)', () => { + it('sends snake_case model.parameters onto the wire as top_p / max_tokens', async () => { + await expect(createLangChainHandler()(baseConfig as any, 'hi')).rejects.toThrow(); + expect(captured.length).toBeGreaterThan(0); + expect(captured[0].body.top_p).toBe(0.42); + expect(captured[0].body.max_tokens).toBe(321); + }, 10000); + + it('keeps the customer key, base URL and headers when a config sets credentials, endpoints and headers', async () => { + const config = { + ...baseConfig, + model: { + ...baseConfig.model, + parameters: { + ...baseConfig.model.parameters, + ...NEVER_FORWARDED_PARAMETERS, + api_key: 'ATTACKER', + base_url: 'https://attacker.example/v1', + configuration: { + apiKey: 'ATTACKER', + baseURL: 'https://attacker.example/v1', + defaultHeaders: { Authorization: 'Bearer ATTACKER' }, + }, + model_kwargs: { injected: 'NEVER_FORWARDED' }, + }, + }, + }; + await expect(createLangChainHandler()(config as any, 'hi')).rejects.toThrow(); + expect(captured.length).toBeGreaterThan(0); + for (const request of captured) { + expect(request.url.startsWith('https://api.openai.com/')).toBe(true); + expect(request.headers.authorization).toBe('Bearer CUSTOMER'); + expect(JSON.stringify(request)).not.toContain('NEVER_FORWARDED'); + expect(JSON.stringify(request)).not.toContain('ATTACKER'); + } + expect(captured[0].body.top_p).toBe(0.42); + }, 10000); +}); diff --git a/packages/langchain-messages/src/handler.ts b/packages/langchain-messages/src/handler.ts index f4dabcad..479dd86b 100644 --- a/packages/langchain-messages/src/handler.ts +++ b/packages/langchain-messages/src/handler.ts @@ -33,6 +33,7 @@ import { type ToolHandlerFn, } from '@launchdarkly/ai-server'; import { type Context, context, type Span, SpanStatusCode, trace } from '@opentelemetry/api'; +import { type LangChainModelClass, modelConstructorParameters } from './model-parameters.js'; const TRACER_NAME = '@launchdarkly/ai-langchain-messages'; @@ -169,13 +170,16 @@ function normalizeOutputSchema(schema: Record): Record BaseChatModel | Promise); -function modelConstructorArgs(config: AiConfigRep, fallbackName: string): Record { - const parameters = { - ...(config.model?.parameters && typeof config.model.parameters === 'object' ? config.model.parameters : {}), +function modelConstructorArgs( + config: AiConfigRep, + fallbackName: string, + modelClass: LangChainModelClass, +): Record { + // Name from the config always wins; `model` is never on a forwarded-keys list anyway. + return { + ...modelConstructorParameters(config.model?.parameters, modelClass), + model: resolvedModelName(config, fallbackName), }; - if ((config.provider?.name ?? '').toLowerCase() === 'bedrock') delete parameters.tools; - // Name from the config always wins over a colliding `model` key in the parameter bag. - return { ...parameters, model: resolvedModelName(config, fallbackName) }; } /** @@ -184,7 +188,8 @@ function modelConstructorArgs(config: AiConfigRep, fallbackName: string): Record * Otherwise, the provider and model name from the AI config are used to * instantiate the appropriate model via a dynamic import, so that neither * @langchain/openai, @langchain/anthropic, nor @langchain/aws is a hard - * dependency. Parameters are passed through unchanged. + * dependency. Only the model class's allowlisted model.parameters are passed; see + * modelConstructorParameters. */ async function resolveBaseModel(config: AiConfigRep, llm?: LangChainModelSource): Promise { const invocation = configForModelCall(config); @@ -201,7 +206,7 @@ async function resolveBaseModel(config: AiConfigRep, llm?: LangChainModelSource) 'Using Anthropic models requires @langchain/anthropic. Install it with: npm install @langchain/anthropic', ); } - return new mod.ChatAnthropic(modelConstructorArgs(invocation, 'claude-3-5-sonnet-20241022')); + return new mod.ChatAnthropic(modelConstructorArgs(invocation, 'claude-3-5-sonnet-20241022', 'anthropic')); } if (providerName === 'bedrock') { // biome-ignore lint/suspicious/noExplicitAny: @langchain/aws loaded via dynamic import with no static types @@ -211,7 +216,7 @@ async function resolveBaseModel(config: AiConfigRep, llm?: LangChainModelSource) } catch { throw new Error('Using Bedrock models requires @langchain/aws. Install it with: npm install @langchain/aws'); } - return new mod.ChatBedrockConverse(modelConstructorArgs(invocation, '')); + return new mod.ChatBedrockConverse(modelConstructorArgs(invocation, '', 'bedrock')); } // biome-ignore lint/suspicious/noExplicitAny: @langchain/openai loaded via dynamic import with no static types let mod: any; @@ -220,7 +225,7 @@ async function resolveBaseModel(config: AiConfigRep, llm?: LangChainModelSource) } catch { throw new Error('Using OpenAI models requires @langchain/openai. Install it with: npm install @langchain/openai'); } - return new mod.ChatOpenAI(modelConstructorArgs(invocation, 'gpt-4o')); + return new mod.ChatOpenAI(modelConstructorArgs(invocation, 'gpt-4o', 'openai')); } const buildTools = ( diff --git a/packages/langchain-messages/src/model-parameters.ts b/packages/langchain-messages/src/model-parameters.ts new file mode 100644 index 00000000..9c56c87e --- /dev/null +++ b/packages/langchain-messages/src/model-parameters.ts @@ -0,0 +1,146 @@ +import type { ChatAnthropicInput } from '@langchain/anthropic'; +import type { ChatBedrockConverseInput } from '@langchain/aws'; +import type { ChatOpenAIFields } from '@langchain/openai'; +import { + type AiConfigRep, + camelizeModelParameters, + normalizeModelParameters, + pickForwardedModelParameters, +} from '@launchdarkly/ai-server'; + +/** + * Constructor fields every LangChain chat model takes from `BaseChatModelParams` that a config + * can set as JSON. `callbacks`, `callbackManager`, `cache` and `maxConcurrency` are wiring, and + * `maxRetries` is client retry policy, so none of them are forwarded. + */ +const BASE_CHAT_MODEL_KEYS = ['disableStreaming', 'metadata', 'outputVersion', 'tags', 'verbose'] as const; + +/** + * `ChatOpenAI` constructor fields this package takes from `config.model.parameters`, after + * camelizing the bag. Mirrors the Python SDK's `ChatOpenAI` list. + * + * Never forwarded: `apiKey`, `openAIApiKey`, `configuration` (client options, including the base + * URL, default headers, fetch and proxy settings), `timeout`, `maxRetries`, `modelKwargs` (merged + * into the request body as written), `prefixMessages` (prompt content), `model` / `modelName` + * (the handler sets the model), and the client internals `completions`, `responses`, + * `__includeRawResponse` and `supportsStrictToolCalling`. + */ +export const CHAT_OPENAI_FORWARDED_KEYS = [ + ...BASE_CHAT_MODEL_KEYS, + 'audio', + 'frequencyPenalty', + 'logitBias', + 'logprobs', + 'maxCompletionTokens', + 'maxTokens', + 'modalities', + 'n', + 'presencePenalty', + 'promptCacheKey', + 'promptCacheRetention', + 'reasoning', + 'service_tier', + 'stop', + 'stopSequences', + 'streamUsage', + 'streaming', + 'temperature', + 'topLogprobs', + 'topP', + 'useResponsesApi', + 'user', + 'verbosity', + 'zdrEnabled', +] as const satisfies ReadonlyArray; + +/** + * `ChatAnthropic` constructor fields this package takes from `config.model.parameters`, after + * camelizing the bag. Mirrors the Python SDK's `ChatAnthropic` list, minus `mcp_servers` (remote + * tools) and `model_kwargs` (request injection), neither of which a config may set. + * + * Never forwarded: `apiKey`, `anthropicApiKey`, `anthropicApiUrl`, `clientOptions` (including + * default headers and fetch), `createClient`, `invocationKwargs` (merged into the request body as + * written), and `model` / `modelName`. + * + * Nested values (`thinking`, `contextManagement`, `outputConfig`) are the Anthropic Messages API's + * own snake_case shapes, which `ChatAnthropic` sends as written, so they are not converted. A + * top-level `effort` (the name the UI writes) becomes `outputConfig.effort`; see + * `modelConstructorParameters`. + */ +export const CHAT_ANTHROPIC_FORWARDED_KEYS = [ + ...BASE_CHAT_MODEL_KEYS, + 'betas', + 'contextManagement', + 'inferenceGeo', + 'maxTokens', + 'outputConfig', + 'stopSequences', + 'streamUsage', + 'streaming', + 'temperature', + 'thinking', + 'topK', + 'topP', +] as const satisfies ReadonlyArray; + +/** + * `ChatBedrockConverse` constructor fields this package takes from `config.model.parameters`, + * after camelizing the bag. Mirrors the Python SDK's `ChatBedrockConverse` list, minus + * `region_name`, which is connection configuration. + * + * Never forwarded: `additionalModelRequestFields` (merged into the request body as written), + * `region`, `credentials`, `bedrockApiKey`, `bedrockApiSecret`, + * `bedrockApiSessionToken`, the AWS credential-provider settings (`profile`, `roleArn`, ...), + * `endpointHost`, `client`, `clientOptions`, `defaultHeaders`, `model`, and + * `applicationInferenceProfile` (which replaces the model the handler resolved). + */ +export const CHAT_BEDROCK_CONVERSE_FORWARDED_KEYS = [ + ...BASE_CHAT_MODEL_KEYS, + 'guardrailConfig', + 'maxTokens', + 'performanceConfig', + 'serviceTier', + 'streamUsage', + 'streaming', + 'supportsToolChoiceValues', + 'temperature', + 'topP', +] as const satisfies ReadonlyArray; + +export type LangChainModelClass = 'openai' | 'anthropic' | 'bedrock'; + +const FORWARDED_KEYS: Record> = { + openai: CHAT_OPENAI_FORWARDED_KEYS, + anthropic: CHAT_ANTHROPIC_FORWARDED_KEYS, + bedrock: CHAT_BEDROCK_CONVERSE_FORWARDED_KEYS, +}; + +/** + * `config.model.parameters` as constructor fields for one LangChain chat model class: camelized + * (`top_p` becomes `topP`; nested values are left alone, and snake_case wins when both spellings + * are set), then narrowed to that class's allowlist. Unknown keys are dropped. + * + * Renames before the allowlist is applied: + * - `ChatOpenAI` reads its service tier as `service_tier`, so a camelized `serviceTier` is + * renamed back. + * - `ChatAnthropic` has no `effort` field; a top-level `effort` becomes `outputConfig.effort`. + * An explicit `output_config.effort` wins. + */ +export function modelConstructorParameters( + parameters: AiConfigRep['model']['parameters'] | unknown, + modelClass: LangChainModelClass, +): Record { + const camelized = camelizeModelParameters(normalizeModelParameters(parameters)); + if (modelClass === 'openai' && camelized.serviceTier !== undefined) { + camelized.service_tier = camelized.serviceTier; + delete camelized.serviceTier; + } + if (modelClass === 'anthropic' && camelized.effort !== undefined) { + const outputConfig = + camelized.outputConfig && typeof camelized.outputConfig === 'object' && !Array.isArray(camelized.outputConfig) + ? (camelized.outputConfig as Record) + : {}; + camelized.outputConfig = { effort: camelized.effort, ...outputConfig }; + } + return pickForwardedModelParameters(camelized, FORWARDED_KEYS[modelClass]); +} diff --git a/packages/openai-agents/src/__tests__/handler.test.ts b/packages/openai-agents/src/__tests__/handler.test.ts index 17c05cfa..abceb97d 100644 --- a/packages/openai-agents/src/__tests__/handler.test.ts +++ b/packages/openai-agents/src/__tests__/handler.test.ts @@ -101,6 +101,7 @@ vi.mock('@launchdarkly/ai-server', async (importOriginal) => { import { openaiGraph } from '../graph.js'; import { createOpenAIAgentHandler } from '../handler.js'; +import { expectNoNeverForwardedValue, NEVER_FORWARDED_PARAMETERS } from './never-forwarded.js'; // ─── Fixtures ───────────────────────────────────────────────────────────────── @@ -227,6 +228,145 @@ describe('createOpenAIAgentHandler', () => { expect(mockAgentConstructor.mock.calls[0][0].instructions).toBe('From instructions.'); }); + // ── 1.2b model.parameters forwarding ───────────────────────────────────────── + + it('forwards recognized model.parameters keys to Agent modelSettings', async () => { + mockRun.mockResolvedValue(mockRunResult()); + const config = { + ...baseConfig, + model: { + ...baseConfig.model, + parameters: { temperature: 0.4, topP: 0.9, maxTokens: 256, frequencyPenalty: 0.1, presencePenalty: 0.2 }, + }, + }; + await createOpenAIAgentHandler()(config as any, 'q'); + const agentArgs = mockAgentConstructor.mock.calls[0][0]; + expect(agentArgs.modelSettings).toEqual({ + temperature: 0.4, + topP: 0.9, + maxTokens: 256, + frequencyPenalty: 0.1, + presencePenalty: 0.2, + }); + }); + + it('drops model.parameters keys that are not ModelSettings fields', async () => { + mockRun.mockResolvedValue(mockRunResult()); + const config = { + ...baseConfig, + model: { ...baseConfig.model, parameters: { someUnknownKey: 'nope', anotherOne: 42, temperature: 0.1 } }, + }; + await createOpenAIAgentHandler()(config as any, 'q'); + const agentArgs = mockAgentConstructor.mock.calls[0][0]; + expect(agentArgs.modelSettings).toEqual({ temperature: 0.1 }); + }); + + it('never forwards request headers, body or query overrides, credentials or retry policy', async () => { + mockRun.mockResolvedValue(mockRunResult()); + const config = { ...baseConfig, model: { ...baseConfig.model, parameters: NEVER_FORWARDED_PARAMETERS } }; + await createOpenAIAgentHandler()(config as any, 'q'); + const agentArgs = mockAgentConstructor.mock.calls[0][0]; + expect(agentArgs.modelSettings).toBeUndefined(); + expectNoNeverForwardedValue(agentArgs); + expect(mockRun.mock.calls[0][2]).toBeUndefined(); + }); + + it('forwards every allowed ModelSettings field under its SDK name and shape', async () => { + mockRun.mockResolvedValue(mockRunResult()); + const config = { + ...baseConfig, + model: { + ...baseConfig.model, + parameters: { + temperature: 0.4, + top_p: 0.9, + frequency_penalty: 0.1, + presence_penalty: 0.2, + tool_choice: 'auto', + parallel_tool_calls: false, + truncation: 'auto', + max_tokens: 256, + store: false, + prompt_cache_retention: '24h', + reasoning: { effort: 'low', summary: 'auto', extra_key: 'dropped' }, + verbosity: 'low', + context_management: [ + { type: 'compaction', compact_threshold: 1000, extra_key: 'dropped' }, + { type: 'compaction', compactThreshold: 2000 }, + { compact_threshold: 3000 }, + 'not-an-entry', + ], + retry: { maxRetries: 5 }, + provider_data: { extra_headers: { Authorization: 'Bearer ATTACKER' } }, + }, + }, + }; + await createOpenAIAgentHandler()(config as any, 'q'); + expect(mockAgentConstructor.mock.calls[0][0].modelSettings).toEqual({ + temperature: 0.4, + topP: 0.9, + frequencyPenalty: 0.1, + presencePenalty: 0.2, + toolChoice: 'auto', + parallelToolCalls: false, + truncation: 'auto', + maxTokens: 256, + store: false, + promptCacheRetention: '24h', + reasoning: { effort: 'low', summary: 'auto' }, + text: { verbosity: 'low' }, + // Entries are rebuilt as { type, compactThreshold }; ones without a type are dropped. + contextManagement: [ + { type: 'compaction', compactThreshold: 1000 }, + { type: 'compaction', compactThreshold: 2000 }, + ], + }); + }); + + it('prefers an explicit text.verbosity over a top-level verbosity', async () => { + mockRun.mockResolvedValue(mockRunResult()); + const config = { + ...baseConfig, + model: { ...baseConfig.model, parameters: { verbosity: 'low', text: { verbosity: 'high', format: 'x' } } }, + }; + await createOpenAIAgentHandler()(config as any, 'q'); + expect(mockAgentConstructor.mock.calls[0][0].modelSettings).toEqual({ text: { verbosity: 'high' } }); + }); + + it('does not set modelSettings or maxTurns when model.parameters is absent', async () => { + mockRun.mockResolvedValue(mockRunResult()); + await createOpenAIAgentHandler()(baseConfig as any, 'q'); + const agentArgs = mockAgentConstructor.mock.calls[0][0]; + expect(agentArgs.modelSettings).toBeUndefined(); + const runOptions = mockRun.mock.calls[0][2]; + expect(runOptions).toBeUndefined(); + }); + + it('forwards model.parameters.maxTurns to Runner.run options', async () => { + mockRun.mockResolvedValue(mockRunResult()); + const config = { ...baseConfig, model: { ...baseConfig.model, parameters: { maxTurns: 3 } } }; + await createOpenAIAgentHandler()(config as any, 'q'); + const runOptions = mockRun.mock.calls[0][2]; + expect(runOptions).toEqual({ maxTurns: 3 }); + }); + + // The LaunchDarkly UI writes model.parameters in snake_case (the wire name every provider + // client expects), but the Agents SDK's ModelSettings/Runner.run only read camelCase — so a + // config saved as `max_turns` / `top_p` must reach the Agents SDK as `maxTurns` / `topP`. + it('converts snake_case model.parameters keys to camelCase for modelSettings and Runner.run', async () => { + mockRun.mockResolvedValue(mockRunResult()); + const config = { + ...baseConfig, + model: { ...baseConfig.model, parameters: { max_turns: 4, top_p: 0.9, max_tokens: 256 } }, + }; + await createOpenAIAgentHandler()(config as any, 'q'); + const agentArgs = mockAgentConstructor.mock.calls[0][0]; + // maxTurns is a Runner.run option, so it is not left in modelSettings. + expect(agentArgs.modelSettings).toEqual({ topP: 0.9, maxTokens: 256 }); + const runOptions = mockRun.mock.calls[0][2]; + expect(runOptions).toEqual({ maxTurns: 4 }); + }); + // ── 1.3 Tool conversion ───────────────────────────────────────────────────── it('creates agent tools when config.tools is present', async () => { @@ -560,6 +700,54 @@ describe('createOpenAIAgentHandler', () => { expect(done.usage).toMatchObject({ input_tokens: 4, output_tokens: 6 }); }); + it('forwards model.parameters (modelSettings and maxTurns) on the streaming path', async () => { + const streamedResult = { + [Symbol.asyncIterator]: async function* () {}, + state: { usage: { inputTokens: 1, outputTokens: 1, totalTokens: 2 } }, + finalOutput: '', + }; + mockRun.mockResolvedValue(streamedResult); + const config = { + ...baseConfig, + model: { ...baseConfig.model, parameters: { temperature: 0.5, maxTurns: 5, unknownKey: 'nope' } }, + }; + const handler = createOpenAIAgentHandler(); + await collectStream(handler.stream?.(config as any, 'q', {}, {})); + // `maxTurns` goes to Runner.run only; the unknown key is dropped. + expect(mockAgentConstructor.mock.calls[0][0].modelSettings).toEqual({ temperature: 0.5 }); + const runOptions = mockRun.mock.calls[0][2]; + expect(runOptions).toMatchObject({ maxTurns: 5 }); + }); + + it('converts a snake_case max_turns to maxTurns on the streaming path', async () => { + const streamedResult = { + [Symbol.asyncIterator]: async function* () {}, + state: { usage: { inputTokens: 1, outputTokens: 1, totalTokens: 2 } }, + finalOutput: '', + }; + mockRun.mockResolvedValue(streamedResult); + const config = { ...baseConfig, model: { ...baseConfig.model, parameters: { max_turns: 6, top_p: 0.8 } } }; + const handler = createOpenAIAgentHandler(); + await collectStream(handler.stream?.(config as any, 'q', {}, {})); + expect(mockAgentConstructor.mock.calls[0][0].modelSettings).toEqual({ topP: 0.8 }); + const runOptions = mockRun.mock.calls[0][2]; + expect(runOptions).toMatchObject({ maxTurns: 6 }); + }); + + it('does not set modelSettings or maxTurns on the streaming path when model.parameters is absent', async () => { + const streamedResult = { + [Symbol.asyncIterator]: async function* () {}, + state: { usage: { inputTokens: 1, outputTokens: 1, totalTokens: 2 } }, + finalOutput: '', + }; + mockRun.mockResolvedValue(streamedResult); + const handler = createOpenAIAgentHandler(); + await collectStream(handler.stream?.(baseConfig as any, 'q', {}, {})); + expect(mockAgentConstructor.mock.calls[0][0].modelSettings).toBeUndefined(); + const runOptions = mockRun.mock.calls[0][2]; + expect(runOptions.maxTurns).toBeUndefined(); + }); + it('sets invoke_agent on the root and emits a chat span on streaming', async () => { mockInnerGetStreamed.mockImplementation(async function* () { yield { type: 'response_done', response: { usage: { inputTokens: 1, outputTokens: 1, totalTokens: 2 } } }; diff --git a/packages/openai-agents/src/__tests__/native-graph.test.ts b/packages/openai-agents/src/__tests__/native-graph.test.ts index 6d1b11bc..158653f2 100644 --- a/packages/openai-agents/src/__tests__/native-graph.test.ts +++ b/packages/openai-agents/src/__tests__/native-graph.test.ts @@ -65,6 +65,7 @@ vi.mock('@launchdarkly/ai-server', async (importOriginal) => { }); import { toOpenAIAgents } from '../native-graph.js'; +import { expectNoNeverForwardedValue, NEVER_FORWARDED_PARAMETERS } from './never-forwarded.js'; // ─── Fixture helpers ────────────────────────────────────────────────────────── @@ -134,6 +135,38 @@ describe('toOpenAIAgents', () => { await expect(toOpenAIAgents(Promise.resolve(def as any)).invoke('hi')).rejects.toThrow(/root/i); }); + // ── model.parameters forwarding ─────────────────────────────────────────── + + it("passes each node's allowlisted model.parameters as that node's modelSettings, and the root's maxTurns to run()", async () => { + const def = makeTwoNodeGraph(); + (def.root.config.model as any).parameters = { temperature: 0.2, max_turns: 7, top_p: 0.5 }; + (def.getNode('leaf-agent')!.config.model as any).parameters = { max_tokens: 64, max_turns: 2 }; + await toOpenAIAgents(Promise.resolve(def as any)).invoke('hi'); + const byName = Object.fromEntries(mockAgentConstructor.mock.calls.map(([args]) => [args.name, args])); + expect(byName['root-agent'].modelSettings).toEqual({ temperature: 0.2, topP: 0.5 }); + expect(byName['leaf-agent'].modelSettings).toEqual({ maxTokens: 64 }); + // One run drives the whole graph, so only the root's turn cap applies, as in the Python SDK. + expect(mockRunnerRun.mock.calls[0][2]).toEqual({ maxTurns: 7 }); + }); + + it('sets no modelSettings and no run options when no node has model.parameters', async () => { + await toOpenAIAgents(Promise.resolve(makeTwoNodeGraph())).invoke('hi'); + for (const [args] of mockAgentConstructor.mock.calls) { + expect(args).not.toHaveProperty('modelSettings'); + } + expect(mockRunnerRun.mock.calls[0][2]).toBeUndefined(); + }); + + it('never forwards request overrides, credentials or retry policy from any node', async () => { + const def = makeTwoNodeGraph(); + (def.root.config.model as any).parameters = NEVER_FORWARDED_PARAMETERS; + (def.getNode('leaf-agent')!.config.model as any).parameters = NEVER_FORWARDED_PARAMETERS; + await toOpenAIAgents(Promise.resolve(def as any)).invoke('hi'); + for (const [args] of mockAgentConstructor.mock.calls) { + expectNoNeverForwardedValue(args); + } + }); + // ── Topology translation ────────────────────────────────────────────────── it('creates one Agent per node in a two-node graph', async () => { diff --git a/packages/openai-agents/src/__tests__/never-forwarded.ts b/packages/openai-agents/src/__tests__/never-forwarded.ts new file mode 100644 index 00000000..300d6e51 --- /dev/null +++ b/packages/openai-agents/src/__tests__/never-forwarded.ts @@ -0,0 +1,163 @@ +import { expect } from 'vitest'; + +/** + * The value every never-forwarded key carries in `NEVER_FORWARDED_PARAMETERS`. A handler that + * passes any of those keys on, under any name or nested anywhere, leaves this string somewhere in + * the options it builds, which is what `expectNoNeverForwardedValue` looks for. + */ +export const NEVER_FORWARDED = 'NEVER_FORWARDED'; + +/** + * Keys that must never reach a provider, framework or host-process option from + * `model.parameters`, in both the snake_case the LaunchDarkly UI writes and the camelCase the + * frameworks read: credentials, endpoints and connection settings, request injection, remote + * tools, and Claude Agents host-process settings. + */ +export const NEVER_FORWARDED_KEYS = [ + // credentials + 'api_key', + 'apiKey', + 'openai_api_key', + 'openAIApiKey', + 'anthropic_api_key', + 'anthropicApiKey', + 'bedrock_api_key', + 'bedrockApiKey', + 'bedrock_api_secret', + 'bedrockApiSecret', + 'credentials', + 'auth_token', + 'authToken', + 'access_token', + 'accessToken', + 'aws_access_key_id', + 'awsAccessKeyId', + 'aws_secret_access_key', + 'awsSecretAccessKey', + 'aws_session_token', + 'awsSessionToken', + 'organization', + // endpoints and connection + 'base_url', + 'baseURL', + 'baseUrl', + 'anthropic_api_url', + 'anthropicApiUrl', + 'endpoint', + 'endpoint_host', + 'endpointHost', + 'endpoint_url', + 'endpointUrl', + 'region', + 'region_name', + 'regionName', + 'configuration', + 'client_options', + 'clientOptions', + 'default_headers', + 'defaultHeaders', + 'timeout', + 'max_retries', + 'maxRetries', + 'fetch', + 'http_client', + 'httpClient', + 'http_agent', + 'httpAgent', + 'proxy', + 'proxies', + 'client', + // request injection + 'headers', + 'extra_headers', + 'extraHeaders', + 'extra_body', + 'extraBody', + 'extra_query', + 'extraQuery', + 'model_kwargs', + 'modelKwargs', + 'invocation_kwargs', + 'invocationKwargs', + 'additional_model_request_fields', + 'additionalModelRequestFields', + 'provider_data', + 'providerData', + 'provider_options', + 'providerOptions', + // remote tools + 'mcp_servers', + 'mcpServers', + // Claude Agents host-process settings + 'path_to_claude_code_executable', + 'pathToClaudeCodeExecutable', + 'cli_path', + 'cliPath', + 'executable', + 'executable_args', + 'executableArgs', + 'extra_args', + 'extraArgs', + 'env', + 'cwd', + 'additional_directories', + 'additionalDirectories', + 'add_dirs', + 'addDirs', + 'permission_mode', + 'permissionMode', + 'allow_dangerously_skip_permissions', + 'allowDangerouslySkipPermissions', + 'settings', + 'setting_sources', + 'settingSources', + 'managed_settings', + 'managedSettings', + 'plugins', + 'sandbox', + 'resume', + 'resume_session_at', + 'resumeSessionAt', + 'session_id', + 'sessionId', + 'fork_session', + 'forkSession', + 'continue', + 'hooks', + 'can_use_tool', + 'canUseTool', + 'stderr', + 'abort_controller', + 'abortController', + 'spawn_claude_code_process', + 'spawnClaudeCodeProcess', +] as const; + +/** A `model.parameters` bag that sets every key in `NEVER_FORWARDED_KEYS` to `NEVER_FORWARDED`. */ +export const NEVER_FORWARDED_PARAMETERS: Record = Object.fromEntries( + NEVER_FORWARDED_KEYS.map((key) => [key, NEVER_FORWARDED]), +); + +function findNeverForwarded(value: unknown, path: string, seen: Set): string | undefined { + if (value === NEVER_FORWARDED) return path || '(root)'; + if (!value || typeof value !== 'object' || seen.has(value)) return undefined; + seen.add(value); + for (const [key, child] of Object.entries(value)) { + const found = findNeverForwarded(child, path ? `${path}.${key}` : key, seen); + if (found) return found; + } + return undefined; +} + +/** + * Asserts that no `NEVER_FORWARDED` value appears anywhere in `options`, and that none of the + * `NEVER_FORWARDED_KEYS` the handler does not own itself is set on it. `handlerOwned` lists the + * keys the handler legitimately sets from its own state (Claude Agents sets `mcpServers` and + * `hooks`, for example); those must still not carry the config's value. + */ +export function expectNoNeverForwardedValue(options: unknown, handlerOwned: ReadonlyArray = []): void { + expect(findNeverForwarded(options, '', new Set())).toBeUndefined(); + const record = options as Record; + const leaked = NEVER_FORWARDED_KEYS.filter((key) => !handlerOwned.includes(key) && record[key] !== undefined); + expect(leaked).toEqual([]); +} diff --git a/packages/openai-agents/src/handler.ts b/packages/openai-agents/src/handler.ts index 557f934e..c1e9934b 100644 --- a/packages/openai-agents/src/handler.ts +++ b/packages/openai-agents/src/handler.ts @@ -39,6 +39,7 @@ import type { } from '@openai/agents'; import { Agent, Runner, tool } from '@openai/agents'; import { type Context, context, type Span, SpanStatusCode, trace } from '@opentelemetry/api'; +import { buildMaxTurns, buildModelSettings } from './model-parameters.js'; import { buildOutputType } from './utils.js'; const TRACER_NAME = '@launchdarkly/ai-openai-agents'; @@ -536,6 +537,7 @@ function buildAgentAndPrompt( const tools = config.tools ? buildAgentTools(config.tools, toolHandlers) : []; const outputType = includeOutputType ? buildOutputType(config.outputFormat) : undefined; + const modelSettings = buildModelSettings(config.model.parameters); const agent = new Agent({ name: 'assistant', @@ -543,6 +545,7 @@ function buildAgentAndPrompt( ...(instructions ? { instructions } : {}), ...(tools.length > 0 ? { tools } : {}), ...(outputType ? { outputType } : {}), + ...(modelSettings ? { modelSettings } : {}), }); return { agent, prompt, instructions }; @@ -606,8 +609,9 @@ export function createOpenAIAgentHandler({ captureContent = false }: ContentCapt modelProvider: new SpanningModelProvider(defaultModelProvider(), config, parentContext, captureContent), }); try { + const maxTurns = buildMaxTurns(config.model.parameters); // biome-ignore lint/suspicious/noExplicitAny: Runner.run accepts string | AgentInputItem[]; our item shape is structurally compatible - const result = await runner.run(agent, prompt as any); + const result = await runner.run(agent, prompt as any, maxTurns !== undefined ? { maxTurns } : undefined); const finalOutput = result.finalOutput ?? ''; const { inputTokens, outputTokens } = result.state.usage; @@ -673,8 +677,10 @@ export function createOpenAIAgentHandler({ captureContent = false }: ContentCapt modelProvider: new SpanningModelProvider(defaultModelProvider(), config, parentContext, captureContent), }); try { + const maxTurns = buildMaxTurns(config.model.parameters); + const runOptions = { stream: true, signal: abortRun.signal, ...(maxTurns !== undefined ? { maxTurns } : {}) }; // biome-ignore lint/suspicious/noExplicitAny: Agents SDK run() stream overload requires an any-cast option - const streamed = await runner.run(agent, prompt as any, { stream: true, signal: abortRun.signal } as any); + const streamed = await runner.run(agent, prompt as any, runOptions as any); // biome-ignore lint/suspicious/noExplicitAny: StreamedRunResult generics are irrelevant to this handler const streamedResult = streamed as StreamedRunResult; let fullOutput = ''; diff --git a/packages/openai-agents/src/model-parameters.ts b/packages/openai-agents/src/model-parameters.ts new file mode 100644 index 00000000..54422a31 --- /dev/null +++ b/packages/openai-agents/src/model-parameters.ts @@ -0,0 +1,122 @@ +import { + type AiConfigRep, + camelizeModelParameters, + normalizeModelParameters, + pickForwardedModelParameters, +} from '@launchdarkly/ai-server'; +import type { ModelSettings } from '@openai/agents'; + +/** + * `ModelSettings` fields this package takes from `config.model.parameters`, after camelizing the + * bag (`top_p` becomes `topP`). Every other key is dropped. + */ +const FORWARDED_MODEL_SETTINGS_KEYS = [ + 'contextManagement', + 'frequencyPenalty', + 'maxTokens', + 'parallelToolCalls', + 'presencePenalty', + 'promptCacheRetention', + 'reasoning', + 'store', + 'temperature', + 'text', + 'toolChoice', + 'topP', + 'truncation', +] as const satisfies ReadonlyArray; + +/** + * `ModelSettings` fields that are never taken from a config: + * + * - `providerData` is merged into the Responses API request as written, and carries the Agents + * SDK's transport overrides (`extraHeaders`, `extraBody`, `extraQuery`), so a config could + * replace the request's headers (including authorization) or body. + * - `retry` is client retry policy, not a model setting. + */ +type ExcludedModelSettingsKeys = 'providerData' | 'retry'; +// If the Agents SDK adds a ModelSettings field and it is not classified above as forwarded or +// excluded, this type resolves to something other than `never` and the assignment below fails to +// compile, naming the unclassified key. +type UndecidedModelSettingsKeys = Exclude< + keyof ModelSettings, + ExcludedModelSettingsKeys | (typeof FORWARDED_MODEL_SETTINGS_KEYS)[number] +>; +const _modelSettingsKeysExhaustive: Record = {} as Record; + +function isPlainObject(value: unknown): value is Record { + return !!value && typeof value === 'object' && !Array.isArray(value); +} + +/** Keeps only the named keys of a nested object, or returns `undefined` if none are set. */ +function pickNested(value: unknown, keys: ReadonlyArray): Record | undefined { + if (!isPlainObject(value)) return undefined; + const picked: Record = {}; + for (const key of keys) { + if (value[key] !== undefined) picked[key] = value[key]; + } + return Object.keys(picked).length > 0 ? picked : undefined; +} + +/** + * `context_management` as `ModelSettings.contextManagement`: each entry is rebuilt as + * `{ type, compactThreshold }`, accepting `compact_threshold` (the Responses API spelling the UI + * writes) or `compactThreshold`. The Agents SDK snake-cases every key of an entry into the + * request, so any other key is dropped rather than passed along. Entries without a string `type` + * are dropped. + */ +function toContextManagement(value: unknown): ModelSettings['contextManagement'] | undefined { + if (!Array.isArray(value)) return undefined; + const entries = value.flatMap((entry) => { + if (!isPlainObject(entry) || typeof entry.type !== 'string') return []; + const threshold = entry.compact_threshold ?? entry.compactThreshold; + return [{ type: entry.type, ...(typeof threshold === 'number' ? { compactThreshold: threshold } : {}) }]; + }); + return entries.length > 0 ? entries : undefined; +} + +/** + * `config.model.parameters` as the Agents SDK's `ModelSettings`: camelized, narrowed to + * `FORWARDED_MODEL_SETTINGS_KEYS`, with nested values rebuilt in the shapes the SDK reads: + * + * - `reasoning` keeps `effort` and `summary`. + * - `text` keeps `verbosity`. A top-level `verbosity` (the name the UI and the Python SDK use) + * becomes `text.verbosity`; an explicit `text.verbosity` wins. + * - `context_management` entries are rebuilt by `toContextManagement`. + * + * `max_turns` is not a `ModelSettings` field; see `buildMaxTurns`. A config that sets none of + * these produces `undefined`, so the Agent is constructed exactly as it always has been. + */ +export function buildModelSettings( + parameters: AiConfigRep['model']['parameters'] | unknown, +): ModelSettings | undefined { + const camelized = camelizeModelParameters(normalizeModelParameters(parameters)); + const settings = pickForwardedModelParameters(camelized, FORWARDED_MODEL_SETTINGS_KEYS); + + const reasoning = pickNested(settings.reasoning, ['effort', 'summary']); + if (reasoning) settings.reasoning = reasoning; + else delete settings.reasoning; + + const text = pickNested({ verbosity: camelized.verbosity, ...pickNested(settings.text, ['verbosity']) }, [ + 'verbosity', + ]); + if (text) settings.text = text; + else delete settings.text; + + const contextManagement = toContextManagement(settings.contextManagement); + if (contextManagement) settings.contextManagement = contextManagement; + else delete settings.contextManagement; + + return Object.keys(settings).length > 0 ? (settings as ModelSettings) : undefined; +} + +/** + * `maxTurns` is a `Runner.run` option, not a `ModelSettings` field: it caps the agentic loop + * rather than tuning any single model call, so it is read out of `model.parameters` separately + * and forwarded to `run()` instead of the `Agent` constructor. Both `max_turns` (the UI's + * spelling) and `maxTurns` are read. + */ +export function buildMaxTurns(parameters: AiConfigRep['model']['parameters'] | unknown): number | undefined { + const maxTurns = camelizeModelParameters(normalizeModelParameters(parameters)).maxTurns; + return typeof maxTurns === 'number' ? maxTurns : undefined; +} diff --git a/packages/openai-agents/src/native-graph.ts b/packages/openai-agents/src/native-graph.ts index 48d251b3..d100e7e2 100644 --- a/packages/openai-agents/src/native-graph.ts +++ b/packages/openai-agents/src/native-graph.ts @@ -16,6 +16,7 @@ import { } from '@launchdarkly/ai-server'; import { Agent, handoff, Runner, tool } from '@openai/agents'; import { SpanStatusCode, trace } from '@opentelemetry/api'; +import { buildMaxTurns, buildModelSettings } from './model-parameters.js'; // ─── helpers ───────────────────────────────────────────────────────────────── @@ -150,9 +151,11 @@ export const toOpenAIAgents = ( const agentName = sanitizeName(node.key); agentNameToKey.set(agentName, node.key); + const modelSettings = buildModelSettings(node.config.model.parameters); const agent = new Agent({ name: agentName, model: node.config.model.name, + ...(modelSettings ? { modelSettings } : {}), ...(instructions ? { instructions } : {}), ...(tools.length > 0 ? { tools } : {}), ...(childHandoffs.length > 0 ? { handoffs: childHandoffs } : {}), @@ -214,11 +217,19 @@ export const toOpenAIAgents = ( // handoffs and receive their context from the Runner, not from `history`. const rootInput = history && history.length > 0 ? toRunnerInput(history, input) : input; + // One Runner.run drives the whole graph through handoffs, so the turn cap is the root + // node's `max_turns`, as in the Python SDK. + const maxTurns = buildMaxTurns(root.config.model.parameters); + // biome-ignore lint/suspicious/noImplicitAnyLet: assigned immediately in try; catch always re-throws let result; try { // biome-ignore lint/suspicious/noExplicitAny: Runner.run accepts string | AgentInputItem[]; our item shape is structurally compatible - result = await runner.run(rootAgent, rootInput as any); + const runInput = rootInput as any; + result = + maxTurns !== undefined + ? await runner.run(rootAgent, runInput, { maxTurns }) + : await runner.run(rootAgent, runInput); span.setStatus({ code: SpanStatusCode.OK }); } catch (err) { span.recordException(err instanceof Error ? err : new Error(String(err))); diff --git a/packages/openai-messages/src/__tests__/handler.test.ts b/packages/openai-messages/src/__tests__/handler.test.ts index 1659a83e..4d120fb8 100644 --- a/packages/openai-messages/src/__tests__/handler.test.ts +++ b/packages/openai-messages/src/__tests__/handler.test.ts @@ -57,6 +57,7 @@ vi.mock('@launchdarkly/ai-server', async (importOriginal) => { }); import { createOpenAIHandler } from '../handler.js'; +import { expectNoNeverForwardedValue, NEVER_FORWARDED_PARAMETERS } from './never-forwarded.js'; // ─── Fixtures ───────────────────────────────────────────────────────────────── @@ -301,6 +302,193 @@ describe('createOpenAIHandler', () => { expect(call.tools ?? []).toHaveLength(0); }); + // ── 1.3b model.parameters forwarding ──────────────────────────────────────── + + it('forwards allowlisted model.parameters fields to the provider call', async () => { + mockResponsesCreate.mockResolvedValue(mockFinalResponse()); + const config = { + ...baseConfig, + model: { + ...baseConfig.model, + parameters: { temperature: 0.4, top_p: 0.9, max_output_tokens: 256 }, + }, + }; + await createOpenAIHandler()(config as any, 'q'); + const call = mockResponsesCreate.mock.calls[0][0]; + expect(call.temperature).toBe(0.4); + expect(call.top_p).toBe(0.9); + expect(call.max_output_tokens).toBe(256); + }); + + it('never forwards credentials, endpoints, request overrides or remote tools from model.parameters', async () => { + mockResponsesCreate.mockResolvedValue(mockFinalResponse()); + const cfg = { ...baseConfig, model: { ...baseConfig.model, parameters: NEVER_FORWARDED_PARAMETERS } }; + await createOpenAIHandler()(cfg as any, 'hi'); + expect(mockResponsesCreate).toHaveBeenCalledTimes(1); + for (const arg of mockResponsesCreate.mock.calls[0]) { + expectNoNeverForwardedValue(arg); + } + }); + + it('drops an unrecognized model.parameters key rather than forwarding it', async () => { + // Unlike the four framework handlers, this handler wraps the raw Responses API client: an + // unsupported key reaching responses.create is a wire-level 400, not a harmlessly ignored extra. + mockResponsesCreate.mockResolvedValue(mockFinalResponse()); + const config = { + ...baseConfig, + model: { ...baseConfig.model, parameters: { foo: 'bar' } }, + }; + await createOpenAIHandler()(config as any, 'q'); + const call = mockResponsesCreate.mock.calls[0][0]; + expect(call).not.toHaveProperty('foo'); + }); + + // The rule is: exclude a key only if setting it would break the handler. These keys are real + // Responses API fields with no obvious generation effect, but a config that sets one still + // gets a working call, so they are forwarded rather than excluded. + it('forwards keys with no generation effect (store, user, safety_identifier, prompt_cache_key, prompt_cache_retention, include, context_management)', async () => { + mockResponsesCreate.mockResolvedValue(mockFinalResponse()); + const config = { + ...baseConfig, + model: { + ...baseConfig.model, + parameters: { + store: true, + user: 'user-123', + safety_identifier: 'user-123', + prompt_cache_key: 'cache-key', + prompt_cache_retention: '24h', + include: ['file_search_call.results'], + context_management: [{ type: 'compaction' }], + }, + }, + }; + await createOpenAIHandler()(config as any, 'q'); + const call = mockResponsesCreate.mock.calls[0][0]; + expect(call.store).toBe(true); + expect(call.user).toBe('user-123'); + expect(call.safety_identifier).toBe('user-123'); + expect(call.prompt_cache_key).toBe('cache-key'); + expect(call.prompt_cache_retention).toBe('24h'); + expect(call.include).toEqual(['file_search_call.results']); + expect(call.context_management).toEqual([{ type: 'compaction' }]); + }); + + it('drops background, conversation, and prompt, which would break the handler', async () => { + // background: the call would return before the output exists, so the handler gets no result. + // conversation, prompt: server-side state/templates that conflict with the input this handler + // builds itself. + mockResponsesCreate.mockResolvedValue(mockFinalResponse()); + const config = { + ...baseConfig, + model: { + ...baseConfig.model, + parameters: { background: true, conversation: 'conv_123', prompt: { id: 'pmpt_123' } }, + }, + }; + await createOpenAIHandler()(config as any, 'q'); + const call = mockResponsesCreate.mock.calls[0][0]; + expect(call).not.toHaveProperty('background'); + expect(call).not.toHaveProperty('conversation'); + expect(call).not.toHaveProperty('prompt'); + }); + + it('drops stream and stream_options, which the handler decides itself by calling create() vs stream()', async () => { + mockResponsesCreate.mockResolvedValue(mockFinalResponse()); + const config = { + ...baseConfig, + model: { ...baseConfig.model, parameters: { stream: true, stream_options: { include_obfuscation: false } } }, + }; + await createOpenAIHandler()(config as any, 'q'); + const call = mockResponsesCreate.mock.calls[0][0]; + expect(call).not.toHaveProperty('stream'); + expect(call).not.toHaveProperty('stream_options'); + }); + + it('renames max_tokens (Chat Completions spelling) to max_output_tokens (Responses spelling)', async () => { + // The LaunchDarkly UI offers the Chat Completions parameter set; the Responses API this + // handler calls only accepts max_output_tokens. Without the rename this would drop silently + // (max_tokens is not a Responses key) or, worse, 400 if the UI ever changes to forward it raw. + mockResponsesCreate.mockResolvedValue(mockFinalResponse()); + const config = { ...baseConfig, model: { ...baseConfig.model, parameters: { max_tokens: 999 } } }; + await createOpenAIHandler()(config as any, 'q'); + const call = mockResponsesCreate.mock.calls[0][0]; + expect(call.max_output_tokens).toBe(999); + expect(call).not.toHaveProperty('max_tokens'); + }); + + it('renames max_completion_tokens to max_output_tokens', async () => { + mockResponsesCreate.mockResolvedValue(mockFinalResponse()); + const config = { ...baseConfig, model: { ...baseConfig.model, parameters: { max_completion_tokens: 777 } } }; + await createOpenAIHandler()(config as any, 'q'); + const call = mockResponsesCreate.mock.calls[0][0]; + expect(call.max_output_tokens).toBe(777); + expect(call).not.toHaveProperty('max_completion_tokens'); + }); + + it('precedence: an explicit max_output_tokens wins over max_completion_tokens and max_tokens', async () => { + mockResponsesCreate.mockResolvedValue(mockFinalResponse()); + const config = { + ...baseConfig, + model: { + ...baseConfig.model, + parameters: { max_tokens: 1, max_completion_tokens: 2, max_output_tokens: 3 }, + }, + }; + await createOpenAIHandler()(config as any, 'q'); + expect(mockResponsesCreate.mock.calls[0][0].max_output_tokens).toBe(3); + }); + + it('precedence: max_completion_tokens wins over max_tokens when max_output_tokens is absent', async () => { + mockResponsesCreate.mockResolvedValue(mockFinalResponse()); + const config = { + ...baseConfig, + model: { ...baseConfig.model, parameters: { max_tokens: 1, max_completion_tokens: 2 } }, + }; + await createOpenAIHandler()(config as any, 'q'); + expect(mockResponsesCreate.mock.calls[0][0].max_output_tokens).toBe(2); + }); + + // This handler wraps the raw Responses API client, which reads snake_case keys itself, so + // (unlike the four framework handlers) the UI's snake_case must reach responses.create untouched + // rather than being camelized. + it('forwards snake_case model.parameters keys unchanged, not camelized', async () => { + mockResponsesCreate.mockResolvedValue(mockFinalResponse()); + const config = { + ...baseConfig, + model: { ...baseConfig.model, parameters: { top_p: 0.7, max_output_tokens: 128 } }, + }; + await createOpenAIHandler()(config as any, 'q'); + const call = mockResponsesCreate.mock.calls[0][0]; + expect(call.top_p).toBe(0.7); + expect(call.max_output_tokens).toBe(128); + expect(call.topP).toBeUndefined(); + expect(call.maxOutputTokens).toBeUndefined(); + }); + + it('does not let model.parameters override model, input, tools, or previous_response_id', async () => { + mockResponsesCreate.mockResolvedValue(mockFinalResponse()); + const config = { + ...baseConfig, + model: { + ...baseConfig.model, + parameters: { model: 'evil-model', input: 'evil-input', tools: ['evil'], previous_response_id: 'evil-id' }, + }, + }; + await createOpenAIHandler()(config as any, 'q'); + const call = mockResponsesCreate.mock.calls[0][0]; + expect(call.model).toBe('gpt-4o'); + expect(call.previous_response_id).toBeUndefined(); + expect(call.tools ?? []).toHaveLength(0); + }); + + it('behaves identically to today when model.parameters is absent', async () => { + mockResponsesCreate.mockResolvedValue(mockFinalResponse()); + await createOpenAIHandler()(baseConfig as any, 'q'); + const call = mockResponsesCreate.mock.calls[0][0]; + expect(call).toEqual({ model: 'gpt-4o', input: expect.any(Array) }); + }); + // ── 1.4 Tool execution loop ───────────────────────────────────────────────── it('invokes the tool handler and loops until end', async () => { @@ -700,6 +888,30 @@ describe('createOpenAIHandler', () => { expect(typeof handler.stream).toBe('function'); }); + it('forwards allowlisted model.parameters fields to the streaming provider call', async () => { + mockResponsesStream.mockReturnValue( + makeResponseStreamMock([{ type: 'response.output_text.delta', delta: 'hi' }], baseFinalResponse), + ); + const config = { + ...baseConfig, + model: { ...baseConfig.model, parameters: { temperature: 0.4, top_p: 0.9, max_output_tokens: 256 } }, + }; + await collectStream(createOpenAIHandler().stream?.(config as any, 'q', {}, {})); + const call = mockResponsesStream.mock.calls[0][0]; + expect(call.temperature).toBe(0.4); + expect(call.top_p).toBe(0.9); + expect(call.max_output_tokens).toBe(256); + }); + + it('the streaming call is unchanged from today when model.parameters is absent', async () => { + mockResponsesStream.mockReturnValue( + makeResponseStreamMock([{ type: 'response.output_text.delta', delta: 'hi' }], baseFinalResponse), + ); + await collectStream(createOpenAIHandler().stream?.(baseConfig as any, 'q', {}, {})); + const call = mockResponsesStream.mock.calls[0][0]; + expect(call).toEqual({ model: 'gpt-4o', input: expect.any(Array) }); + }); + it('handler.stream returns an async iterable', () => { mockResponsesStream.mockReturnValue( makeResponseStreamMock([{ type: 'response.output_text.delta', delta: 'hi' }], baseFinalResponse), diff --git a/packages/openai-messages/src/__tests__/never-forwarded.ts b/packages/openai-messages/src/__tests__/never-forwarded.ts new file mode 100644 index 00000000..300d6e51 --- /dev/null +++ b/packages/openai-messages/src/__tests__/never-forwarded.ts @@ -0,0 +1,163 @@ +import { expect } from 'vitest'; + +/** + * The value every never-forwarded key carries in `NEVER_FORWARDED_PARAMETERS`. A handler that + * passes any of those keys on, under any name or nested anywhere, leaves this string somewhere in + * the options it builds, which is what `expectNoNeverForwardedValue` looks for. + */ +export const NEVER_FORWARDED = 'NEVER_FORWARDED'; + +/** + * Keys that must never reach a provider, framework or host-process option from + * `model.parameters`, in both the snake_case the LaunchDarkly UI writes and the camelCase the + * frameworks read: credentials, endpoints and connection settings, request injection, remote + * tools, and Claude Agents host-process settings. + */ +export const NEVER_FORWARDED_KEYS = [ + // credentials + 'api_key', + 'apiKey', + 'openai_api_key', + 'openAIApiKey', + 'anthropic_api_key', + 'anthropicApiKey', + 'bedrock_api_key', + 'bedrockApiKey', + 'bedrock_api_secret', + 'bedrockApiSecret', + 'credentials', + 'auth_token', + 'authToken', + 'access_token', + 'accessToken', + 'aws_access_key_id', + 'awsAccessKeyId', + 'aws_secret_access_key', + 'awsSecretAccessKey', + 'aws_session_token', + 'awsSessionToken', + 'organization', + // endpoints and connection + 'base_url', + 'baseURL', + 'baseUrl', + 'anthropic_api_url', + 'anthropicApiUrl', + 'endpoint', + 'endpoint_host', + 'endpointHost', + 'endpoint_url', + 'endpointUrl', + 'region', + 'region_name', + 'regionName', + 'configuration', + 'client_options', + 'clientOptions', + 'default_headers', + 'defaultHeaders', + 'timeout', + 'max_retries', + 'maxRetries', + 'fetch', + 'http_client', + 'httpClient', + 'http_agent', + 'httpAgent', + 'proxy', + 'proxies', + 'client', + // request injection + 'headers', + 'extra_headers', + 'extraHeaders', + 'extra_body', + 'extraBody', + 'extra_query', + 'extraQuery', + 'model_kwargs', + 'modelKwargs', + 'invocation_kwargs', + 'invocationKwargs', + 'additional_model_request_fields', + 'additionalModelRequestFields', + 'provider_data', + 'providerData', + 'provider_options', + 'providerOptions', + // remote tools + 'mcp_servers', + 'mcpServers', + // Claude Agents host-process settings + 'path_to_claude_code_executable', + 'pathToClaudeCodeExecutable', + 'cli_path', + 'cliPath', + 'executable', + 'executable_args', + 'executableArgs', + 'extra_args', + 'extraArgs', + 'env', + 'cwd', + 'additional_directories', + 'additionalDirectories', + 'add_dirs', + 'addDirs', + 'permission_mode', + 'permissionMode', + 'allow_dangerously_skip_permissions', + 'allowDangerouslySkipPermissions', + 'settings', + 'setting_sources', + 'settingSources', + 'managed_settings', + 'managedSettings', + 'plugins', + 'sandbox', + 'resume', + 'resume_session_at', + 'resumeSessionAt', + 'session_id', + 'sessionId', + 'fork_session', + 'forkSession', + 'continue', + 'hooks', + 'can_use_tool', + 'canUseTool', + 'stderr', + 'abort_controller', + 'abortController', + 'spawn_claude_code_process', + 'spawnClaudeCodeProcess', +] as const; + +/** A `model.parameters` bag that sets every key in `NEVER_FORWARDED_KEYS` to `NEVER_FORWARDED`. */ +export const NEVER_FORWARDED_PARAMETERS: Record = Object.fromEntries( + NEVER_FORWARDED_KEYS.map((key) => [key, NEVER_FORWARDED]), +); + +function findNeverForwarded(value: unknown, path: string, seen: Set): string | undefined { + if (value === NEVER_FORWARDED) return path || '(root)'; + if (!value || typeof value !== 'object' || seen.has(value)) return undefined; + seen.add(value); + for (const [key, child] of Object.entries(value)) { + const found = findNeverForwarded(child, path ? `${path}.${key}` : key, seen); + if (found) return found; + } + return undefined; +} + +/** + * Asserts that no `NEVER_FORWARDED` value appears anywhere in `options`, and that none of the + * `NEVER_FORWARDED_KEYS` the handler does not own itself is set on it. `handlerOwned` lists the + * keys the handler legitimately sets from its own state (Claude Agents sets `mcpServers` and + * `hooks`, for example); those must still not carry the config's value. + */ +export function expectNoNeverForwardedValue(options: unknown, handlerOwned: ReadonlyArray = []): void { + expect(findNeverForwarded(options, '', new Set())).toBeUndefined(); + const record = options as Record; + const leaked = NEVER_FORWARDED_KEYS.filter((key) => !handlerOwned.includes(key) && record[key] !== undefined); + expect(leaked).toEqual([]); +} diff --git a/packages/openai-messages/src/__tests__/wire.test.ts b/packages/openai-messages/src/__tests__/wire.test.ts new file mode 100644 index 00000000..b4e48397 --- /dev/null +++ b/packages/openai-messages/src/__tests__/wire.test.ts @@ -0,0 +1,101 @@ +// ─── Wire-level tests ─────────────────────────────────────────────────────── +// +// Unlike handler.test.ts, this file does NOT mock the `openai` package. It lets the real +// OpenAI SDK build a request and intercepts the outgoing HTTP call via a stubbed global +// `fetch`, so these tests prove the JSON body actually sent to the Responses API — not just +// what reaches a mocked `responses.create`. + +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +function mockFetchResponse(body: unknown) { + return new Response(JSON.stringify(body), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); +} + +function finalResponseBody() { + return { + id: 'resp_1', + object: 'response', + status: 'completed', + model: 'gpt-4o', + output: [{ type: 'message', role: 'assistant', content: [{ type: 'output_text', text: 'hi' }] }], + output_text: 'hi', + usage: { input_tokens: 1, output_tokens: 1 }, + }; +} + +describe('createOpenAIHandler — wire-level request body', () => { + let capturedBody: Record | undefined; + let fetchMock: ReturnType; + + beforeEach(async () => { + vi.resetModules(); + process.env.OPENAI_API_KEY = 'test-key'; + capturedBody = undefined; + fetchMock = vi.fn(async (_url: unknown, init: RequestInit) => { + capturedBody = JSON.parse(init.body as string); + return mockFetchResponse(finalResponseBody()); + }); + vi.stubGlobal('fetch', fetchMock); + }); + + afterEach(() => { + vi.unstubAllGlobals(); + }); + + it('sends max_output_tokens and drops frequency_penalty for a config that sets both Chat Completions keys', async () => { + const { createOpenAIHandler } = await import('../handler.js'); + const config = { + model: { + name: 'gpt-4o', + parameters: { + max_tokens: 256, + // Not a Responses API key; the LaunchDarkly UI offers it because it lists the Chat + // Completions parameter set. Must be dropped, not sent — the real API 400s on it. + frequency_penalty: 0.5, + }, + }, + provider: { name: 'OpenAI' }, + instructions: 'You are helpful.', + }; + + const handler = createOpenAIHandler(); + await handler(config as any, 'hi'); + + expect(fetchMock).toHaveBeenCalled(); + expect(capturedBody?.max_output_tokens).toBe(256); + expect(capturedBody).not.toHaveProperty('max_tokens'); + expect(capturedBody).not.toHaveProperty('frequency_penalty'); + }); + + it('forwards an accepted key (temperature) to the wire request', async () => { + const { createOpenAIHandler } = await import('../handler.js'); + const config = { + model: { name: 'gpt-4o', parameters: { temperature: 0.25 } }, + provider: { name: 'OpenAI' }, + instructions: 'You are helpful.', + }; + + const handler = createOpenAIHandler(); + await handler(config as any, 'hi'); + + expect(capturedBody?.temperature).toBe(0.25); + }); + + it('forwards store (no generation effect, but does not break the handler) and drops background (would break it)', async () => { + const { createOpenAIHandler } = await import('../handler.js'); + const config = { + model: { name: 'gpt-4o', parameters: { store: true, background: true } }, + provider: { name: 'OpenAI' }, + instructions: 'You are helpful.', + }; + + const handler = createOpenAIHandler(); + await handler(config as any, 'hi'); + + expect(capturedBody?.store).toBe(true); + expect(capturedBody).not.toHaveProperty('background'); + }); +}); diff --git a/packages/openai-messages/src/handler.ts b/packages/openai-messages/src/handler.ts index 43a0e21f..2dc08c70 100644 --- a/packages/openai-messages/src/handler.ts +++ b/packages/openai-messages/src/handler.ts @@ -12,8 +12,10 @@ import { isContentBlocks, type LDContext, type Message, + normalizeModelParameters, type ProviderHandler, parseTemplate, + pickForwardedModelParameters, type SpanMessage, type SpanMessagePart, type SpanUsage, @@ -28,6 +30,7 @@ import { } from '@launchdarkly/ai-server'; import { type Context, context, type Span, SpanStatusCode, trace } from '@opentelemetry/api'; import OpenAI from 'openai'; +import type { ResponseCreateParamsBase } from 'openai/resources/responses/responses'; const TRACER_NAME = '@launchdarkly/ai-openai-messages'; @@ -340,6 +343,86 @@ const jsonSchemaFormat = (schema: Record): any => ({ const toToolDefinitions = (tools: FunctionTool[]): ToolDefinitionInput[] => tools.map((tool) => ({ name: tool.name, description: tool.description, parameters: tool.parameters })); +/** + * `ResponseCreateParamsBase` keys this handler forwards verbatim from `config.model.parameters`, + * after the `max_tokens` / `max_completion_tokens` → `max_output_tokens` rename below — the + * LaunchDarkly UI offers the Chat Completions parameter set, and the Responses API this handler + * calls does not accept either of those two names. + * + * The rule for this list: exclude a key only if setting it would BREAK the handler (wrong or + * missing result, or a request the handler cannot build); forward everything else the API + * accepts, even settings with no obvious generation effect (`store`, `user`, + * `safety_identifier`, `prompt_cache_key`, `prompt_cache_retention`, `include`, + * `context_management`, ...) — those are forwarded, not excluded, because a config that sets one + * still gets a working call. + * + * Handler-owned (this handler sets these itself, from the config and the call shape, so a + * `model.parameters` value must not be able to override what it already decided): `model`, + * `input`, `tools`, `previous_response_id`, `text`. + * + * Excluded (would break the handler): + * - `stream`, `stream_options` — the handler chooses streaming itself, by calling + * `responses.create()` vs `responses.stream()`; a config value here fights that choice rather + * than configuring anything. + * - `background` — the call returns before the output exists, so the handler would get no result + * to return. + * - `conversation`, `prompt` — supply server-side conversation state / a stored prompt template + * that conflicts with the `input` this handler already builds from `config.messages` / + * `config.instructions` and threads itself via `previous_response_id`. + */ +const FORWARDED_MODEL_PARAMETER_KEYS = [ + 'context_management', + 'include', + 'instructions', + 'max_output_tokens', + 'metadata', + 'moderation', + 'parallel_tool_calls', + 'prompt_cache_key', + 'prompt_cache_retention', + 'reasoning', + 'safety_identifier', + 'service_tier', + 'store', + 'temperature', + 'tool_choice', + 'top_logprobs', + 'top_p', + 'truncation', + 'user', +] as const; + +type OpenAIHandlerOwnedKeys = 'input' | 'model' | 'previous_response_id' | 'text' | 'tools'; +type OpenAIExcludedKeys = 'background' | 'conversation' | 'prompt' | 'stream' | 'stream_options'; +// If a key of ResponseCreateParamsBase is added to the SDK and not classified above as forwarded, +// handler-owned, or excluded, this type resolves to something other than `never` and the +// assignment below fails to compile, naming the unclassified key. +type OpenAIUndecidedModelParameterKeys = Exclude< + keyof ResponseCreateParamsBase, + OpenAIHandlerOwnedKeys | OpenAIExcludedKeys | (typeof FORWARDED_MODEL_PARAMETER_KEYS)[number] +>; +const _openaiModelParameterKeysExhaustive: Record = {} as Record< + never, + never +>; + +/** + * Picks the subset of `config.model.parameters` that maps onto `ResponseCreateParamsBase`, after + * renaming the two Chat Completions token-limit spellings the LaunchDarkly UI offers — + * `max_tokens` and `max_completion_tokens` — to the Responses API's own `max_output_tokens`. + * Precedence when a config sets more than one spelling: an explicit `max_output_tokens` wins, then + * `max_completion_tokens`, then `max_tokens`. A config that sets nothing here produces `{}`, so the + * provider call sees exactly what it always has. + */ +function buildModelParameterOptions(parameters: AiConfigRep['model']['parameters']): Record { + const normalized = normalizeModelParameters(parameters); + const { max_tokens, max_completion_tokens, max_output_tokens, ...rest } = normalized; + const resolvedMaxOutputTokens = max_output_tokens ?? max_completion_tokens ?? max_tokens; + const withRenamedMaxTokens = + resolvedMaxOutputTokens !== undefined ? { ...rest, max_output_tokens: resolvedMaxOutputTokens } : rest; + return pickForwardedModelParameters(withRenamedMaxTokens, FORWARDED_MODEL_PARAMETER_KEYS); +} + export function createOpenAIHandler({ captureContent = false }: ContentCaptureOptions = {}): ProviderHandler { const openai = new OpenAI(); @@ -414,10 +497,12 @@ export function createOpenAIHandler({ captureContent = false }: ContentCaptureOp let response = await runModelTurn( { + ...buildModelParameterOptions(config.model.parameters), model: config.model.name, input: inputMessages, - ...(tools.length > 0 ? { tools } : {}), - ...(config.outputFormat ? { text: { format: jsonSchemaFormat(config.outputFormat) } } : {}), + tools: tools.length > 0 ? tools : undefined, + previous_response_id: undefined, + text: config.outputFormat ? { format: jsonSchemaFormat(config.outputFormat) } : undefined, }, toolDefinitions, ); @@ -456,9 +541,12 @@ export function createOpenAIHandler({ captureContent = false }: ContentCaptureOp response = await runModelTurn( { + ...buildModelParameterOptions(config.model.parameters), model: config.model.name, previous_response_id: response.id, input: toolOutputs, + tools: undefined, + text: undefined, }, toolDefinitions, ); @@ -539,8 +627,20 @@ export function createOpenAIHandler({ captureContent = false }: ContentCaptureOp let finalResp: OpenAI.Responses.Response; try { const streamParams = previousResponseId - ? { model: config.model.name, previous_response_id: previousResponseId, input: currentInput } - : { model: config.model.name, input: currentInput, ...(tools.length > 0 ? { tools } : {}) }; + ? { + ...buildModelParameterOptions(config.model.parameters), + model: config.model.name, + previous_response_id: previousResponseId, + input: currentInput, + tools: undefined, + } + : { + ...buildModelParameterOptions(config.model.parameters), + model: config.model.name, + input: currentInput, + tools: tools.length > 0 ? tools : undefined, + previous_response_id: undefined, + }; // biome-ignore lint/suspicious/noExplicitAny: streamParams union type does not match the SDK's overloaded stream() signature const responseStream = openai.responses.stream(streamParams as any); diff --git a/packages/vercel-agents/src/__tests__/handler.test.ts b/packages/vercel-agents/src/__tests__/handler.test.ts index 70cc125b..b654d4e0 100644 --- a/packages/vercel-agents/src/__tests__/handler.test.ts +++ b/packages/vercel-agents/src/__tests__/handler.test.ts @@ -179,6 +179,82 @@ describe('createVercelAgentsHandler', () => { }); }); + it('maps snake_case model.parameters onto the ToolLoopAgent call settings', async () => { + const parameters = { + max_tokens: 256, + top_p: 0.9, + top_k: 40, + presence_penalty: 0.1, + frequency_penalty: 0.2, + stop_sequences: ['END'], + seed: 7, + reasoning_effort: 'low', + tool_choice: 'auto', + }; + await createVercelAgentsHandler()({ ...baseConfig, model: { ...baseConfig.model, parameters } } as any, 'hello'); + const settings = aiMocks.agentArguments[0]; + expect(settings).toMatchObject({ + maxOutputTokens: 256, + topP: 0.9, + topK: 40, + presencePenalty: 0.1, + frequencyPenalty: 0.2, + stopSequences: ['END'], + seed: 7, + reasoning: 'low', + toolChoice: 'auto', + }); + for (const key of Object.keys(parameters).filter((k) => k.includes('_'))) { + expect(settings).not.toHaveProperty(key); + } + }); + + it('applies the same mapping on the streaming path', async () => { + aiMocks.stream.mockReturnValue({ + textStream: (async function* () { + yield 'ok'; + })(), + usage: Promise.resolve({ inputTokens: 1, outputTokens: 1, totalTokens: 2 }), + }); + const parameters = { max_tokens: 64, maxOutputTokens: 9, top_p: 0.5, abort_signal: 'bad' }; + await collect( + createVercelAgentsHandler().stream?.( + { ...baseConfig, model: { ...baseConfig.model, parameters } } as any, + 'q', + {}, + {}, + ) as AsyncIterable, + ); + const settings = aiMocks.agentArguments[0]; + // Both spellings of one key set: snake_case wins, as in every other handler. + expect(settings).toMatchObject({ maxOutputTokens: 64, topP: 0.5 }); + expect(settings).not.toHaveProperty('max_tokens'); + expect(settings).not.toHaveProperty('abortSignal'); + }); + + it('does not forward keys that are not AI SDK call settings', async () => { + const parameters = { + temperature: 0.3, + allowSystemInMessages: true, + experimental_telemetry: { isEnabled: true }, + unknown_setting: 1, + abortSignal: 'bad', + }; + await createVercelAgentsHandler()({ ...baseConfig, model: { ...baseConfig.model, parameters } } as any, 'q'); + const settings = aiMocks.agentArguments[0]; + expect(settings.temperature).toBe(0.3); + for (const key of [ + 'allowSystemInMessages', + 'experimental_telemetry', + 'experimentalTelemetry', + 'unknown_setting', + 'unknownSetting', + 'abortSignal', + ]) { + expect(settings).not.toHaveProperty(key); + } + }); + it('passes structured history and user input to agent.generate messages', async () => { const history = [ { role: 'system' as const, content: 'filtered system' }, diff --git a/packages/vercel-agents/src/__tests__/model-parameters.test.ts b/packages/vercel-agents/src/__tests__/model-parameters.test.ts new file mode 100644 index 00000000..11fc9e4f --- /dev/null +++ b/packages/vercel-agents/src/__tests__/model-parameters.test.ts @@ -0,0 +1,119 @@ +import { createOpenAI } from '@ai-sdk/openai'; +import { createGateway, generateText } from 'ai'; +import { MockLanguageModelV4 } from 'ai/test'; +import { describe, expect, it } from 'vitest'; +import { createVercelAgentsHandler } from '../handler.js'; +import { buildModelParameterOptions } from '../model-parameters.js'; +import { expectNoNeverForwardedValue, NEVER_FORWARDED_PARAMETERS } from './never-forwarded.js'; + +// This file deliberately does NOT mock `ai`: it checks what reaches the provider's HTTP request. + +type SeenHeaders = Record[]; + +function capturingFetch(seen: SeenHeaders): typeof globalThis.fetch { + return (async (_url: unknown, init?: RequestInit) => { + seen.push(Object.fromEntries(new Headers(init?.headers).entries())); + return new Response(JSON.stringify({ error: { message: 'captured' } }), { status: 401 }); + }) as typeof globalThis.fetch; +} + +const attackerParameters = { + headers: { Authorization: 'Bearer ATTACKER' }, + extra_headers: { Authorization: 'Bearer ATTACKER' }, + provider_options: { gateway: { byok: { openai: [{ apiKey: 'ATTACKER' }] }, order: ['attacker'] } }, +}; + +function config(parameters: Record) { + return { + model: { name: 'openai/gpt-4o', parameters }, + provider: { name: 'OpenAI' }, + instructions: 'You are helpful.', + }; +} + +describe('model.parameters cannot change how the request authenticates', () => { + it('keeps the customer API key on @ai-sdk/openai when a config sets an Authorization header', async () => { + const seen: SeenHeaders = []; + const provider = createOpenAI({ apiKey: 'CUSTOMER', fetch: capturingFetch(seen) }); + await expect( + createVercelAgentsHandler({ model: provider('gpt-4o') })(config(attackerParameters) as any, 'hi'), + ).rejects.toThrow(); + expect(seen.length).toBeGreaterThan(0); + for (const headers of seen) { + expect(headers.authorization).toBe('Bearer CUSTOMER'); + } + }); + + it('keeps the customer API key on the Vercel AI Gateway when a config sets an Authorization header', async () => { + const seen: SeenHeaders = []; + const gateway = createGateway({ apiKey: 'CUSTOMER', fetch: capturingFetch(seen) }); + await expect( + createVercelAgentsHandler({ model: gateway('openai/gpt-4o') })(config(attackerParameters) as any, 'hi'), + ).rejects.toThrow(); + expect(seen.length).toBeGreaterThan(0); + for (const headers of seen) { + expect(headers.authorization).toBe('Bearer CUSTOMER'); + } + }); + + // Jeff's repro from the review, inverted: the options built from a config that sets + // `headers.Authorization` no longer replace the provider's own key. + it('buildModelParameterOptions output spread into generateText does not override the provider key', async () => { + const seen: SeenHeaders = []; + const provider = createOpenAI({ apiKey: 'CUSTOMER', fetch: capturingFetch(seen) }); + const opts = buildModelParameterOptions({ headers: { Authorization: 'Bearer ATTACKER' } } as never); + await generateText({ model: provider('gpt-4o'), prompt: 'hi', maxRetries: 0, ...opts }).catch(() => {}); + expect(seen[0]?.authorization).toBe('Bearer CUSTOMER'); + }); +}); + +describe('buildModelParameterOptions allowlist', () => { + it('drops every never-forwarded key', () => { + const options = buildModelParameterOptions({ ...NEVER_FORWARDED_PARAMETERS, temperature: 0.5 } as never); + expect(options).toEqual({ temperature: 0.5 }); + expectNoNeverForwardedValue(options); + }); + + it('never-forwarded keys do not reach the model call', async () => { + const model = new MockLanguageModelV4({ + doGenerate: async () => { + throw new Error('aborted: request captured'); + }, + }); + await expect( + createVercelAgentsHandler({ model })(config(NEVER_FORWARDED_PARAMETERS) as any, 'hi'), + ).rejects.toThrow(); + expect(model.doGenerateCalls).toHaveLength(1); + // The AI SDK sets its own `headers` (user agent) on every call; the deep check still proves + // none of the config's values reached it. + expectNoNeverForwardedValue(model.doGenerateCalls[0], ['headers']); + }); + + it('forwards the allowed keys under the AI SDK names', () => { + expect( + buildModelParameterOptions({ + temperature: 0.2, + top_p: 0.9, + top_k: 40, + presence_penalty: 0.1, + frequency_penalty: 0.3, + stop_sequences: ['END'], + seed: 7, + max_tokens: 100, + reasoning_effort: 'low', + tool_choice: 'auto', + } as never), + ).toEqual({ + temperature: 0.2, + topP: 0.9, + topK: 40, + presencePenalty: 0.1, + frequencyPenalty: 0.3, + stopSequences: ['END'], + seed: 7, + maxOutputTokens: 100, + reasoning: 'low', + toolChoice: 'auto', + }); + }); +}); diff --git a/packages/vercel-agents/src/__tests__/native-graph.test.ts b/packages/vercel-agents/src/__tests__/native-graph.test.ts index 02962285..10444426 100644 --- a/packages/vercel-agents/src/__tests__/native-graph.test.ts +++ b/packages/vercel-agents/src/__tests__/native-graph.test.ts @@ -69,6 +69,7 @@ vi.mock('@opentelemetry/api', async (importOriginal) => { }); import { toVercelAgents } from '../native-graph.js'; +import { expectNoNeverForwardedValue, NEVER_FORWARDED_PARAMETERS } from './never-forwarded.js'; function makeNode( key: string, @@ -161,6 +162,45 @@ describe('toVercelAgents', () => { ); }); + it("maps each node's snake_case model.parameters onto that node's ToolLoopAgent call settings", async () => { + const graph = makeGraph(); + (graph.root.config.model as any).parameters = { + max_tokens: 128, + top_p: 0.7, + provider_options: { openai: { reasoning_summary: 'auto' } }, + instructions: 'bad', + tools: {}, + abort_signal: 'bad', + }; + (graph.getNode('leaf')!.config.model as any).parameters = { temperature: 0.1, max_completion_tokens: 32 }; + await toVercelAgents(Promise.resolve(graph as any)).invoke('hi'); + const rootSettings = aiMocks.agentArguments.find((options) => options.model === 'openai/root-model'); + const leafSettings = aiMocks.agentArguments.find((options) => options.model === 'anthropic/leaf-model'); + expect(rootSettings).toMatchObject({ + maxOutputTokens: 128, + topP: 0.7, + instructions: 'Root instructions', + }); + expect(Object.keys(rootSettings.tools)).toEqual(['transfer_to_leaf']); + for (const key of ['max_tokens', 'top_p', 'provider_options', 'providerOptions', 'abort_signal', 'abortSignal']) { + expect(rootSettings).not.toHaveProperty(key); + } + expect(leafSettings).toMatchObject({ temperature: 0.1, maxOutputTokens: 32 }); + expect(leafSettings).not.toHaveProperty('max_completion_tokens'); + }); + + it('never forwards credentials, endpoints, headers or host settings from any node', async () => { + const graph = makeGraph(); + (graph.root.config.model as any).parameters = { ...NEVER_FORWARDED_PARAMETERS, temperature: 0.3 }; + (graph.getNode('leaf')!.config.model as any).parameters = NEVER_FORWARDED_PARAMETERS; + await toVercelAgents(Promise.resolve(graph as any)).invoke('hi'); + const rootSettings = aiMocks.agentArguments.find((options) => options.model === 'openai/root-model'); + expect(rootSettings.temperature).toBe(0.3); + for (const settings of aiMocks.agentArguments) { + expectNoNeverForwardedValue(settings); + } + }); + it('resolves the injected model factory independently for every evaluated node', async () => { const modelFactory = vi.fn((config: any) => ({ modelId: `resolved:${config.model.name}` })); await toVercelAgents(Promise.resolve(makeGraph() as any), { modelFactory } as any).invoke('hi'); diff --git a/packages/vercel-agents/src/__tests__/never-forwarded.ts b/packages/vercel-agents/src/__tests__/never-forwarded.ts new file mode 100644 index 00000000..300d6e51 --- /dev/null +++ b/packages/vercel-agents/src/__tests__/never-forwarded.ts @@ -0,0 +1,163 @@ +import { expect } from 'vitest'; + +/** + * The value every never-forwarded key carries in `NEVER_FORWARDED_PARAMETERS`. A handler that + * passes any of those keys on, under any name or nested anywhere, leaves this string somewhere in + * the options it builds, which is what `expectNoNeverForwardedValue` looks for. + */ +export const NEVER_FORWARDED = 'NEVER_FORWARDED'; + +/** + * Keys that must never reach a provider, framework or host-process option from + * `model.parameters`, in both the snake_case the LaunchDarkly UI writes and the camelCase the + * frameworks read: credentials, endpoints and connection settings, request injection, remote + * tools, and Claude Agents host-process settings. + */ +export const NEVER_FORWARDED_KEYS = [ + // credentials + 'api_key', + 'apiKey', + 'openai_api_key', + 'openAIApiKey', + 'anthropic_api_key', + 'anthropicApiKey', + 'bedrock_api_key', + 'bedrockApiKey', + 'bedrock_api_secret', + 'bedrockApiSecret', + 'credentials', + 'auth_token', + 'authToken', + 'access_token', + 'accessToken', + 'aws_access_key_id', + 'awsAccessKeyId', + 'aws_secret_access_key', + 'awsSecretAccessKey', + 'aws_session_token', + 'awsSessionToken', + 'organization', + // endpoints and connection + 'base_url', + 'baseURL', + 'baseUrl', + 'anthropic_api_url', + 'anthropicApiUrl', + 'endpoint', + 'endpoint_host', + 'endpointHost', + 'endpoint_url', + 'endpointUrl', + 'region', + 'region_name', + 'regionName', + 'configuration', + 'client_options', + 'clientOptions', + 'default_headers', + 'defaultHeaders', + 'timeout', + 'max_retries', + 'maxRetries', + 'fetch', + 'http_client', + 'httpClient', + 'http_agent', + 'httpAgent', + 'proxy', + 'proxies', + 'client', + // request injection + 'headers', + 'extra_headers', + 'extraHeaders', + 'extra_body', + 'extraBody', + 'extra_query', + 'extraQuery', + 'model_kwargs', + 'modelKwargs', + 'invocation_kwargs', + 'invocationKwargs', + 'additional_model_request_fields', + 'additionalModelRequestFields', + 'provider_data', + 'providerData', + 'provider_options', + 'providerOptions', + // remote tools + 'mcp_servers', + 'mcpServers', + // Claude Agents host-process settings + 'path_to_claude_code_executable', + 'pathToClaudeCodeExecutable', + 'cli_path', + 'cliPath', + 'executable', + 'executable_args', + 'executableArgs', + 'extra_args', + 'extraArgs', + 'env', + 'cwd', + 'additional_directories', + 'additionalDirectories', + 'add_dirs', + 'addDirs', + 'permission_mode', + 'permissionMode', + 'allow_dangerously_skip_permissions', + 'allowDangerouslySkipPermissions', + 'settings', + 'setting_sources', + 'settingSources', + 'managed_settings', + 'managedSettings', + 'plugins', + 'sandbox', + 'resume', + 'resume_session_at', + 'resumeSessionAt', + 'session_id', + 'sessionId', + 'fork_session', + 'forkSession', + 'continue', + 'hooks', + 'can_use_tool', + 'canUseTool', + 'stderr', + 'abort_controller', + 'abortController', + 'spawn_claude_code_process', + 'spawnClaudeCodeProcess', +] as const; + +/** A `model.parameters` bag that sets every key in `NEVER_FORWARDED_KEYS` to `NEVER_FORWARDED`. */ +export const NEVER_FORWARDED_PARAMETERS: Record = Object.fromEntries( + NEVER_FORWARDED_KEYS.map((key) => [key, NEVER_FORWARDED]), +); + +function findNeverForwarded(value: unknown, path: string, seen: Set): string | undefined { + if (value === NEVER_FORWARDED) return path || '(root)'; + if (!value || typeof value !== 'object' || seen.has(value)) return undefined; + seen.add(value); + for (const [key, child] of Object.entries(value)) { + const found = findNeverForwarded(child, path ? `${path}.${key}` : key, seen); + if (found) return found; + } + return undefined; +} + +/** + * Asserts that no `NEVER_FORWARDED` value appears anywhere in `options`, and that none of the + * `NEVER_FORWARDED_KEYS` the handler does not own itself is set on it. `handlerOwned` lists the + * keys the handler legitimately sets from its own state (Claude Agents sets `mcpServers` and + * `hooks`, for example); those must still not carry the config's value. + */ +export function expectNoNeverForwardedValue(options: unknown, handlerOwned: ReadonlyArray = []): void { + expect(findNeverForwarded(options, '', new Set())).toBeUndefined(); + const record = options as Record; + const leaked = NEVER_FORWARDED_KEYS.filter((key) => !handlerOwned.includes(key) && record[key] !== undefined); + expect(leaked).toEqual([]); +} diff --git a/packages/vercel-agents/src/__tests__/wire.test.ts b/packages/vercel-agents/src/__tests__/wire.test.ts new file mode 100644 index 00000000..917ee57d --- /dev/null +++ b/packages/vercel-agents/src/__tests__/wire.test.ts @@ -0,0 +1,31 @@ +import { MockLanguageModelV4 } from 'ai/test'; +import { describe, expect, it } from 'vitest'; +import { createVercelAgentsHandler } from '../handler.js'; + +// This file deliberately does NOT mock `ai`. The handler tests mock ToolLoopAgent, so they only +// prove what the handler hands to the AI SDK, not that the AI SDK reads it: it silently drops any +// call setting it does not recognize, which is how snake_case `max_tokens` / `top_p` went nowhere +// before. Running a real ToolLoopAgent against a mock language model checks the settings that +// actually reach the model call. + +const baseConfig = { + model: { + name: 'openai/gpt-5', + parameters: { max_tokens: 321, top_p: 0.42, stop_sequences: ['END'] }, + }, + provider: { name: 'OpenAI' }, + instructions: 'You are an agent.', +}; + +describe('createVercelAgentsHandler, model call (real ToolLoopAgent, mock model)', () => { + it('sends model.parameters to the model as maxOutputTokens / topP / stopSequences', async () => { + const model = new MockLanguageModelV4({ + doGenerate: async () => { + throw new Error('aborted: request captured'); + }, + }); + await expect(createVercelAgentsHandler({ model })(baseConfig as any, 'hi')).rejects.toThrow(); + expect(model.doGenerateCalls).toHaveLength(1); + expect(model.doGenerateCalls[0]).toMatchObject({ maxOutputTokens: 321, topP: 0.42, stopSequences: ['END'] }); + }); +}); diff --git a/packages/vercel-agents/src/handler.ts b/packages/vercel-agents/src/handler.ts index 8b914f67..6fb6fd20 100644 --- a/packages/vercel-agents/src/handler.ts +++ b/packages/vercel-agents/src/handler.ts @@ -35,24 +35,10 @@ import { tool, } from 'ai'; import { gatewayModelId } from './model-id.js'; +import { buildModelParameterOptions } from './model-parameters.js'; const TRACER_NAME = '@launchdarkly/ai-vercel-agents'; const MAX_STEPS = 10; -const OWNED_PARAMETER_NAMES = new Set([ - 'model', - 'messages', - 'prompt', - 'system', - 'instructions', - 'tools', - 'stream', - 'output', - 'outputformat', - 'stopwhen', - 'maxsteps', - 'apikey', - 'baseurl', -]); export interface VercelAgentsOptions extends ContentCaptureOptions { model?: LanguageModel; @@ -112,15 +98,6 @@ function servingProvider(config: AiConfigRep): string { return (config.provider?.name || 'unknown').toLowerCase(); } -function modelSettings(config: AiConfigRep): Record { - return Object.fromEntries( - Object.entries(config.model.parameters ?? {}).filter(([key]) => { - const normalized = key.replaceAll('_', '').replaceAll('-', '').toLowerCase(); - return !OWNED_PARAMETER_NAMES.has(normalized); - }), - ); -} - async function resolveModel(config: AiConfigRep, options: VercelAgentsOptions): Promise { if (options.modelFactory) return options.modelFactory(config); if (options.model) return options.model; @@ -272,7 +249,7 @@ async function buildAgent( ? Output.object({ schema: jsonSchema(normalizeOutputSchema(config.outputFormat)) }) : undefined; const agent = new ToolLoopAgent({ - ...modelSettings(config), + ...buildModelParameterOptions(config.model.parameters), model: await resolveModel(config, options), ...(system ? { instructions: system } : {}), ...(Object.keys(tools).length > 0 ? { tools } : {}), diff --git a/packages/vercel-agents/src/model-parameters.ts b/packages/vercel-agents/src/model-parameters.ts new file mode 100644 index 00000000..3e23ec34 --- /dev/null +++ b/packages/vercel-agents/src/model-parameters.ts @@ -0,0 +1,81 @@ +import { + type AiConfigRep, + camelizeModelParameters, + normalizeModelParameters, + pickForwardedModelParameters, +} from '@launchdarkly/ai-server'; +import type { LanguageModelCallOptions, RequestOptions, ToolLoopAgentSettings } from 'ai'; + +/** + * AI SDK call settings this package's `ToolLoopAgent`s (the single-agent handler and every + * native-graph node) take from `config.model.parameters`, after the renames in + * `buildModelParameterOptions`. + * + * The AI SDK drops any call setting it does not recognize without an error, so a key that is + * not spelled exactly as below (snake_case `max_tokens`, `top_p`) reaches nothing. This list is + * the `LanguageModelCallOptions` generation settings plus `toolChoice`. + * + * Handler-owned (set by the handler from the config and the call shape, never from + * `model.parameters`): `model`, `instructions`, `tools`, `stopWhen`, `output`. + * + * Never forwarded, because they configure the request rather than the model and a config must not + * be able to change them: + * - `headers`: call-level headers win over the provider's own, so a config could replace the + * customer's `Authorization` header. + * - `providerOptions`: the Vercel AI Gateway reads credentials (`gateway.byok`) and routing + * (`gateway.order`, `gateway.only`, `gateway.models`) from it. + * - `maxRetries`, `timeout`: client retry and timeout policy. + * - `abortSignal`: cannot be expressed in a JSON config and belongs to the caller. + * + * Every other `ToolLoopAgent` setting (callbacks, telemetry, tool repair, approval, `activeTools`, + * `include`, ...) is either a function or wiring this handler does not expose, so it is not + * forwarded either. + */ +const FORWARDED_MODEL_PARAMETER_KEYS = [ + 'frequencyPenalty', + 'maxOutputTokens', + 'presencePenalty', + 'reasoning', + 'seed', + 'stopSequences', + 'temperature', + 'toolChoice', + 'topK', + 'topP', +] as const satisfies ReadonlyArray; + +type VercelExcludedKeys = 'abortSignal' | 'headers' | 'maxRetries' | 'providerOptions' | 'timeout'; +// If the AI SDK adds a call setting and it is not classified above as forwarded or excluded, +// this type resolves to something other than `never` and the assignment below fails to compile, +// naming the unclassified key. +type VercelUndecidedModelParameterKeys = Exclude< + keyof LanguageModelCallOptions | keyof RequestOptions, + VercelExcludedKeys | (typeof FORWARDED_MODEL_PARAMETER_KEYS)[number] +>; +const _vercelModelParameterKeysExhaustive: Record = {} as Record< + never, + never +>; + +/** + * `config.model.parameters` as AI SDK call settings. The LaunchDarkly UI writes provider names + * in snake_case, so the bag is camelized (`top_p` becomes `topP`; nested values are left alone, + * and snake_case wins when both spellings are set), after two renames to the AI SDK's own names: + * + * - `max_tokens` / `max_completion_tokens` / `max_output_tokens` become `maxOutputTokens`. An + * explicit `max_output_tokens` wins, then `max_completion_tokens`, then `max_tokens`. + * - `reasoning_effort` (OpenAI's name) becomes `reasoning`, whose values it shares. An explicit + * `reasoning` wins. + * + * Only `FORWARDED_MODEL_PARAMETER_KEYS` survive. A config that sets none of them produces `{}`, + * so the call sees exactly what it always has. + */ +export function buildModelParameterOptions(parameters: AiConfigRep['model']['parameters']): Record { + const { max_tokens, max_completion_tokens, max_output_tokens, reasoning_effort, ...rest } = + normalizeModelParameters(parameters); + const renamed: Record = { ...rest }; + const maxOutputTokens = max_output_tokens ?? max_completion_tokens ?? max_tokens; + if (maxOutputTokens !== undefined) renamed.max_output_tokens = maxOutputTokens; + if (reasoning_effort !== undefined && renamed.reasoning === undefined) renamed.reasoning = reasoning_effort; + return pickForwardedModelParameters(camelizeModelParameters(renamed), FORWARDED_MODEL_PARAMETER_KEYS); +} diff --git a/packages/vercel-agents/src/native-graph.ts b/packages/vercel-agents/src/native-graph.ts index 9403640c..03d54abc 100644 --- a/packages/vercel-agents/src/native-graph.ts +++ b/packages/vercel-agents/src/native-graph.ts @@ -16,24 +16,10 @@ import { SpanStatusCode, trace } from '@opentelemetry/api'; import { jsonSchema, type LanguageModel, type ModelMessage, stepCountIs, ToolLoopAgent, type ToolSet, tool } from 'ai'; import type { VercelAgentsOptions } from './handler.js'; import { gatewayModelId } from './model-id.js'; +import { buildModelParameterOptions } from './model-parameters.js'; const TRACER_NAME = '@launchdarkly/ai-vercel-agents'; const MAX_STEPS = 10; -const OWNED_PARAMETER_NAMES = new Set([ - 'model', - 'messages', - 'prompt', - 'system', - 'instructions', - 'tools', - 'stream', - 'output', - 'outputformat', - 'stopwhen', - 'maxsteps', - 'apikey', - 'baseurl', -]); export interface VercelNativeGraphOptions extends VercelAgentsOptions { toolHandlers?: Record; @@ -54,15 +40,6 @@ function normalizeUsage(usage: unknown): Usage { return { input, output, total: numberOrZero(raw.totalTokens ?? raw.total_tokens) || input + output }; } -function modelSettings(node: GraphNode): Record { - return Object.fromEntries( - Object.entries(node.config.model.parameters ?? {}).filter(([key]) => { - const normalized = key.replaceAll('_', '').replaceAll('-', '').toLowerCase(); - return !OWNED_PARAMETER_NAMES.has(normalized); - }), - ); -} - async function resolveModel(node: GraphNode, options: VercelNativeGraphOptions): Promise { if (options.modelFactory) return options.modelFactory(node.config); if (options.model) return options.model; @@ -181,7 +158,7 @@ export const toVercelAgents = ( agents.set( node.key, new ToolLoopAgent({ - ...modelSettings(node), + ...buildModelParameterOptions(node.config.model.parameters), model: await resolveModel(node, options), ...(nodeInstructions(node, variables) ? { instructions: nodeInstructions(node, variables) } : {}), ...(Object.keys(tools).length > 0 ? { tools } : {}), diff --git a/packages/vercel-messages/src/__tests__/evaluate.test.ts b/packages/vercel-messages/src/__tests__/evaluate.test.ts index 262e5c28..f51c4b85 100644 --- a/packages/vercel-messages/src/__tests__/evaluate.test.ts +++ b/packages/vercel-messages/src/__tests__/evaluate.test.ts @@ -57,6 +57,7 @@ vi.mock('@opentelemetry/api', async (importOriginal) => { }); import { vercelEvaluate } from '../evaluate.js'; +import { expectNoNeverForwardedValue, NEVER_FORWARDED_PARAMETERS } from './never-forwarded.js'; const questions = { refunded: { type: 'boolean' as const, instructions: 'Was a refund issued?' }, @@ -167,6 +168,34 @@ describe('vercelEvaluate', () => { ); }); + it('does not forward model.parameters to experimental_evaluate', async () => { + // experimental_evaluate takes no generation settings, and the request settings it does take + // (headers, maxRetries, providerOptions) are never read from a config: call-level headers + // replace the provider's Authorization header, and the Gateway reads credentials and routing + // from providerOptions. + serverMocks.inspectConfig.mockResolvedValue({ + enabled: true, + config: { + ...config, + model: { ...config.model, parameters: { ...NEVER_FORWARDED_PARAMETERS, temperature: 0.2, max_tokens: 10 } }, + }, + meta, + }); + await vercelEvaluate('flag', 'state', context, { questions }); + const request = aiMocks.experimental_evaluate.mock.calls[0][0]; + expectNoNeverForwardedValue(request); + for (const key of ['temperature', 'max_tokens', 'maxOutputTokens']) { + expect(request).not.toHaveProperty(key); + } + }); + + it('passes options given to vercelEvaluate through', async () => { + await vercelEvaluate('flag', 'state', context, { questions, maxRetries: 0, headers: { 'x-caller': '1' } }); + expect(aiMocks.experimental_evaluate).toHaveBeenCalledWith( + expect.objectContaining({ maxRetries: 0, headers: { 'x-caller': '1' } }), + ); + }); + it('uses evaluated provider telemetry, preserves the model id, and disables runtime telemetry', async () => { await vercelEvaluate('flag', 'state', context, { questions }); expect(spanMocks.startActiveSpan).toHaveBeenCalledWith('evaluate', expect.any(Function)); diff --git a/packages/vercel-messages/src/__tests__/handler.test.ts b/packages/vercel-messages/src/__tests__/handler.test.ts index 9718bf2d..f79ddecf 100644 --- a/packages/vercel-messages/src/__tests__/handler.test.ts +++ b/packages/vercel-messages/src/__tests__/handler.test.ts @@ -189,6 +189,111 @@ describe('createVercelMessagesHandler', () => { expect(request).not.toHaveProperty('output_format'); }); + it('maps snake_case model.parameters onto the AI SDK call settings generateText reads', async () => { + const parameters = { + max_tokens: 256, + top_p: 0.9, + top_k: 40, + presence_penalty: 0.1, + frequency_penalty: 0.2, + stop_sequences: ['END'], + seed: 7, + reasoning_effort: 'low', + tool_choice: 'auto', + }; + await createVercelMessagesHandler()({ ...baseConfig, model: { ...baseConfig.model, parameters } } as any, 'hello'); + const request = aiMocks.generateText.mock.calls[0][0]; + expect(request).toMatchObject({ + maxOutputTokens: 256, + topP: 0.9, + topK: 40, + presencePenalty: 0.1, + frequencyPenalty: 0.2, + stopSequences: ['END'], + seed: 7, + reasoning: 'low', + toolChoice: 'auto', + }); + for (const key of Object.keys(parameters).filter((k) => k.includes('_'))) { + expect(request).not.toHaveProperty(key); + } + expect(request).not.toHaveProperty('maxTokens'); + expect(request).not.toHaveProperty('reasoningEffort'); + }); + + it('maps model.parameters onto streamText the same way', async () => { + aiMocks.streamText.mockReturnValue({ + textStream: (async function* () { + yield 'ok'; + })(), + usage: Promise.resolve({ inputTokens: 1, outputTokens: 1, totalTokens: 2 }), + }); + const parameters = { max_tokens: 64, top_p: 0.5, abort_signal: 'bad', stream: false }; + await collect( + createVercelMessagesHandler().stream?.( + { ...baseConfig, model: { ...baseConfig.model, parameters } } as any, + 'q', + {}, + {}, + ) as AsyncIterable, + ); + const request = aiMocks.streamText.mock.calls[0][0]; + expect(request).toMatchObject({ maxOutputTokens: 64, topP: 0.5 }); + expect(request).not.toHaveProperty('max_tokens'); + expect(request).not.toHaveProperty('abortSignal'); + expect(request).not.toHaveProperty('stream'); + }); + + it('resolves the token-limit spellings to maxOutputTokens, explicit max_output_tokens first', async () => { + const run = async (parameters: Record) => { + aiMocks.generateText.mockClear(); + await createVercelMessagesHandler()({ ...baseConfig, model: { ...baseConfig.model, parameters } } as any, 'q'); + return aiMocks.generateText.mock.calls[0][0].maxOutputTokens; + }; + expect(await run({ max_tokens: 1, max_completion_tokens: 2, max_output_tokens: 3 })).toBe(3); + expect(await run({ max_tokens: 1, max_completion_tokens: 2 })).toBe(2); + expect(await run({ max_tokens: 1 })).toBe(1); + // Both spellings of one key set: snake_case wins, as in every other handler. + expect(await run({ maxOutputTokens: 9, max_tokens: 1 })).toBe(1); + expect(await run({ maxOutputTokens: 9 })).toBe(9); + }); + + it('prefers an explicit reasoning over reasoning_effort', async () => { + await createVercelMessagesHandler()( + { + ...baseConfig, + model: { ...baseConfig.model, parameters: { reasoning: 'high', reasoning_effort: 'low' } }, + } as any, + 'q', + ); + expect(aiMocks.generateText.mock.calls[0][0].reasoning).toBe('high'); + }); + + it('does not forward keys that are not AI SDK call settings', async () => { + const parameters = { + temperature: 0.3, + allowSystemInMessages: true, + allow_system_in_messages: true, + experimental_telemetry: { isEnabled: true }, + unknown_setting: 1, + abortSignal: 'bad', + }; + await createVercelMessagesHandler()({ ...baseConfig, model: { ...baseConfig.model, parameters } } as any, 'q'); + const request = aiMocks.generateText.mock.calls[0][0]; + expect(request.temperature).toBe(0.3); + for (const key of [ + 'allowSystemInMessages', + 'allow_system_in_messages', + 'experimental_telemetry', + 'experimentalTelemetry', + 'unknown_setting', + 'unknownSetting', + 'abortSignal', + ]) { + expect(request).not.toHaveProperty(key); + } + }); + it('uses generateText messages rather than a prompt string', async () => { await createVercelMessagesHandler()(baseConfig as any, 'hello'); const request = aiMocks.generateText.mock.calls[0][0]; diff --git a/packages/vercel-messages/src/__tests__/model-parameters.test.ts b/packages/vercel-messages/src/__tests__/model-parameters.test.ts new file mode 100644 index 00000000..7f7e64fd --- /dev/null +++ b/packages/vercel-messages/src/__tests__/model-parameters.test.ts @@ -0,0 +1,119 @@ +import { createOpenAI } from '@ai-sdk/openai'; +import { createGateway, generateText } from 'ai'; +import { MockLanguageModelV4 } from 'ai/test'; +import { describe, expect, it } from 'vitest'; +import { createVercelMessagesHandler } from '../handler.js'; +import { buildModelParameterOptions } from '../model-parameters.js'; +import { expectNoNeverForwardedValue, NEVER_FORWARDED_PARAMETERS } from './never-forwarded.js'; + +// This file deliberately does NOT mock `ai`: it checks what reaches the provider's HTTP request. + +type SeenHeaders = Record[]; + +function capturingFetch(seen: SeenHeaders): typeof globalThis.fetch { + return (async (_url: unknown, init?: RequestInit) => { + seen.push(Object.fromEntries(new Headers(init?.headers).entries())); + return new Response(JSON.stringify({ error: { message: 'captured' } }), { status: 401 }); + }) as typeof globalThis.fetch; +} + +const attackerParameters = { + headers: { Authorization: 'Bearer ATTACKER' }, + extra_headers: { Authorization: 'Bearer ATTACKER' }, + provider_options: { gateway: { byok: { openai: [{ apiKey: 'ATTACKER' }] }, order: ['attacker'] } }, +}; + +function config(parameters: Record) { + return { + model: { name: 'openai/gpt-4o', parameters }, + provider: { name: 'OpenAI' }, + instructions: 'You are helpful.', + }; +} + +describe('model.parameters cannot change how the request authenticates', () => { + it('keeps the customer API key on @ai-sdk/openai when a config sets an Authorization header', async () => { + const seen: SeenHeaders = []; + const provider = createOpenAI({ apiKey: 'CUSTOMER', fetch: capturingFetch(seen) }); + await expect( + createVercelMessagesHandler({ model: provider('gpt-4o') })(config(attackerParameters) as any, 'hi'), + ).rejects.toThrow(); + expect(seen.length).toBeGreaterThan(0); + for (const headers of seen) { + expect(headers.authorization).toBe('Bearer CUSTOMER'); + } + }); + + it('keeps the customer API key on the Vercel AI Gateway when a config sets an Authorization header', async () => { + const seen: SeenHeaders = []; + const gateway = createGateway({ apiKey: 'CUSTOMER', fetch: capturingFetch(seen) }); + await expect( + createVercelMessagesHandler({ model: gateway('openai/gpt-4o') })(config(attackerParameters) as any, 'hi'), + ).rejects.toThrow(); + expect(seen.length).toBeGreaterThan(0); + for (const headers of seen) { + expect(headers.authorization).toBe('Bearer CUSTOMER'); + } + }); + + // Jeff's repro from the review, inverted: the options built from a config that sets + // `headers.Authorization` no longer replace the provider's own key. + it('buildModelParameterOptions output spread into generateText does not override the provider key', async () => { + const seen: SeenHeaders = []; + const provider = createOpenAI({ apiKey: 'CUSTOMER', fetch: capturingFetch(seen) }); + const opts = buildModelParameterOptions({ headers: { Authorization: 'Bearer ATTACKER' } } as never); + await generateText({ model: provider('gpt-4o'), prompt: 'hi', maxRetries: 0, ...opts }).catch(() => {}); + expect(seen[0]?.authorization).toBe('Bearer CUSTOMER'); + }); +}); + +describe('buildModelParameterOptions allowlist', () => { + it('drops every never-forwarded key', () => { + const options = buildModelParameterOptions({ ...NEVER_FORWARDED_PARAMETERS, temperature: 0.5 } as never); + expect(options).toEqual({ temperature: 0.5 }); + expectNoNeverForwardedValue(options); + }); + + it('never-forwarded keys do not reach the model call', async () => { + const model = new MockLanguageModelV4({ + doGenerate: async () => { + throw new Error('aborted: request captured'); + }, + }); + await expect( + createVercelMessagesHandler({ model })(config(NEVER_FORWARDED_PARAMETERS) as any, 'hi'), + ).rejects.toThrow(); + expect(model.doGenerateCalls).toHaveLength(1); + // The AI SDK sets its own `headers` (user agent) on every call; the deep check still proves + // none of the config's values reached it. + expectNoNeverForwardedValue(model.doGenerateCalls[0], ['headers']); + }); + + it('forwards the allowed keys under the AI SDK names', () => { + expect( + buildModelParameterOptions({ + temperature: 0.2, + top_p: 0.9, + top_k: 40, + presence_penalty: 0.1, + frequency_penalty: 0.3, + stop_sequences: ['END'], + seed: 7, + max_tokens: 100, + reasoning_effort: 'low', + tool_choice: 'auto', + } as never), + ).toEqual({ + temperature: 0.2, + topP: 0.9, + topK: 40, + presencePenalty: 0.1, + frequencyPenalty: 0.3, + stopSequences: ['END'], + seed: 7, + maxOutputTokens: 100, + reasoning: 'low', + toolChoice: 'auto', + }); + }); +}); diff --git a/packages/vercel-messages/src/__tests__/never-forwarded.ts b/packages/vercel-messages/src/__tests__/never-forwarded.ts new file mode 100644 index 00000000..300d6e51 --- /dev/null +++ b/packages/vercel-messages/src/__tests__/never-forwarded.ts @@ -0,0 +1,163 @@ +import { expect } from 'vitest'; + +/** + * The value every never-forwarded key carries in `NEVER_FORWARDED_PARAMETERS`. A handler that + * passes any of those keys on, under any name or nested anywhere, leaves this string somewhere in + * the options it builds, which is what `expectNoNeverForwardedValue` looks for. + */ +export const NEVER_FORWARDED = 'NEVER_FORWARDED'; + +/** + * Keys that must never reach a provider, framework or host-process option from + * `model.parameters`, in both the snake_case the LaunchDarkly UI writes and the camelCase the + * frameworks read: credentials, endpoints and connection settings, request injection, remote + * tools, and Claude Agents host-process settings. + */ +export const NEVER_FORWARDED_KEYS = [ + // credentials + 'api_key', + 'apiKey', + 'openai_api_key', + 'openAIApiKey', + 'anthropic_api_key', + 'anthropicApiKey', + 'bedrock_api_key', + 'bedrockApiKey', + 'bedrock_api_secret', + 'bedrockApiSecret', + 'credentials', + 'auth_token', + 'authToken', + 'access_token', + 'accessToken', + 'aws_access_key_id', + 'awsAccessKeyId', + 'aws_secret_access_key', + 'awsSecretAccessKey', + 'aws_session_token', + 'awsSessionToken', + 'organization', + // endpoints and connection + 'base_url', + 'baseURL', + 'baseUrl', + 'anthropic_api_url', + 'anthropicApiUrl', + 'endpoint', + 'endpoint_host', + 'endpointHost', + 'endpoint_url', + 'endpointUrl', + 'region', + 'region_name', + 'regionName', + 'configuration', + 'client_options', + 'clientOptions', + 'default_headers', + 'defaultHeaders', + 'timeout', + 'max_retries', + 'maxRetries', + 'fetch', + 'http_client', + 'httpClient', + 'http_agent', + 'httpAgent', + 'proxy', + 'proxies', + 'client', + // request injection + 'headers', + 'extra_headers', + 'extraHeaders', + 'extra_body', + 'extraBody', + 'extra_query', + 'extraQuery', + 'model_kwargs', + 'modelKwargs', + 'invocation_kwargs', + 'invocationKwargs', + 'additional_model_request_fields', + 'additionalModelRequestFields', + 'provider_data', + 'providerData', + 'provider_options', + 'providerOptions', + // remote tools + 'mcp_servers', + 'mcpServers', + // Claude Agents host-process settings + 'path_to_claude_code_executable', + 'pathToClaudeCodeExecutable', + 'cli_path', + 'cliPath', + 'executable', + 'executable_args', + 'executableArgs', + 'extra_args', + 'extraArgs', + 'env', + 'cwd', + 'additional_directories', + 'additionalDirectories', + 'add_dirs', + 'addDirs', + 'permission_mode', + 'permissionMode', + 'allow_dangerously_skip_permissions', + 'allowDangerouslySkipPermissions', + 'settings', + 'setting_sources', + 'settingSources', + 'managed_settings', + 'managedSettings', + 'plugins', + 'sandbox', + 'resume', + 'resume_session_at', + 'resumeSessionAt', + 'session_id', + 'sessionId', + 'fork_session', + 'forkSession', + 'continue', + 'hooks', + 'can_use_tool', + 'canUseTool', + 'stderr', + 'abort_controller', + 'abortController', + 'spawn_claude_code_process', + 'spawnClaudeCodeProcess', +] as const; + +/** A `model.parameters` bag that sets every key in `NEVER_FORWARDED_KEYS` to `NEVER_FORWARDED`. */ +export const NEVER_FORWARDED_PARAMETERS: Record = Object.fromEntries( + NEVER_FORWARDED_KEYS.map((key) => [key, NEVER_FORWARDED]), +); + +function findNeverForwarded(value: unknown, path: string, seen: Set): string | undefined { + if (value === NEVER_FORWARDED) return path || '(root)'; + if (!value || typeof value !== 'object' || seen.has(value)) return undefined; + seen.add(value); + for (const [key, child] of Object.entries(value)) { + const found = findNeverForwarded(child, path ? `${path}.${key}` : key, seen); + if (found) return found; + } + return undefined; +} + +/** + * Asserts that no `NEVER_FORWARDED` value appears anywhere in `options`, and that none of the + * `NEVER_FORWARDED_KEYS` the handler does not own itself is set on it. `handlerOwned` lists the + * keys the handler legitimately sets from its own state (Claude Agents sets `mcpServers` and + * `hooks`, for example); those must still not carry the config's value. + */ +export function expectNoNeverForwardedValue(options: unknown, handlerOwned: ReadonlyArray = []): void { + expect(findNeverForwarded(options, '', new Set())).toBeUndefined(); + const record = options as Record; + const leaked = NEVER_FORWARDED_KEYS.filter((key) => !handlerOwned.includes(key) && record[key] !== undefined); + expect(leaked).toEqual([]); +} diff --git a/packages/vercel-messages/src/__tests__/wire.test.ts b/packages/vercel-messages/src/__tests__/wire.test.ts new file mode 100644 index 00000000..d5ab1cbd --- /dev/null +++ b/packages/vercel-messages/src/__tests__/wire.test.ts @@ -0,0 +1,55 @@ +import { MockLanguageModelV4 } from 'ai/test'; +import { describe, expect, it } from 'vitest'; +import { createVercelMessagesHandler } from '../handler.js'; + +// This file deliberately does NOT mock `ai`. The handler tests mock generateText/streamText, so +// they only prove what the handler hands to the AI SDK, not that the AI SDK reads it: it silently +// drops any call setting it does not recognize, which is how snake_case `max_tokens` / `top_p` +// went nowhere before. Running the real generateText/streamText against a mock language model +// checks the settings that actually reach the model call. + +const parameters = { + max_tokens: 321, + top_p: 0.42, + stop_sequences: ['END'], +}; + +const baseConfig = { + model: { name: 'openai/gpt-5', parameters }, + provider: { name: 'OpenAI' }, + instructions: 'You are helpful.', +}; + +function capturingModel() { + return new MockLanguageModelV4({ + doGenerate: async () => { + throw new Error('aborted: request captured'); + }, + doStream: async () => { + throw new Error('aborted: request captured'); + }, + }); +} + +describe('createVercelMessagesHandler, model call (real AI SDK, mock model)', () => { + it('generateText sends model.parameters to the model as maxOutputTokens / topP / stopSequences', async () => { + const model = capturingModel(); + await expect(createVercelMessagesHandler({ model })(baseConfig as any, 'hi')).rejects.toThrow(); + expect(model.doGenerateCalls).toHaveLength(1); + expect(model.doGenerateCalls[0]).toMatchObject({ maxOutputTokens: 321, topP: 0.42, stopSequences: ['END'] }); + }); + + it('streamText sends the same settings', async () => { + const model = capturingModel(); + const stream = createVercelMessagesHandler({ model }).stream?.(baseConfig as any, 'hi', {}, {}); + await expect( + (async () => { + for await (const _event of stream as AsyncIterable) { + // drain + } + })(), + ).rejects.toThrow(); + expect(model.doStreamCalls).toHaveLength(1); + expect(model.doStreamCalls[0]).toMatchObject({ maxOutputTokens: 321, topP: 0.42, stopSequences: ['END'] }); + }); +}); diff --git a/packages/vercel-messages/src/handler.ts b/packages/vercel-messages/src/handler.ts index 66f6dafd..711dad87 100644 --- a/packages/vercel-messages/src/handler.ts +++ b/packages/vercel-messages/src/handler.ts @@ -36,25 +36,11 @@ import { tool, } from 'ai'; import { gatewayModelId } from './model-id.js'; +import { buildModelParameterOptions } from './model-parameters.js'; const TRACER_NAME = '@launchdarkly/ai-vercel-messages'; const MAX_STEPS = 10; -const OWNED_PARAMETER_NAMES = new Set([ - 'model', - 'messages', - 'prompt', - 'system', - 'tools', - 'stream', - 'output', - 'outputformat', - 'stopwhen', - 'maxsteps', - 'apikey', - 'baseurl', -]); - export type VercelModelSource = LanguageModel | ((config: AiConfigRep) => LanguageModel | Promise); export interface VercelMessagesOptions extends ContentCaptureOptions { @@ -88,15 +74,6 @@ function servingProvider(config: AiConfigRep): string { return (config.provider?.name || 'unknown').toLowerCase(); } -function modelSettings(config: AiConfigRep): Record { - return Object.fromEntries( - Object.entries(config.model.parameters ?? {}).filter(([key]) => { - const normalized = key.replaceAll('_', '').replaceAll('-', '').toLowerCase(); - return !OWNED_PARAMETER_NAMES.has(normalized); - }), - ); -} - async function resolveModel(config: AiConfigRep, options: VercelMessagesOptions): Promise { if (options.modelFactory) return options.modelFactory(config); if (options.model) return options.model; @@ -300,7 +277,7 @@ export function createVercelMessagesHandler(options: VercelMessagesOptions = {}) }); const output = requestOutput(configRep.outputFormat); const result = await generateText({ - ...modelSettings(configRep), + ...buildModelParameterOptions(configRep.model.parameters), model: await resolveModel(configRep, options), messages, ...(system ? { system } : {}), @@ -349,7 +326,7 @@ export function createVercelMessagesHandler(options: VercelMessagesOptions = {}) }); chatSpan = startChatSpan(configRep, rootSpan); const result = streamText({ - ...modelSettings(configRep), + ...buildModelParameterOptions(configRep.model.parameters), model: await resolveModel(configRep, options), messages, ...(system ? { system } : {}), diff --git a/packages/vercel-messages/src/model-parameters.ts b/packages/vercel-messages/src/model-parameters.ts new file mode 100644 index 00000000..9b8087cd --- /dev/null +++ b/packages/vercel-messages/src/model-parameters.ts @@ -0,0 +1,81 @@ +import { + type AiConfigRep, + camelizeModelParameters, + normalizeModelParameters, + pickForwardedModelParameters, +} from '@launchdarkly/ai-server'; +import type { generateText, LanguageModelCallOptions, RequestOptions } from 'ai'; + +/** + * AI SDK call settings this package's `generateText` / `streamText` calls take from + * `config.model.parameters`, after the renames in `buildModelParameterOptions`. + * + * The AI SDK drops any call setting it does not recognize without an error, so a key that is + * not spelled exactly as below (snake_case `max_tokens`, `top_p`) reaches nothing. This list is + * the `LanguageModelCallOptions` generation settings plus `toolChoice`. + * + * Handler-owned (set by the handler from the config and the call shape, never from + * `model.parameters`): `model`, `messages`, `prompt`, `system`, `instructions`, `tools`, + * `stopWhen`, `output`. + * + * Never forwarded, because they configure the request rather than the model and a config must not + * be able to change them: + * - `headers`: call-level headers win over the provider's own, so a config could replace the + * customer's `Authorization` header. + * - `providerOptions`: the Vercel AI Gateway reads credentials (`gateway.byok`) and routing + * (`gateway.order`, `gateway.only`, `gateway.models`) from it. + * - `maxRetries`, `timeout`: client retry and timeout policy. + * - `abortSignal`: cannot be expressed in a JSON config and belongs to the caller. + * + * Every other `generateText` field (callbacks, telemetry, tool repair, approval, `activeTools`, + * `include`, ...) is either a function or wiring this handler does not expose, so it is not + * forwarded either. + */ +const FORWARDED_MODEL_PARAMETER_KEYS = [ + 'frequencyPenalty', + 'maxOutputTokens', + 'presencePenalty', + 'reasoning', + 'seed', + 'stopSequences', + 'temperature', + 'toolChoice', + 'topK', + 'topP', +] as const satisfies ReadonlyArray[0]>; + +type VercelExcludedKeys = 'abortSignal' | 'headers' | 'maxRetries' | 'providerOptions' | 'timeout'; +// If the AI SDK adds a call setting and it is not classified above as forwarded or excluded, +// this type resolves to something other than `never` and the assignment below fails to compile, +// naming the unclassified key. +type VercelUndecidedModelParameterKeys = Exclude< + keyof LanguageModelCallOptions | keyof RequestOptions, + VercelExcludedKeys | (typeof FORWARDED_MODEL_PARAMETER_KEYS)[number] +>; +const _vercelModelParameterKeysExhaustive: Record = {} as Record< + never, + never +>; + +/** + * `config.model.parameters` as AI SDK call settings. The LaunchDarkly UI writes provider names + * in snake_case, so the bag is camelized (`top_p` becomes `topP`; nested values are left alone, + * and snake_case wins when both spellings are set), after two renames to the AI SDK's own names: + * + * - `max_tokens` / `max_completion_tokens` / `max_output_tokens` become `maxOutputTokens`. An + * explicit `max_output_tokens` wins, then `max_completion_tokens`, then `max_tokens`. + * - `reasoning_effort` (OpenAI's name) becomes `reasoning`, whose values it shares. An explicit + * `reasoning` wins. + * + * Only `FORWARDED_MODEL_PARAMETER_KEYS` survive. A config that sets none of them produces `{}`, + * so the call sees exactly what it always has. + */ +export function buildModelParameterOptions(parameters: AiConfigRep['model']['parameters']): Record { + const { max_tokens, max_completion_tokens, max_output_tokens, reasoning_effort, ...rest } = + normalizeModelParameters(parameters); + const renamed: Record = { ...rest }; + const maxOutputTokens = max_output_tokens ?? max_completion_tokens ?? max_tokens; + if (maxOutputTokens !== undefined) renamed.max_output_tokens = maxOutputTokens; + if (reasoning_effort !== undefined && renamed.reasoning === undefined) renamed.reasoning = reasoning_effort; + return pickForwardedModelParameters(camelizeModelParameters(renamed), FORWARDED_MODEL_PARAMETER_KEYS); +}