From ef566522904921d909c61537fd1664abbbd09172 Mon Sep 17 00:00:00 2001 From: Alexis Georges Date: Wed, 28 Oct 2026 15:38:49 -0400 Subject: [PATCH 01/11] feat(client): pass the evaluated variation to handlers extractVariation now returns the raw variation it already gets from client.variation, and invoke, stream, inline judges and graph nodes hand it to handlers as variables.__ldValue, next to __ld. It is kept out of TrackData on purpose: TrackData is the payload of every $ld:ai:* track event, and the value includes the prompt. --- .../client/src/__tests__/tracking.test.ts | 28 +++++++++++++++++++ packages/client/src/client.ts | 6 ++-- packages/client/src/graph.ts | 7 ++++- packages/client/src/judges.ts | 5 ++-- packages/client/src/lifecycle.ts | 6 ++-- packages/client/src/tracking.ts | 22 ++++++++++++++- 6 files changed, 66 insertions(+), 8 deletions(-) diff --git a/packages/client/src/__tests__/tracking.test.ts b/packages/client/src/__tests__/tracking.test.ts index c82d2ab9..bf95bc0d 100644 --- a/packages/client/src/__tests__/tracking.test.ts +++ b/packages/client/src/__tests__/tracking.test.ts @@ -149,6 +149,34 @@ describe('executeAndTrack', () => { expect(result.response).toBe('Hello!'); }); + it('passes evaluatedValue to the handler as __ldValue and keeps it out of trackData and events', async () => { + const handler = makeHandler('ok'); + const raw = { ...execConfig, _ldMeta: { enabled: true, variationKey: 'v1' } }; + const result = await executeAndTrack({ + configKey: 'my-flag', + config: execConfig as any, + meta: execMeta, + userContext: execContext, + handler, + evaluatedValue: raw, + }); + expect(handler.mock.calls[0][3].__ldValue).toBe(raw); + expect(result.trackData).not.toHaveProperty('__ldValue'); + for (const call of mockTrack.mock.calls) expect(JSON.stringify(call[2])).not.toContain('You are helpful.'); + }); + + it('does not set __ldValue when evaluatedValue is absent', async () => { + const handler = makeHandler('ok'); + await executeAndTrack({ + configKey: 'my-flag', + config: execConfig as any, + meta: execMeta, + userContext: execContext, + handler, + }); + expect(handler.mock.calls[0][3]).not.toHaveProperty('__ldValue'); + }); + it('returns parsed usage with correct totals', async () => { const handler = makeHandler('hi', { input_tokens: 5, output_tokens: 10 }); const result = await executeAndTrack({ diff --git a/packages/client/src/client.ts b/packages/client/src/client.ts index 63c96c07..a01e8710 100644 --- a/packages/client/src/client.ts +++ b/packages/client/src/client.ts @@ -75,7 +75,7 @@ export const configInternal = ({ key, handler, toolHandlers, registry, skipJudge const resolvedHandlerArray = resolveHandlers(registry, normalizeHandlers()); const resolvedTools = resolveTools(registry, toolHandlers); - const { config: aiConfig, meta } = await extractVariation(key, context); + const { config: aiConfig, meta, value: evaluatedValue } = await extractVariation(key, context); const resolvedHandler = selectHandler(aiConfig, meta, resolvedHandlerArray); const { @@ -93,6 +93,7 @@ export const configInternal = ({ key, handler, toolHandlers, registry, skipJudge toolHandlers: resolvedTools, variables, history, + evaluatedValue, }); const parsedResponse = resolveOutputFormatResponse(rawResponse, aiConfig.outputFormat); @@ -161,7 +162,7 @@ export const configInternal = ({ key, handler, toolHandlers, registry, skipJudge const resolvedHandlerArray = resolveHandlers(registry, normalizeHandlers()); const resolvedTools = resolveTools(registry, toolHandlers); - const { config: aiConfig, meta } = await extractVariation(key, context); + const { config: aiConfig, meta, value: evaluatedValue } = await extractVariation(key, context); const resolvedHandler = selectHandler(aiConfig, meta, resolvedHandlerArray); let doneEvent: ExecuteStreamDoneEvent | undefined; @@ -176,6 +177,7 @@ export const configInternal = ({ key, handler, toolHandlers, registry, skipJudge toolHandlers: resolvedTools, variables, history, + evaluatedValue, })) { if (event.type === 'chunk') { yield event; diff --git a/packages/client/src/graph.ts b/packages/client/src/graph.ts index de9899f5..6ab22f85 100644 --- a/packages/client/src/graph.ts +++ b/packages/client/src/graph.ts @@ -178,9 +178,12 @@ const buildGraph = async ( const edgesFrom = (nodeKey: string): GraphEdge[] => edges.filter((e) => e.sourceKey === nodeKey); const nodes = new Map(); + // Raw variation per node, kept beside `GraphNode` so the public type is unchanged. + const nodeValues = new Map(); try { for (const nodeKey of allKeys) { - const { config, meta: nodeMeta } = await extractVariation(nodeKey, context); + const { config, meta: nodeMeta, value } = await extractVariation(nodeKey, context); + nodeValues.set(nodeKey, value); const nodeEdges = edgesFrom(nodeKey); nodes.set(nodeKey, { key: nodeKey, @@ -320,6 +323,7 @@ const buildGraph = async ( variables: opts.variables, graphKey: key, history: opts.history, + evaluatedValue: nodeValues.get(node.key), })) { if (event.type === 'chunk') { yield { type: 'chunk', text: event.text, nodeKey: node.key }; @@ -427,6 +431,7 @@ const buildGraph = async ( variables: opts.variables, graphKey: key, history: opts.history, + evaluatedValue: nodeValues.get(node.key), })) { if (event.type === 'chunk') { yield { type: 'chunk', text: event.text, nodeKey: node.key }; diff --git a/packages/client/src/judges.ts b/packages/client/src/judges.ts index c9310e14..4b2ef594 100644 --- a/packages/client/src/judges.ts +++ b/packages/client/src/judges.ts @@ -34,7 +34,7 @@ import { collapseMessagesToInstructions, normalizeMode, parseJSONWithPossibleFen const resolveJudge = async ( key: string, userContext: LDContext, -): Promise<{ config: AiConfigRep; meta: VariationMeta } | null> => { +): Promise<{ config: AiConfigRep; meta: VariationMeta; value: unknown } | null> => { try { return await extractVariation(key, userContext); } catch (err) { @@ -144,7 +144,7 @@ export const runJudges = async ({ const resolved = await resolveJudge(judge.key, userContext); if (!resolved) continue; - const { config: judgeConfig, meta: judgeMeta } = resolved; + const { config: judgeConfig, meta: judgeMeta, value: judgeValue } = resolved; const judgeProvider = judgeConfig.provider?.name; const judgeMode = normalizeMode(judgeMeta.mode); @@ -198,6 +198,7 @@ export const runJudges = async ({ configKey: judge.key, config: effectiveJudgeConfig, meta: judgeMeta, + evaluatedValue: judgeValue, userContext, handler: judgeHandler, userInput: llmResponse, diff --git a/packages/client/src/lifecycle.ts b/packages/client/src/lifecycle.ts index 1f1bce60..4322c3f3 100644 --- a/packages/client/src/lifecycle.ts +++ b/packages/client/src/lifecycle.ts @@ -578,7 +578,7 @@ export async function inspectConfigInternal(key: string, context: LDContext): Pr export const extractVariation = async ( key: string, userContext: LDContext, -): Promise<{ config: AiConfigRep; meta: VariationMeta }> => { +): Promise<{ config: AiConfigRep; meta: VariationMeta; value: unknown }> => { await initClient(); const variation = await getClient().variation(key, userContext, { enabled: false }); // biome-ignore lint/suspicious/noExplicitAny: _ldMeta is LaunchDarkly private metadata not in public variation type @@ -590,6 +590,8 @@ export const extractVariation = async ( throw new Error(`Invalid AI config for "${key}": ${parsed.error.message}`); } + // `value` is the variation exactly as the SDK returned it, before parsing. It is written to the + // `feature_flag` span event as `feature_flag.result.value`. // biome-ignore lint/suspicious/noExplicitAny: _ldMeta is LaunchDarkly private metadata not in public variation type - return { config: parsed.data, meta: (variation as any)._ldMeta as VariationMeta }; + return { config: parsed.data, meta: (variation as any)._ldMeta as VariationMeta, value: variation }; }; diff --git a/packages/client/src/tracking.ts b/packages/client/src/tracking.ts index 1852cbc3..9369fbfc 100644 --- a/packages/client/src/tracking.ts +++ b/packages/client/src/tracking.ts @@ -224,6 +224,7 @@ export const executeAndTrack = async ({ variables, graphKey, history, + evaluatedValue, }: { configKey: string; config: AiConfigRep; @@ -235,6 +236,12 @@ export const executeAndTrack = async ({ variables?: Record; graphKey?: string; history?: Message[]; + /** + * The flag value as the LaunchDarkly SDK returned it, before parsing. Passed to the handler as + * `variables.__ldValue` (not in `TrackData`, which is the `$ld:ai:*` event payload) so + * `setLdSpanAttributes` can write `feature_flag.result.value`. Omit it when there is none. + */ + evaluatedValue?: unknown; }): Promise<{ usage: TokenUsage; response: unknown; trackData: TrackData; trajectory: string }> => { const trackData = makeRunTrackData({ configKey, config, meta, graphKey }); @@ -252,6 +259,7 @@ export const executeAndTrack = async ({ ...variables, ldContext: { ...userContext }, __ld: trackData, + ...(evaluatedValue !== undefined ? { __ldValue: evaluatedValue } : {}), }, history, ); @@ -302,6 +310,7 @@ export async function* executeAndStream({ variables, graphKey, history, + evaluatedValue, }: { configKey: string; config: AiConfigRep; @@ -313,11 +322,22 @@ export async function* executeAndStream({ variables?: Record; graphKey?: string; history?: Message[]; + /** + * The flag value as the LaunchDarkly SDK returned it, before parsing. Passed to the handler as + * `variables.__ldValue` (not in `TrackData`, which is the `$ld:ai:*` event payload) so + * `setLdSpanAttributes` can write `feature_flag.result.value`. Omit it when there is none. + */ + evaluatedValue?: unknown; }): AsyncGenerator { const trackData = makeRunTrackData({ configKey, config, meta, graphKey }); const { tools: trackedToolHandlers, recorder } = instrumentTools(config, toolHandlers, userContext, trackData); - const mergedVariables = { ...variables, ldContext: { ...userContext }, __ld: trackData }; + const mergedVariables = { + ...variables, + ldContext: { ...userContext }, + __ld: trackData, + ...(evaluatedValue !== undefined ? { __ldValue: evaluatedValue } : {}), + }; const startTime = Date.now(); let fullText = ''; From 0d6b0287ab9fd813199e9e43fb9e82959d29576f Mon Sep 17 00:00:00 2001 From: Alexis Georges Date: Wed, 28 Oct 2026 15:38:50 -0400 Subject: [PATCH 02/11] feat(client): add result variant and value to the feature_flag span event AI Config monitoring only finds traces whose feature_flag event carries feature_flag.result.value, which the LaunchDarkly observability plugins always send. Write the variation key as feature_flag.result.variant (conditionally required by the OpenTelemetry feature-flag conventions) and the evaluated value as feature_flag.result.value, serialized with JSON.stringify exactly like TracingHook with includeValue. Both are omitted when there is nothing to write, for example on native-graph spans. --- AGENTS.md | 12 ++++++- .../src/__tests__/handler.test.ts | 3 ++ packages/client/src/__tests__/utils.test.ts | 35 +++++++++++++++++++ packages/client/src/utils.ts | 21 ++++++++++- .../src/__tests__/handler.test.ts | 3 ++ .../src/__tests__/handler.test.ts | 3 ++ .../src/__tests__/handler.test.ts | 3 ++ 7 files changed, 78 insertions(+), 2 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index c64bc6da..c2bf2e40 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -630,7 +630,17 @@ Span events are not a content carrier. OTEP 4430 deprecated the span-event recording API, and the `gen_ai.content.prompt` / `gen_ai.content.completion` events these handlers used to emit were read by nothing on the LaunchDarkly side. The only event a handler emits is `feature_flag`, on the root, for -trace correlation. When `variables.ldContext` has a usable identity, that +trace correlation. It carries `feature_flag.result.variant` (the served +variation key) whenever that key is non-empty; graph spans can have an empty +one, and then it is left off. It also carries `feature_flag.result.value`: +`JSON.stringify` of the raw variation the SDK returned (messages or +instructions, model, provider, `_ldMeta`, judge configuration), the same +string the LaunchDarkly OTel `TracingHook` writes with `includeValue: true`. +The core client passes that raw value to handlers as `variables.__ldValue`, +beside `__ld`, because `TrackData` is the `$ld:ai:*` event payload. It is not +gated by content capture, and it is left off when there is no evaluated +value, such as on native graph spans. +When `variables.ldContext` has a usable identity, that event also carries `feature_flag.context.id` and `feature_flag.contextKeys`, and the root span gets `context.contextKeys.`. Child spans must not. diff --git a/packages/claude-agents/src/__tests__/handler.test.ts b/packages/claude-agents/src/__tests__/handler.test.ts index 38f8be66..5381a50d 100644 --- a/packages/claude-agents/src/__tests__/handler.test.ts +++ b/packages/claude-agents/src/__tests__/handler.test.ts @@ -531,6 +531,7 @@ describe('createClaudeAgentsHandler', () => { modelName: 'test-model', providerName: 'TestProvider', }, + __ldValue: { model: { name: 'test-model' }, _ldMeta: { enabled: true, variationKey: 'variation-a' } }, }; it('emits the feature_flag correlation event on the invoke_agent root', async () => { @@ -541,6 +542,8 @@ describe('createClaudeAgentsHandler', () => { 'feature_flag.key': 'test-config', 'feature_flag.provider.name': 'LaunchDarkly', 'feature_flag.set.id': 'env-abc', + 'feature_flag.result.variant': 'variation-a', + 'feature_flag.result.value': JSON.stringify(ldFixture.__ldValue), }); expect(mockSpan.setAttribute).toHaveBeenCalledWith('launchdarkly.operation.type', 'gen_ai'); expect(mockSpan.setAttribute).toHaveBeenCalledWith('launchdarkly.config.key', 'test-config'); diff --git a/packages/client/src/__tests__/utils.test.ts b/packages/client/src/__tests__/utils.test.ts index d7ccf2fc..ca667e9d 100644 --- a/packages/client/src/__tests__/utils.test.ts +++ b/packages/client/src/__tests__/utils.test.ts @@ -380,6 +380,41 @@ describe('setLdSpanAttributes', () => { const eventCall = span.addEvent.mock.calls.find((c: any[]) => c[0] === 'feature_flag'); expect(eventCall?.[1]).not.toHaveProperty('feature_flag.set.id'); }); + + it('includes feature_flag.result.variant in the event from variationKey', () => { + const span = makeMockSpan(); + setLdSpanAttributes(span as any, { __ld: { ...ldFixture, variationKey: 'var-a' } }); + expect(span.addEvent).toHaveBeenCalledWith( + 'feature_flag', + expect.objectContaining({ 'feature_flag.result.variant': 'var-a' }), + ); + }); + + it('omits feature_flag.result.variant when variationKey is empty', () => { + const span = makeMockSpan(); + setLdSpanAttributes(span as any, { __ld: { ...ldFixture, variationKey: '', graphKey: 'my-graph' } }); + const eventCall = span.addEvent.mock.calls.find((c: any[]) => c[0] === 'feature_flag'); + expect(eventCall?.[1]).not.toHaveProperty('feature_flag.result.variant'); + }); + + it('includes feature_flag.result.value as the JSON of the raw evaluated variation', () => { + const span = makeMockSpan(); + const raw = { + model: { name: 'gpt-x', parameters: { temperature: 0.2 } }, + provider: { name: 'openai' }, + messages: [{ role: 'system', content: 'Be brief.' }], + _ldMeta: { enabled: true, variationKey: 'var-a', version: 3 }, + }; + setLdSpanAttributes(span as any, { __ld: ldFixture, __ldValue: raw }); + // Same serialization as the LaunchDarkly OTel TracingHook with includeValue: JSON.stringify(detail.value). + expect(featureFlagAttrs(span)['feature_flag.result.value']).toBe(JSON.stringify(raw)); + }); + + it('omits feature_flag.result.value when there is no evaluated value (graph spans)', () => { + const span = makeMockSpan(); + setLdSpanAttributes(span as any, { __ld: { ...ldFixture, variationKey: '', graphKey: 'my-graph' } }); + expect(featureFlagAttrs(span)).not.toHaveProperty('feature_flag.result.value'); + }); }); // ─── setLdSpanAttributes — context identity ─────────────────────────────────── diff --git a/packages/client/src/utils.ts b/packages/client/src/utils.ts index d07f0f9e..7fe975e2 100644 --- a/packages/client/src/utils.ts +++ b/packages/client/src/utils.ts @@ -471,7 +471,15 @@ function contextIdentityFromLdContext(ldContext: unknown): ContextIdentity | und * Span event (required for AI Config Monitoring tab trace correlation): * name: 'feature_flag' * attributes: feature_flag.key, feature_flag.provider.name, feature_flag.set.id, - * feature_flag.context.id, feature_flag.contextKeys + * feature_flag.result.variant, feature_flag.result.value, + * feature_flag.context.id, + * feature_flag.contextKeys + * feature_flag.result.variant is the served variationKey, omitted when empty + * (graph spans can carry an empty variationKey). + * feature_flag.result.value is `JSON.stringify` of `variables.__ldValue`, the + * raw variation the SDK returned (as the OTel TracingHook writes it with + * `includeValue: true`). Omitted when absent, e.g. on native graph spans. + * Not gated by content capture. * The observability backend projects these event attrs to span-level attrs, * satisfying the query: events.name=feature_flag AND * events.attributes.feature_flag.key= AND @@ -493,6 +501,17 @@ export function setLdSpanAttributes(span: Span, variables: Record { modelName: 'test-model', providerName: 'TestProvider', }, + __ldValue: { model: { name: 'test-model' }, _ldMeta: { enabled: true, variationKey: 'variation-a' } }, }; it('emits the feature_flag correlation event on the invoke_agent root', async () => { @@ -360,6 +361,8 @@ describe('createLangChainAgentsHandler', () => { 'feature_flag.key': 'test-config', 'feature_flag.provider.name': 'LaunchDarkly', 'feature_flag.set.id': 'env-abc', + 'feature_flag.result.variant': 'variation-a', + 'feature_flag.result.value': JSON.stringify(ldFixture.__ldValue), }); expect(mockSpan.setAttribute).toHaveBeenCalledWith('launchdarkly.operation.type', 'gen_ai'); expect(mockSpan.setAttribute).toHaveBeenCalledWith('launchdarkly.config.key', 'test-config'); diff --git a/packages/langchain-messages/src/__tests__/handler.test.ts b/packages/langchain-messages/src/__tests__/handler.test.ts index c740468a..52fea7a3 100644 --- a/packages/langchain-messages/src/__tests__/handler.test.ts +++ b/packages/langchain-messages/src/__tests__/handler.test.ts @@ -582,6 +582,7 @@ describe('createLangChainMessagesHandler', () => { modelName: 'test-model', providerName: 'TestProvider', }, + __ldValue: { model: { name: 'test-model' }, _ldMeta: { enabled: true, variationKey: 'variation-a' } }, }; it('emits the feature_flag correlation event and launchdarkly attributes on the root span', async () => { @@ -591,6 +592,8 @@ describe('createLangChainMessagesHandler', () => { 'feature_flag.key': 'test-config', 'feature_flag.provider.name': 'LaunchDarkly', 'feature_flag.set.id': 'env-abc', + 'feature_flag.result.variant': 'variation-a', + 'feature_flag.result.value': JSON.stringify(ldFixture.__ldValue), }); expect(mockSpan.setAttribute).toHaveBeenCalledWith('launchdarkly.operation.type', 'gen_ai'); expect(mockSpan.setAttribute).toHaveBeenCalledWith('launchdarkly.config.key', 'test-config'); diff --git a/packages/openai-agents/src/__tests__/handler.test.ts b/packages/openai-agents/src/__tests__/handler.test.ts index dc562563..02beb596 100644 --- a/packages/openai-agents/src/__tests__/handler.test.ts +++ b/packages/openai-agents/src/__tests__/handler.test.ts @@ -606,6 +606,7 @@ describe('createOpenAIAgentsHandler', () => { modelName: 'test-model', providerName: 'TestProvider', }, + __ldValue: { model: { name: 'test-model' }, _ldMeta: { enabled: true, variationKey: 'variation-a' } }, }; it('emits the feature_flag correlation event on the invoke_agent root', async () => { @@ -617,6 +618,8 @@ describe('createOpenAIAgentsHandler', () => { 'feature_flag.key': 'test-config', 'feature_flag.provider.name': 'LaunchDarkly', 'feature_flag.set.id': 'env-abc', + 'feature_flag.result.variant': 'variation-a', + 'feature_flag.result.value': JSON.stringify(ldFixture.__ldValue), }); expect(mockSpan.setAttribute).toHaveBeenCalledWith('launchdarkly.operation.type', 'gen_ai'); expect(mockSpan.setAttribute).toHaveBeenCalledWith('launchdarkly.config.key', 'test-config'); From f6ba4377949570065450892e2e32daeb7c7ea258 Mon Sep 17 00:00:00 2001 From: Alexis Georges Date: Wed, 28 Oct 2026 16:01:32 -0400 Subject: [PATCH 03/11] feat(client): respect content capture for feature_flag.result.value The raw variation holds the prompt, so writing it on every run's span event bypassed the content-capture opt-in. setLdSpanAttributes now takes a captureContent argument (default false). With capture on and a raw variation present, feature_flag.result.value is its JSON, as before. Otherwise it is the JSON-encoded variation key, and it is omitted when there is neither. Every handler passes the capture setting it already uses for content attributes, as does vercelEvaluate. Native graph and client graph spans carry no raw variation, so they keep the default. --- AGENTS.md | 15 ++++--- .../src/__tests__/handler.test.ts | 2 +- packages/claude-agents/src/handler.ts | 4 +- packages/claude-messages/src/handler.ts | 4 +- packages/client/etc/ai-server.api.md | 2 +- packages/client/src/__tests__/utils.test.ts | 43 +++++++++++++------ packages/client/src/utils.ts | 34 ++++++++++----- .../src/__tests__/handler.test.ts | 2 +- packages/langchain-agents/src/handler.ts | 4 +- .../src/__tests__/handler.test.ts | 2 +- packages/langchain-messages/src/handler.ts | 4 +- packages/litellm-agents/src/handler.ts | 4 +- packages/litellm-messages/src/handler.ts | 4 +- .../src/__tests__/handler.test.ts | 2 +- packages/openai-agents/src/handler.ts | 4 +- packages/openai-messages/src/handler.ts | 4 +- packages/typesafe/src/handler.ts | 2 +- packages/vercel-agents/src/handler.ts | 4 +- packages/vercel-messages/src/evaluate.ts | 2 +- packages/vercel-messages/src/handler.ts | 4 +- 20 files changed, 90 insertions(+), 56 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index c2bf2e40..a7f08d9b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -632,14 +632,17 @@ events these handlers used to emit were read by nothing on the LaunchDarkly side. The only event a handler emits is `feature_flag`, on the root, for trace correlation. It carries `feature_flag.result.variant` (the served variation key) whenever that key is non-empty; graph spans can have an empty -one, and then it is left off. It also carries `feature_flag.result.value`: -`JSON.stringify` of the raw variation the SDK returned (messages or +one, and then it is left off. It also carries `feature_flag.result.value`, +which follows content capture. With capture on and a raw variation present, +it is `JSON.stringify` of the raw variation the SDK returned (messages or instructions, model, provider, `_ldMeta`, judge configuration), the same string the LaunchDarkly OTel `TracingHook` writes with `includeValue: true`. -The core client passes that raw value to handlers as `variables.__ldValue`, -beside `__ld`, because `TrackData` is the `$ld:ai:*` event payload. It is not -gated by content capture, and it is left off when there is no evaluated -value, such as on native graph spans. +Otherwise it is `JSON.stringify` of the variation key (for example +`"\"opus\""`), and it is left off when there is neither. Every handler passes +its own `captureContent` as the third argument to `setLdSpanAttributes`; the +default is `false`. The core client passes the raw value to handlers as +`variables.__ldValue`, beside `__ld`, because `TrackData` is the `$ld:ai:*` +event payload. When `variables.ldContext` has a usable identity, that event also carries `feature_flag.context.id` and `feature_flag.contextKeys`, and the root span gets `context.contextKeys.`. Child spans must not. diff --git a/packages/claude-agents/src/__tests__/handler.test.ts b/packages/claude-agents/src/__tests__/handler.test.ts index 5381a50d..396a6f93 100644 --- a/packages/claude-agents/src/__tests__/handler.test.ts +++ b/packages/claude-agents/src/__tests__/handler.test.ts @@ -543,7 +543,7 @@ describe('createClaudeAgentsHandler', () => { 'feature_flag.provider.name': 'LaunchDarkly', 'feature_flag.set.id': 'env-abc', 'feature_flag.result.variant': 'variation-a', - 'feature_flag.result.value': JSON.stringify(ldFixture.__ldValue), + 'feature_flag.result.value': JSON.stringify('variation-a'), }); expect(mockSpan.setAttribute).toHaveBeenCalledWith('launchdarkly.operation.type', 'gen_ai'); expect(mockSpan.setAttribute).toHaveBeenCalledWith('launchdarkly.config.key', 'test-config'); diff --git a/packages/claude-agents/src/handler.ts b/packages/claude-agents/src/handler.ts index b678e849..e7ae9743 100644 --- a/packages/claude-agents/src/handler.ts +++ b/packages/claude-agents/src/handler.ts @@ -1055,7 +1055,7 @@ export function createClaudeAgentsHandlerInternal( return trace.getTracer('@launchdarkly/ai-claude-agents').startActiveSpan('invoke_agent', async (span) => { span.setAttribute('gen_ai.operation.name', 'invoke_agent'); setModelIdentityAttributes(span, 'anthropic', config.model.name); - setLdSpanAttributes(span, variables); + setLdSpanAttributes(span, variables, captureContent); // Explicit rather than a bare `context.active()`: the active context only carries this // span while an OTel ContextManager is registered, so a host app that installs its own // TracerProvider without one would otherwise get a flat trace. @@ -1191,7 +1191,7 @@ export function createClaudeAgentsHandlerInternal( const span = trace.getTracer('@launchdarkly/ai-claude-agents').startSpan('invoke_agent'); span.setAttribute('gen_ai.operation.name', 'invoke_agent'); setModelIdentityAttributes(span, 'anthropic', config.model.name); - setLdSpanAttributes(span, variables); + setLdSpanAttributes(span, variables, captureContent); const parentContext = trace.setSpan(context.active(), span); // A consumer that `break`s out of `for await`, or throws inside the loop body, makes this diff --git a/packages/claude-messages/src/handler.ts b/packages/claude-messages/src/handler.ts index fae0ac67..8c2c3fe0 100644 --- a/packages/claude-messages/src/handler.ts +++ b/packages/claude-messages/src/handler.ts @@ -604,7 +604,7 @@ function createClaudeMessagesHandlerInternal({ captureContent = false }: Content return trace.getTracer(TRACER_NAME).startActiveSpan('invoke_agent', async (span) => { span.setAttribute('gen_ai.operation.name', 'invoke_agent'); setModelIdentityAttributes(span, 'anthropic', config.model.name); - setLdSpanAttributes(span, variables); + setLdSpanAttributes(span, variables, captureContent); // Explicit rather than a bare `context.active()`: the active context only carries this // span while an OTel ContextManager is registered, so a host app that installs its own // TracerProvider without one would otherwise get a flat trace. @@ -646,7 +646,7 @@ function createClaudeMessagesHandlerInternal({ captureContent = false }: Content const span = trace.getTracer(TRACER_NAME).startSpan('invoke_agent'); span.setAttribute('gen_ai.operation.name', 'invoke_agent'); setModelIdentityAttributes(span, 'anthropic', config.model.name); - setLdSpanAttributes(span, variables); + setLdSpanAttributes(span, variables, captureContent); const parentContext = trace.setSpan(context.active(), span); // A consumer that `break`s out of `for await`, or throws inside the loop body, makes this diff --git a/packages/client/etc/ai-server.api.md b/packages/client/etc/ai-server.api.md index 71180621..ae64606a 100644 --- a/packages/client/etc/ai-server.api.md +++ b/packages/client/etc/ai-server.api.md @@ -711,7 +711,7 @@ export function setInputContentAttributes(span: Span, capture: boolean, input: { }): void; // @public -export function setLdSpanAttributes(span: Span, variables: Record | undefined): void; +export function setLdSpanAttributes(span: Span, variables: Record | undefined, captureContent?: boolean): void; // @public export function setModelIdentityAttributes(span: Span, providerName: string, requestModel: string, legacySystem?: string): void; diff --git a/packages/client/src/__tests__/utils.test.ts b/packages/client/src/__tests__/utils.test.ts index ca667e9d..3d2d78a5 100644 --- a/packages/client/src/__tests__/utils.test.ts +++ b/packages/client/src/__tests__/utils.test.ts @@ -397,22 +397,41 @@ describe('setLdSpanAttributes', () => { expect(eventCall?.[1]).not.toHaveProperty('feature_flag.result.variant'); }); - it('includes feature_flag.result.value as the JSON of the raw evaluated variation', () => { - const span = makeMockSpan(); - const raw = { - model: { name: 'gpt-x', parameters: { temperature: 0.2 } }, - provider: { name: 'openai' }, - messages: [{ role: 'system', content: 'Be brief.' }], - _ldMeta: { enabled: true, variationKey: 'var-a', version: 3 }, - }; - setLdSpanAttributes(span as any, { __ld: ldFixture, __ldValue: raw }); + const rawVariation = { + model: { name: 'gpt-x', parameters: { temperature: 0.2 } }, + provider: { name: 'openai' }, + messages: [{ role: 'system', content: 'Be brief.' }], + _ldMeta: { enabled: true, variationKey: 'var-a', version: 3 }, + }; + + it('writes the raw variation JSON as feature_flag.result.value when content capture is on', () => { + const span = makeMockSpan(); + setLdSpanAttributes(span as any, { __ld: { ...ldFixture, variationKey: 'var-a' }, __ldValue: rawVariation }, true); // Same serialization as the LaunchDarkly OTel TracingHook with includeValue: JSON.stringify(detail.value). - expect(featureFlagAttrs(span)['feature_flag.result.value']).toBe(JSON.stringify(raw)); + expect(featureFlagAttrs(span)['feature_flag.result.value']).toBe(JSON.stringify(rawVariation)); }); - it('omits feature_flag.result.value when there is no evaluated value (graph spans)', () => { + it('writes the variation key JSON as feature_flag.result.value when content capture is off', () => { const span = makeMockSpan(); - setLdSpanAttributes(span as any, { __ld: { ...ldFixture, variationKey: '', graphKey: 'my-graph' } }); + setLdSpanAttributes(span as any, { __ld: { ...ldFixture, variationKey: 'opus' }, __ldValue: rawVariation }, false); + expect(featureFlagAttrs(span)['feature_flag.result.value']).toBe('"opus"'); + }); + + it('defaults content capture to off, so the raw variation is not written', () => { + const span = makeMockSpan(); + setLdSpanAttributes(span as any, { __ld: { ...ldFixture, variationKey: 'opus' }, __ldValue: rawVariation }); + expect(featureFlagAttrs(span)['feature_flag.result.value']).toBe('"opus"'); + }); + + it('writes the variation key JSON when capture is on but there is no raw variation', () => { + const span = makeMockSpan(); + setLdSpanAttributes(span as any, { __ld: { ...ldFixture, variationKey: 'opus' } }, true); + expect(featureFlagAttrs(span)['feature_flag.result.value']).toBe('"opus"'); + }); + + it('omits feature_flag.result.value when there is no variation key and no raw variation (graph spans)', () => { + const span = makeMockSpan(); + setLdSpanAttributes(span as any, { __ld: { ...ldFixture, variationKey: '', graphKey: 'my-graph' } }, true); expect(featureFlagAttrs(span)).not.toHaveProperty('feature_flag.result.value'); }); }); diff --git a/packages/client/src/utils.ts b/packages/client/src/utils.ts index 7fe975e2..b133518c 100644 --- a/packages/client/src/utils.ts +++ b/packages/client/src/utils.ts @@ -476,16 +476,26 @@ function contextIdentityFromLdContext(ldContext: unknown): ContextIdentity | und * feature_flag.contextKeys * feature_flag.result.variant is the served variationKey, omitted when empty * (graph spans can carry an empty variationKey). - * feature_flag.result.value is `JSON.stringify` of `variables.__ldValue`, the - * raw variation the SDK returned (as the OTel TracingHook writes it with - * `includeValue: true`). Omitted when absent, e.g. on native graph spans. - * Not gated by content capture. + * feature_flag.result.value follows content capture. With `captureContent` + * on and `variables.__ldValue` present, it is `JSON.stringify` of that raw + * variation (as the OTel TracingHook writes it with `includeValue: true`). + * Otherwise it is `JSON.stringify(variationKey)` when the key is non-empty, + * so the prompt never leaves the process unless capture is on. Omitted when + * there is neither. + * + * @param captureContent - The calling handler's content-capture setting. + * Defaults to `false`, so a caller that does not pass it never writes the + * raw variation. * The observability backend projects these event attrs to span-level attrs, * satisfying the query: events.name=feature_flag AND * events.attributes.feature_flag.key= AND * feature_flag.set.id= */ -export function setLdSpanAttributes(span: Span, variables: Record | undefined): void { +export function setLdSpanAttributes( + span: Span, + variables: Record | undefined, + captureContent = false, +): void { span.setAttribute('launchdarkly.operation.type', 'gen_ai'); const ld = variables?.__ld as TrackData | undefined; if (!ld) return; @@ -505,12 +515,14 @@ export function setLdSpanAttributes(span: Span, variables: Record { 'feature_flag.provider.name': 'LaunchDarkly', 'feature_flag.set.id': 'env-abc', 'feature_flag.result.variant': 'variation-a', - 'feature_flag.result.value': JSON.stringify(ldFixture.__ldValue), + 'feature_flag.result.value': JSON.stringify('variation-a'), }); expect(mockSpan.setAttribute).toHaveBeenCalledWith('launchdarkly.operation.type', 'gen_ai'); expect(mockSpan.setAttribute).toHaveBeenCalledWith('launchdarkly.config.key', 'test-config'); diff --git a/packages/langchain-agents/src/handler.ts b/packages/langchain-agents/src/handler.ts index 91d33428..629af555 100644 --- a/packages/langchain-agents/src/handler.ts +++ b/packages/langchain-agents/src/handler.ts @@ -409,7 +409,7 @@ export function createLangChainAgentsHandlerInternal( return trace.getTracer(TRACER_NAME).startActiveSpan('invoke_agent', async (span) => { span.setAttribute('gen_ai.operation.name', 'invoke_agent'); setModelIdentityAttributes(span, servingProvider(config), config.model.name, 'langchain'); - setLdSpanAttributes(span, variables); + setLdSpanAttributes(span, variables, captureContent); // Explicit rather than a bare `context.active()`: the active context only carries this // span while an OTel ContextManager is registered, so a host app that installs its own // TracerProvider without one would otherwise get a flat trace. @@ -482,7 +482,7 @@ export function createLangChainAgentsHandlerInternal( const span = trace.getTracer(TRACER_NAME).startSpan('invoke_agent'); span.setAttribute('gen_ai.operation.name', 'invoke_agent'); setModelIdentityAttributes(span, servingProvider(config), config.model.name, 'langchain'); - setLdSpanAttributes(span, variables); + setLdSpanAttributes(span, variables, captureContent); const parentContext = trace.setSpan(context.active(), span); // A consumer that `break`s out of `for await`, or throws inside the loop body, makes this diff --git a/packages/langchain-messages/src/__tests__/handler.test.ts b/packages/langchain-messages/src/__tests__/handler.test.ts index 52fea7a3..4d9249ab 100644 --- a/packages/langchain-messages/src/__tests__/handler.test.ts +++ b/packages/langchain-messages/src/__tests__/handler.test.ts @@ -593,7 +593,7 @@ describe('createLangChainMessagesHandler', () => { 'feature_flag.provider.name': 'LaunchDarkly', 'feature_flag.set.id': 'env-abc', 'feature_flag.result.variant': 'variation-a', - 'feature_flag.result.value': JSON.stringify(ldFixture.__ldValue), + 'feature_flag.result.value': JSON.stringify('variation-a'), }); expect(mockSpan.setAttribute).toHaveBeenCalledWith('launchdarkly.operation.type', 'gen_ai'); expect(mockSpan.setAttribute).toHaveBeenCalledWith('launchdarkly.config.key', 'test-config'); diff --git a/packages/langchain-messages/src/handler.ts b/packages/langchain-messages/src/handler.ts index e9334a39..18e73b51 100644 --- a/packages/langchain-messages/src/handler.ts +++ b/packages/langchain-messages/src/handler.ts @@ -355,7 +355,7 @@ function createLangChainMessagesHandlerInternal( return trace.getTracer(TRACER_NAME).startActiveSpan('invoke_agent', async (span) => { span.setAttribute('gen_ai.operation.name', 'invoke_agent'); setModelIdentityAttributes(span, servingProvider(config), config.model.name, 'langchain'); - setLdSpanAttributes(span, variables); + setLdSpanAttributes(span, variables, captureContent); // Explicit rather than a bare `context.active()`: the active context only carries this // span while an OTel ContextManager is registered, so a host app that installs its own // TracerProvider without one would otherwise get a flat trace. @@ -533,7 +533,7 @@ function createLangChainMessagesHandlerInternal( const span = trace.getTracer(TRACER_NAME).startSpan('invoke_agent'); span.setAttribute('gen_ai.operation.name', 'invoke_agent'); setModelIdentityAttributes(span, servingProvider(config), config.model.name, 'langchain'); - setLdSpanAttributes(span, variables); + setLdSpanAttributes(span, variables, captureContent); const parentContext = trace.setSpan(context.active(), span); // A consumer that `break`s out of `for await`, or throws inside the loop body, makes this diff --git a/packages/litellm-agents/src/handler.ts b/packages/litellm-agents/src/handler.ts index 7d9dffaf..6270c3be 100644 --- a/packages/litellm-agents/src/handler.ts +++ b/packages/litellm-agents/src/handler.ts @@ -411,7 +411,7 @@ export function createLiteLLMAgentsHandlerInternal(options: LiteLLMAgentsOptions trace.getTracer(TRACER_NAME).startActiveSpan('invoke_agent', async (span) => { span.setAttribute('gen_ai.operation.name', 'invoke_agent'); setIdentity(span, configRep); - setLdSpanAttributes(span, variables); + setLdSpanAttributes(span, variables, captureContent); const parentContext = trace.setSpan(context.active(), span); const client = resolveLiteLLMClient(resolvedOptions, configRep); const { agent, model, instructions, runOptions } = buildAgent( @@ -461,7 +461,7 @@ export function createLiteLLMAgentsHandlerInternal(options: LiteLLMAgentsOptions const span = trace.getTracer(TRACER_NAME).startSpan('invoke_agent'); span.setAttribute('gen_ai.operation.name', 'invoke_agent'); setIdentity(span, configRep); - setLdSpanAttributes(span, variables); + setLdSpanAttributes(span, variables, captureContent); const parentContext = trace.setSpan(context.active(), span); const client = resolveLiteLLMClient(resolvedOptions, configRep); const { agent, model, instructions, runOptions } = buildAgent( diff --git a/packages/litellm-messages/src/handler.ts b/packages/litellm-messages/src/handler.ts index e81cd4e9..e5493b25 100644 --- a/packages/litellm-messages/src/handler.ts +++ b/packages/litellm-messages/src/handler.ts @@ -287,7 +287,7 @@ export function createLiteLLMMessagesHandlerInternal(options: LiteLLMMessagesOpt trace.getTracer(TRACER_NAME).startActiveSpan('invoke_agent', async (span) => { span.setAttribute('gen_ai.operation.name', 'invoke_agent'); setIdentity(span, configRep); - setLdSpanAttributes(span, variables); + setLdSpanAttributes(span, variables, captureContent); const parentContext = trace.setSpan(context.active(), span); const messages = buildMessages(configRep, userInput, variables, history); setInputContentAttributes(span, captureContent, { messages: spanMessages(messages) }); @@ -383,7 +383,7 @@ export function createLiteLLMMessagesHandlerInternal(options: LiteLLMMessagesOpt const span = trace.getTracer(TRACER_NAME).startSpan('invoke_agent'); span.setAttribute('gen_ai.operation.name', 'invoke_agent'); setIdentity(span, configRep); - setLdSpanAttributes(span, variables); + setLdSpanAttributes(span, variables, captureContent); const parentContext = trace.setSpan(context.active(), span); const messages = buildMessages(configRep, userInput, variables, history); setInputContentAttributes(span, captureContent, { messages: spanMessages(messages) }); diff --git a/packages/openai-agents/src/__tests__/handler.test.ts b/packages/openai-agents/src/__tests__/handler.test.ts index 02beb596..21f02a2a 100644 --- a/packages/openai-agents/src/__tests__/handler.test.ts +++ b/packages/openai-agents/src/__tests__/handler.test.ts @@ -619,7 +619,7 @@ describe('createOpenAIAgentsHandler', () => { 'feature_flag.provider.name': 'LaunchDarkly', 'feature_flag.set.id': 'env-abc', 'feature_flag.result.variant': 'variation-a', - 'feature_flag.result.value': JSON.stringify(ldFixture.__ldValue), + 'feature_flag.result.value': JSON.stringify('variation-a'), }); expect(mockSpan.setAttribute).toHaveBeenCalledWith('launchdarkly.operation.type', 'gen_ai'); expect(mockSpan.setAttribute).toHaveBeenCalledWith('launchdarkly.config.key', 'test-config'); diff --git a/packages/openai-agents/src/handler.ts b/packages/openai-agents/src/handler.ts index 1a7888a7..bb126a02 100644 --- a/packages/openai-agents/src/handler.ts +++ b/packages/openai-agents/src/handler.ts @@ -619,7 +619,7 @@ export function createOpenAIAgentsHandlerInternal({ return trace.getTracer(TRACER_NAME).startActiveSpan('invoke_agent', async (span) => { span.setAttribute('gen_ai.operation.name', 'invoke_agent'); setModelIdentityAttributes(span, 'openai', config.model.name); - setLdSpanAttributes(span, variables); + setLdSpanAttributes(span, variables, captureContent); // Explicit rather than a bare `context.active()`: the active context only carries this // span while an OTel ContextManager is registered, so a host app that installs its own // TracerProvider without one would otherwise get a flat trace. @@ -678,7 +678,7 @@ export function createOpenAIAgentsHandlerInternal({ const span = trace.getTracer(TRACER_NAME).startSpan('invoke_agent'); span.setAttribute('gen_ai.operation.name', 'invoke_agent'); setModelIdentityAttributes(span, 'openai', config.model.name); - setLdSpanAttributes(span, variables); + setLdSpanAttributes(span, variables, captureContent); const parentContext = trace.setSpan(context.active(), span); // A consumer that `break`s out of `for await`, or throws inside the loop body, makes this diff --git a/packages/openai-messages/src/handler.ts b/packages/openai-messages/src/handler.ts index 577acbae..e0fe7116 100644 --- a/packages/openai-messages/src/handler.ts +++ b/packages/openai-messages/src/handler.ts @@ -506,7 +506,7 @@ function createOpenAIMessagesHandlerInternal({ captureContent = false }: Content return trace.getTracer(TRACER_NAME).startActiveSpan('invoke_agent', async (span) => { span.setAttribute('gen_ai.operation.name', 'invoke_agent'); setModelIdentityAttributes(span, 'openai', config.model.name); - setLdSpanAttributes(span, variables); + setLdSpanAttributes(span, variables, captureContent); // Explicit rather than a bare `context.active()`: the active context only carries this // span while an OTel ContextManager is registered, so a host app that installs its own // TracerProvider without one would otherwise get a flat trace. @@ -652,7 +652,7 @@ function createOpenAIMessagesHandlerInternal({ captureContent = false }: Content const span = trace.getTracer(TRACER_NAME).startSpan('invoke_agent'); span.setAttribute('gen_ai.operation.name', 'invoke_agent'); setModelIdentityAttributes(span, 'openai', config.model.name); - setLdSpanAttributes(span, variables); + setLdSpanAttributes(span, variables, captureContent); const parentContext = trace.setSpan(context.active(), span); // A consumer that `break`s out of `for await`, or throws inside the loop body, makes this diff --git a/packages/typesafe/src/handler.ts b/packages/typesafe/src/handler.ts index ba327c42..9e85c889 100644 --- a/packages/typesafe/src/handler.ts +++ b/packages/typesafe/src/handler.ts @@ -84,7 +84,7 @@ export function createTypesafeHandler(options: TypesafeHandlerOptions = {}): Pro const modelName = config.model?.name || 'jev'; const root = tracer.startSpan('invoke_agent'); const chat = tracer.startSpan(`chat ${modelName}`, undefined, trace.setSpan(context.active(), root)); - setLdSpanAttributes(root, variables); + setLdSpanAttributes(root, variables, captureContent); setModelIdentityAttributes(root, 'typesafe', modelName); setModelIdentityAttributes(chat, 'typesafe', modelName); if (captureContent) { diff --git a/packages/vercel-agents/src/handler.ts b/packages/vercel-agents/src/handler.ts index a1fc2830..9c3c0008 100644 --- a/packages/vercel-agents/src/handler.ts +++ b/packages/vercel-agents/src/handler.ts @@ -276,7 +276,7 @@ export function createVercelAgentsHandlerInternal(options: VercelAgentsOptions = trace.getTracer(TRACER_NAME).startActiveSpan('invoke_agent', async (rootSpan) => { rootSpan.setAttribute('gen_ai.operation.name', 'invoke_agent'); setModelIdentityAttributes(rootSpan, servingProvider(configRep), configRep.model.name); - setLdSpanAttributes(rootSpan, variables); + setLdSpanAttributes(rootSpan, variables, captureContent); let chatSpan: Span | undefined; try { const messages = buildMessages(configRep, userInput, variables, history); @@ -318,7 +318,7 @@ export function createVercelAgentsHandlerInternal(options: VercelAgentsOptions = const rootSpan = trace.getTracer(TRACER_NAME).startSpan('invoke_agent'); rootSpan.setAttribute('gen_ai.operation.name', 'invoke_agent'); setModelIdentityAttributes(rootSpan, servingProvider(configRep), configRep.model.name); - setLdSpanAttributes(rootSpan, variables); + setLdSpanAttributes(rootSpan, variables, captureContent); let chatSpan: Span | undefined; let iterator: AsyncIterator | undefined; let completed = false; diff --git a/packages/vercel-messages/src/evaluate.ts b/packages/vercel-messages/src/evaluate.ts index 7b6ddcab..3f138a91 100644 --- a/packages/vercel-messages/src/evaluate.ts +++ b/packages/vercel-messages/src/evaluate.ts @@ -110,7 +110,7 @@ export async function vercelEvaluate { span.setAttribute('gen_ai.operation.name', 'evaluate'); setModelIdentityAttributes(span, servingProvider(config), config.model.name); - setLdSpanAttributes(span, { __ld: trackData, ldContext: { ...context } }); + setLdSpanAttributes(span, { __ld: trackData, ldContext: { ...context } }, captureContent); const payload = JSON.stringify({ state, questions: options.questions }); setInputContentAttributes(span, captureContent, { messages: [textMessage('user', payload)], diff --git a/packages/vercel-messages/src/handler.ts b/packages/vercel-messages/src/handler.ts index 9d7891f3..47222049 100644 --- a/packages/vercel-messages/src/handler.ts +++ b/packages/vercel-messages/src/handler.ts @@ -266,7 +266,7 @@ function createVercelMessagesHandlerInternal(options: VercelMessagesOptions = {} trace.getTracer(TRACER_NAME).startActiveSpan('invoke_agent', async (rootSpan) => { rootSpan.setAttribute('gen_ai.operation.name', 'invoke_agent'); setModelIdentityAttributes(rootSpan, servingProvider(configRep), configRep.model.name); - setLdSpanAttributes(rootSpan, variables); + setLdSpanAttributes(rootSpan, variables, captureContent); let chatSpan: Span | undefined; try { const system = systemInstructions(configRep, variables); @@ -316,7 +316,7 @@ function createVercelMessagesHandlerInternal(options: VercelMessagesOptions = {} const rootSpan = trace.getTracer(TRACER_NAME).startSpan('invoke_agent'); rootSpan.setAttribute('gen_ai.operation.name', 'invoke_agent'); setModelIdentityAttributes(rootSpan, servingProvider(configRep), configRep.model.name); - setLdSpanAttributes(rootSpan, variables); + setLdSpanAttributes(rootSpan, variables, captureContent); let chatSpan: Span | undefined; let chatEnded = false; let rootEnded = false; From d2ad455fb3687d397c4e4131fc2ed75ac5fa8415 Mon Sep 17 00:00:00 2001 From: Alexis Georges Date: Wed, 28 Oct 2026 16:09:54 -0400 Subject: [PATCH 04/11] refactor(client): gate the evaluated variation in the core Move the content-capture gate for feature_flag.result.value out of the handlers and into executeAndTrack/executeAndStream. The core now sets variables.__ldValue only when the handler that runs has captureContent on (ProviderHandler.captureContent, the same gate judge reasoning uses), so the raw config never reaches a capture-off handler. Config runs, judges and graph nodes all go through these two functions. setLdSpanAttributes goes back to its original signature, and the handler call sites and the API report match main again. The helper writes the raw variation's JSON when __ldValue is present, the JSON-encoded variation key otherwise, and nothing when there is neither. --- AGENTS.md | 21 ++++----- .../src/__tests__/handler.test.ts | 1 - packages/claude-agents/src/handler.ts | 4 +- packages/claude-messages/src/handler.ts | 4 +- packages/client/etc/ai-server.api.md | 2 +- .../client/src/__tests__/tracking.test.ts | 46 ++++++++++++++++++- packages/client/src/__tests__/utils.test.ts | 22 ++------- packages/client/src/tracking.ts | 10 ++-- packages/client/src/utils.ts | 28 ++++------- .../src/__tests__/handler.test.ts | 1 - packages/langchain-agents/src/handler.ts | 4 +- .../src/__tests__/handler.test.ts | 1 - packages/langchain-messages/src/handler.ts | 4 +- packages/litellm-agents/src/handler.ts | 4 +- packages/litellm-messages/src/handler.ts | 4 +- .../src/__tests__/handler.test.ts | 1 - packages/openai-agents/src/handler.ts | 4 +- packages/openai-messages/src/handler.ts | 4 +- packages/typesafe/src/handler.ts | 2 +- packages/vercel-agents/src/handler.ts | 4 +- packages/vercel-messages/src/evaluate.ts | 2 +- packages/vercel-messages/src/handler.ts | 4 +- 22 files changed, 99 insertions(+), 78 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index a7f08d9b..16899d80 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -632,17 +632,16 @@ events these handlers used to emit were read by nothing on the LaunchDarkly side. The only event a handler emits is `feature_flag`, on the root, for trace correlation. It carries `feature_flag.result.variant` (the served variation key) whenever that key is non-empty; graph spans can have an empty -one, and then it is left off. It also carries `feature_flag.result.value`, -which follows content capture. With capture on and a raw variation present, -it is `JSON.stringify` of the raw variation the SDK returned (messages or -instructions, model, provider, `_ldMeta`, judge configuration), the same -string the LaunchDarkly OTel `TracingHook` writes with `includeValue: true`. -Otherwise it is `JSON.stringify` of the variation key (for example -`"\"opus\""`), and it is left off when there is neither. Every handler passes -its own `captureContent` as the third argument to `setLdSpanAttributes`; the -default is `false`. The core client passes the raw value to handlers as -`variables.__ldValue`, beside `__ld`, because `TrackData` is the `$ld:ai:*` -event payload. +one, and then it is left off. It also carries `feature_flag.result.value`. +When `variables.__ldValue` is present, it is `JSON.stringify` of that raw +variation (messages or instructions, model, provider, `_ldMeta`, judge +configuration), the same string the LaunchDarkly OTel `TracingHook` writes +with `includeValue: true`. Otherwise it is `JSON.stringify` of the variation +key (for example `"\"opus\""`), and it is left off when there is neither. +The core client sets `__ldValue`, beside `__ld`, only when the handler that +runs has `captureContent` on (`ProviderHandler.captureContent`, the same gate +judge reasoning uses), so the raw config never reaches a capture-off handler. +It is not in `TrackData`, because `TrackData` is the `$ld:ai:*` event payload. When `variables.ldContext` has a usable identity, that event also carries `feature_flag.context.id` and `feature_flag.contextKeys`, and the root span gets `context.contextKeys.`. Child spans must not. diff --git a/packages/claude-agents/src/__tests__/handler.test.ts b/packages/claude-agents/src/__tests__/handler.test.ts index 396a6f93..c797195e 100644 --- a/packages/claude-agents/src/__tests__/handler.test.ts +++ b/packages/claude-agents/src/__tests__/handler.test.ts @@ -531,7 +531,6 @@ describe('createClaudeAgentsHandler', () => { modelName: 'test-model', providerName: 'TestProvider', }, - __ldValue: { model: { name: 'test-model' }, _ldMeta: { enabled: true, variationKey: 'variation-a' } }, }; it('emits the feature_flag correlation event on the invoke_agent root', async () => { diff --git a/packages/claude-agents/src/handler.ts b/packages/claude-agents/src/handler.ts index e7ae9743..b678e849 100644 --- a/packages/claude-agents/src/handler.ts +++ b/packages/claude-agents/src/handler.ts @@ -1055,7 +1055,7 @@ export function createClaudeAgentsHandlerInternal( return trace.getTracer('@launchdarkly/ai-claude-agents').startActiveSpan('invoke_agent', async (span) => { span.setAttribute('gen_ai.operation.name', 'invoke_agent'); setModelIdentityAttributes(span, 'anthropic', config.model.name); - setLdSpanAttributes(span, variables, captureContent); + setLdSpanAttributes(span, variables); // Explicit rather than a bare `context.active()`: the active context only carries this // span while an OTel ContextManager is registered, so a host app that installs its own // TracerProvider without one would otherwise get a flat trace. @@ -1191,7 +1191,7 @@ export function createClaudeAgentsHandlerInternal( const span = trace.getTracer('@launchdarkly/ai-claude-agents').startSpan('invoke_agent'); span.setAttribute('gen_ai.operation.name', 'invoke_agent'); setModelIdentityAttributes(span, 'anthropic', config.model.name); - setLdSpanAttributes(span, variables, captureContent); + setLdSpanAttributes(span, variables); const parentContext = trace.setSpan(context.active(), span); // A consumer that `break`s out of `for await`, or throws inside the loop body, makes this diff --git a/packages/claude-messages/src/handler.ts b/packages/claude-messages/src/handler.ts index 8c2c3fe0..fae0ac67 100644 --- a/packages/claude-messages/src/handler.ts +++ b/packages/claude-messages/src/handler.ts @@ -604,7 +604,7 @@ function createClaudeMessagesHandlerInternal({ captureContent = false }: Content return trace.getTracer(TRACER_NAME).startActiveSpan('invoke_agent', async (span) => { span.setAttribute('gen_ai.operation.name', 'invoke_agent'); setModelIdentityAttributes(span, 'anthropic', config.model.name); - setLdSpanAttributes(span, variables, captureContent); + setLdSpanAttributes(span, variables); // Explicit rather than a bare `context.active()`: the active context only carries this // span while an OTel ContextManager is registered, so a host app that installs its own // TracerProvider without one would otherwise get a flat trace. @@ -646,7 +646,7 @@ function createClaudeMessagesHandlerInternal({ captureContent = false }: Content const span = trace.getTracer(TRACER_NAME).startSpan('invoke_agent'); span.setAttribute('gen_ai.operation.name', 'invoke_agent'); setModelIdentityAttributes(span, 'anthropic', config.model.name); - setLdSpanAttributes(span, variables, captureContent); + setLdSpanAttributes(span, variables); const parentContext = trace.setSpan(context.active(), span); // A consumer that `break`s out of `for await`, or throws inside the loop body, makes this diff --git a/packages/client/etc/ai-server.api.md b/packages/client/etc/ai-server.api.md index ae64606a..71180621 100644 --- a/packages/client/etc/ai-server.api.md +++ b/packages/client/etc/ai-server.api.md @@ -711,7 +711,7 @@ export function setInputContentAttributes(span: Span, capture: boolean, input: { }): void; // @public -export function setLdSpanAttributes(span: Span, variables: Record | undefined, captureContent?: boolean): void; +export function setLdSpanAttributes(span: Span, variables: Record | undefined): void; // @public export function setModelIdentityAttributes(span: Span, providerName: string, requestModel: string, legacySystem?: string): void; diff --git a/packages/client/src/__tests__/tracking.test.ts b/packages/client/src/__tests__/tracking.test.ts index bf95bc0d..a5a905c8 100644 --- a/packages/client/src/__tests__/tracking.test.ts +++ b/packages/client/src/__tests__/tracking.test.ts @@ -149,8 +149,9 @@ describe('executeAndTrack', () => { expect(result.response).toBe('Hello!'); }); - it('passes evaluatedValue to the handler as __ldValue and keeps it out of trackData and events', async () => { + it('passes evaluatedValue as __ldValue to a capture-on handler and keeps it out of trackData and events', async () => { const handler = makeHandler('ok'); + handler.captureContent = true; const raw = { ...execConfig, _ldMeta: { enabled: true, variationKey: 'v1' } }; const result = await executeAndTrack({ configKey: 'my-flag', @@ -165,8 +166,26 @@ describe('executeAndTrack', () => { for (const call of mockTrack.mock.calls) expect(JSON.stringify(call[2])).not.toContain('You are helpful.'); }); + it.each([ + ['capture off', false], + ['capture unset', undefined], + ])('does not pass __ldValue to a handler with %s', async (_label, capture) => { + const handler = makeHandler('ok'); + if (capture !== undefined) handler.captureContent = capture; + await executeAndTrack({ + configKey: 'my-flag', + config: execConfig as any, + meta: execMeta, + userContext: execContext, + handler, + evaluatedValue: { ...execConfig }, + }); + expect(handler.mock.calls[0][3]).not.toHaveProperty('__ldValue'); + }); + it('does not set __ldValue when evaluatedValue is absent', async () => { const handler = makeHandler('ok'); + handler.captureContent = true; await executeAndTrack({ configKey: 'my-flag', config: execConfig as any, @@ -353,6 +372,31 @@ describe('executeAndStream', () => { return fn; } + it.each([ + [true, true], + [false, false], + ])('passes __ldValue to the stream only when captureContent is %s', async (capture, expected) => { + const handler = makeStreamingHandler(['Hi']); + const seen: Record[] = []; + const inner = handler.stream; + handler.stream = (c: unknown, u: unknown, t: unknown, v: Record, h: unknown) => { + seen.push(v); + return inner(c, u, t, v, h); + }; + handler.captureContent = capture; + await collectExecStream( + executeAndStream({ + configKey: 'my-flag', + config: execConfig as any, + meta: execMeta, + userContext: execContext, + handler, + evaluatedValue: { ...execConfig }, + }), + ); + expect('__ldValue' in seen[0]).toBe(expected); + }); + it('copies modelKey and modelVersion from meta onto every streamed event payload', async () => { const handler = makeStreamingHandler(['Hi']); await collectExecStream( diff --git a/packages/client/src/__tests__/utils.test.ts b/packages/client/src/__tests__/utils.test.ts index 3d2d78a5..674a36af 100644 --- a/packages/client/src/__tests__/utils.test.ts +++ b/packages/client/src/__tests__/utils.test.ts @@ -404,34 +404,22 @@ describe('setLdSpanAttributes', () => { _ldMeta: { enabled: true, variationKey: 'var-a', version: 3 }, }; - it('writes the raw variation JSON as feature_flag.result.value when content capture is on', () => { + it('writes the raw variation JSON as feature_flag.result.value when __ldValue is present', () => { const span = makeMockSpan(); - setLdSpanAttributes(span as any, { __ld: { ...ldFixture, variationKey: 'var-a' }, __ldValue: rawVariation }, true); + setLdSpanAttributes(span as any, { __ld: { ...ldFixture, variationKey: 'var-a' }, __ldValue: rawVariation }); // Same serialization as the LaunchDarkly OTel TracingHook with includeValue: JSON.stringify(detail.value). expect(featureFlagAttrs(span)['feature_flag.result.value']).toBe(JSON.stringify(rawVariation)); }); - it('writes the variation key JSON as feature_flag.result.value when content capture is off', () => { + it('writes the variation key JSON as feature_flag.result.value when there is no raw variation', () => { const span = makeMockSpan(); - setLdSpanAttributes(span as any, { __ld: { ...ldFixture, variationKey: 'opus' }, __ldValue: rawVariation }, false); - expect(featureFlagAttrs(span)['feature_flag.result.value']).toBe('"opus"'); - }); - - it('defaults content capture to off, so the raw variation is not written', () => { - const span = makeMockSpan(); - setLdSpanAttributes(span as any, { __ld: { ...ldFixture, variationKey: 'opus' }, __ldValue: rawVariation }); - expect(featureFlagAttrs(span)['feature_flag.result.value']).toBe('"opus"'); - }); - - it('writes the variation key JSON when capture is on but there is no raw variation', () => { - const span = makeMockSpan(); - setLdSpanAttributes(span as any, { __ld: { ...ldFixture, variationKey: 'opus' } }, true); + setLdSpanAttributes(span as any, { __ld: { ...ldFixture, variationKey: 'opus' } }); expect(featureFlagAttrs(span)['feature_flag.result.value']).toBe('"opus"'); }); it('omits feature_flag.result.value when there is no variation key and no raw variation (graph spans)', () => { const span = makeMockSpan(); - setLdSpanAttributes(span as any, { __ld: { ...ldFixture, variationKey: '', graphKey: 'my-graph' } }, true); + setLdSpanAttributes(span as any, { __ld: { ...ldFixture, variationKey: '', graphKey: 'my-graph' } }); expect(featureFlagAttrs(span)).not.toHaveProperty('feature_flag.result.value'); }); }); diff --git a/packages/client/src/tracking.ts b/packages/client/src/tracking.ts index 9369fbfc..732e666c 100644 --- a/packages/client/src/tracking.ts +++ b/packages/client/src/tracking.ts @@ -239,7 +239,8 @@ export const executeAndTrack = async ({ /** * The flag value as the LaunchDarkly SDK returned it, before parsing. Passed to the handler as * `variables.__ldValue` (not in `TrackData`, which is the `$ld:ai:*` event payload) so - * `setLdSpanAttributes` can write `feature_flag.result.value`. Omit it when there is none. + * `setLdSpanAttributes` can write `feature_flag.result.value`, but only when the handler has + * `captureContent` on: the value holds the prompt. Omit it when there is none. */ evaluatedValue?: unknown; }): Promise<{ usage: TokenUsage; response: unknown; trackData: TrackData; trajectory: string }> => { @@ -259,7 +260,7 @@ export const executeAndTrack = async ({ ...variables, ldContext: { ...userContext }, __ld: trackData, - ...(evaluatedValue !== undefined ? { __ldValue: evaluatedValue } : {}), + ...(evaluatedValue !== undefined && handler.captureContent === true ? { __ldValue: evaluatedValue } : {}), }, history, ); @@ -325,7 +326,8 @@ export async function* executeAndStream({ /** * The flag value as the LaunchDarkly SDK returned it, before parsing. Passed to the handler as * `variables.__ldValue` (not in `TrackData`, which is the `$ld:ai:*` event payload) so - * `setLdSpanAttributes` can write `feature_flag.result.value`. Omit it when there is none. + * `setLdSpanAttributes` can write `feature_flag.result.value`, but only when the handler has + * `captureContent` on: the value holds the prompt. Omit it when there is none. */ evaluatedValue?: unknown; }): AsyncGenerator { @@ -336,7 +338,7 @@ export async function* executeAndStream({ ...variables, ldContext: { ...userContext }, __ld: trackData, - ...(evaluatedValue !== undefined ? { __ldValue: evaluatedValue } : {}), + ...(evaluatedValue !== undefined && handler.captureContent === true ? { __ldValue: evaluatedValue } : {}), }; const startTime = Date.now(); diff --git a/packages/client/src/utils.ts b/packages/client/src/utils.ts index b133518c..5da90009 100644 --- a/packages/client/src/utils.ts +++ b/packages/client/src/utils.ts @@ -476,26 +476,17 @@ function contextIdentityFromLdContext(ldContext: unknown): ContextIdentity | und * feature_flag.contextKeys * feature_flag.result.variant is the served variationKey, omitted when empty * (graph spans can carry an empty variationKey). - * feature_flag.result.value follows content capture. With `captureContent` - * on and `variables.__ldValue` present, it is `JSON.stringify` of that raw - * variation (as the OTel TracingHook writes it with `includeValue: true`). - * Otherwise it is `JSON.stringify(variationKey)` when the key is non-empty, - * so the prompt never leaves the process unless capture is on. Omitted when - * there is neither. - * - * @param captureContent - The calling handler's content-capture setting. - * Defaults to `false`, so a caller that does not pass it never writes the - * raw variation. + * feature_flag.result.value is `JSON.stringify` of `variables.__ldValue`, the + * raw variation (as the OTel TracingHook writes it with `includeValue: true`), + * when present. The core sets `__ldValue` only for handlers with + * `captureContent` on. Otherwise it is `JSON.stringify(variationKey)` when the + * key is non-empty, and omitted when there is neither. * The observability backend projects these event attrs to span-level attrs, * satisfying the query: events.name=feature_flag AND * events.attributes.feature_flag.key= AND * feature_flag.set.id= */ -export function setLdSpanAttributes( - span: Span, - variables: Record | undefined, - captureContent = false, -): void { +export function setLdSpanAttributes(span: Span, variables: Record | undefined): void { span.setAttribute('launchdarkly.operation.type', 'gen_ai'); const ld = variables?.__ld as TrackData | undefined; if (!ld) return; @@ -515,9 +506,10 @@ export function setLdSpanAttributes( featureFlagAttrs['feature_flag.result.variant'] = ld.variationKey; } // Same serialization as the LaunchDarkly OTel TracingHook with `includeValue: true`: - // `JSON.stringify(detail.value)`. The raw variation holds the prompt, so it is written only - // with content capture on; otherwise the variation key stands in, JSON-encoded like any flag value. - const raw = captureContent ? variables?.__ldValue : undefined; + // `JSON.stringify(detail.value)`. The raw variation holds the prompt, so the core passes it only + // to handlers with content capture on; otherwise the variation key stands in, JSON-encoded like + // any flag value. + const raw = variables?.__ldValue; const serializedRaw = raw !== undefined ? JSON.stringify(raw) : undefined; if (typeof serializedRaw === 'string') { featureFlagAttrs['feature_flag.result.value'] = serializedRaw; diff --git a/packages/langchain-agents/src/__tests__/handler.test.ts b/packages/langchain-agents/src/__tests__/handler.test.ts index fc071014..955129d8 100644 --- a/packages/langchain-agents/src/__tests__/handler.test.ts +++ b/packages/langchain-agents/src/__tests__/handler.test.ts @@ -350,7 +350,6 @@ describe('createLangChainAgentsHandler', () => { modelName: 'test-model', providerName: 'TestProvider', }, - __ldValue: { model: { name: 'test-model' }, _ldMeta: { enabled: true, variationKey: 'variation-a' } }, }; it('emits the feature_flag correlation event on the invoke_agent root', async () => { diff --git a/packages/langchain-agents/src/handler.ts b/packages/langchain-agents/src/handler.ts index 629af555..91d33428 100644 --- a/packages/langchain-agents/src/handler.ts +++ b/packages/langchain-agents/src/handler.ts @@ -409,7 +409,7 @@ export function createLangChainAgentsHandlerInternal( return trace.getTracer(TRACER_NAME).startActiveSpan('invoke_agent', async (span) => { span.setAttribute('gen_ai.operation.name', 'invoke_agent'); setModelIdentityAttributes(span, servingProvider(config), config.model.name, 'langchain'); - setLdSpanAttributes(span, variables, captureContent); + setLdSpanAttributes(span, variables); // Explicit rather than a bare `context.active()`: the active context only carries this // span while an OTel ContextManager is registered, so a host app that installs its own // TracerProvider without one would otherwise get a flat trace. @@ -482,7 +482,7 @@ export function createLangChainAgentsHandlerInternal( const span = trace.getTracer(TRACER_NAME).startSpan('invoke_agent'); span.setAttribute('gen_ai.operation.name', 'invoke_agent'); setModelIdentityAttributes(span, servingProvider(config), config.model.name, 'langchain'); - setLdSpanAttributes(span, variables, captureContent); + setLdSpanAttributes(span, variables); const parentContext = trace.setSpan(context.active(), span); // A consumer that `break`s out of `for await`, or throws inside the loop body, makes this diff --git a/packages/langchain-messages/src/__tests__/handler.test.ts b/packages/langchain-messages/src/__tests__/handler.test.ts index 4d9249ab..ebfc7d32 100644 --- a/packages/langchain-messages/src/__tests__/handler.test.ts +++ b/packages/langchain-messages/src/__tests__/handler.test.ts @@ -582,7 +582,6 @@ describe('createLangChainMessagesHandler', () => { modelName: 'test-model', providerName: 'TestProvider', }, - __ldValue: { model: { name: 'test-model' }, _ldMeta: { enabled: true, variationKey: 'variation-a' } }, }; it('emits the feature_flag correlation event and launchdarkly attributes on the root span', async () => { diff --git a/packages/langchain-messages/src/handler.ts b/packages/langchain-messages/src/handler.ts index 18e73b51..e9334a39 100644 --- a/packages/langchain-messages/src/handler.ts +++ b/packages/langchain-messages/src/handler.ts @@ -355,7 +355,7 @@ function createLangChainMessagesHandlerInternal( return trace.getTracer(TRACER_NAME).startActiveSpan('invoke_agent', async (span) => { span.setAttribute('gen_ai.operation.name', 'invoke_agent'); setModelIdentityAttributes(span, servingProvider(config), config.model.name, 'langchain'); - setLdSpanAttributes(span, variables, captureContent); + setLdSpanAttributes(span, variables); // Explicit rather than a bare `context.active()`: the active context only carries this // span while an OTel ContextManager is registered, so a host app that installs its own // TracerProvider without one would otherwise get a flat trace. @@ -533,7 +533,7 @@ function createLangChainMessagesHandlerInternal( const span = trace.getTracer(TRACER_NAME).startSpan('invoke_agent'); span.setAttribute('gen_ai.operation.name', 'invoke_agent'); setModelIdentityAttributes(span, servingProvider(config), config.model.name, 'langchain'); - setLdSpanAttributes(span, variables, captureContent); + setLdSpanAttributes(span, variables); const parentContext = trace.setSpan(context.active(), span); // A consumer that `break`s out of `for await`, or throws inside the loop body, makes this diff --git a/packages/litellm-agents/src/handler.ts b/packages/litellm-agents/src/handler.ts index 6270c3be..7d9dffaf 100644 --- a/packages/litellm-agents/src/handler.ts +++ b/packages/litellm-agents/src/handler.ts @@ -411,7 +411,7 @@ export function createLiteLLMAgentsHandlerInternal(options: LiteLLMAgentsOptions trace.getTracer(TRACER_NAME).startActiveSpan('invoke_agent', async (span) => { span.setAttribute('gen_ai.operation.name', 'invoke_agent'); setIdentity(span, configRep); - setLdSpanAttributes(span, variables, captureContent); + setLdSpanAttributes(span, variables); const parentContext = trace.setSpan(context.active(), span); const client = resolveLiteLLMClient(resolvedOptions, configRep); const { agent, model, instructions, runOptions } = buildAgent( @@ -461,7 +461,7 @@ export function createLiteLLMAgentsHandlerInternal(options: LiteLLMAgentsOptions const span = trace.getTracer(TRACER_NAME).startSpan('invoke_agent'); span.setAttribute('gen_ai.operation.name', 'invoke_agent'); setIdentity(span, configRep); - setLdSpanAttributes(span, variables, captureContent); + setLdSpanAttributes(span, variables); const parentContext = trace.setSpan(context.active(), span); const client = resolveLiteLLMClient(resolvedOptions, configRep); const { agent, model, instructions, runOptions } = buildAgent( diff --git a/packages/litellm-messages/src/handler.ts b/packages/litellm-messages/src/handler.ts index e5493b25..e81cd4e9 100644 --- a/packages/litellm-messages/src/handler.ts +++ b/packages/litellm-messages/src/handler.ts @@ -287,7 +287,7 @@ export function createLiteLLMMessagesHandlerInternal(options: LiteLLMMessagesOpt trace.getTracer(TRACER_NAME).startActiveSpan('invoke_agent', async (span) => { span.setAttribute('gen_ai.operation.name', 'invoke_agent'); setIdentity(span, configRep); - setLdSpanAttributes(span, variables, captureContent); + setLdSpanAttributes(span, variables); const parentContext = trace.setSpan(context.active(), span); const messages = buildMessages(configRep, userInput, variables, history); setInputContentAttributes(span, captureContent, { messages: spanMessages(messages) }); @@ -383,7 +383,7 @@ export function createLiteLLMMessagesHandlerInternal(options: LiteLLMMessagesOpt const span = trace.getTracer(TRACER_NAME).startSpan('invoke_agent'); span.setAttribute('gen_ai.operation.name', 'invoke_agent'); setIdentity(span, configRep); - setLdSpanAttributes(span, variables, captureContent); + setLdSpanAttributes(span, variables); const parentContext = trace.setSpan(context.active(), span); const messages = buildMessages(configRep, userInput, variables, history); setInputContentAttributes(span, captureContent, { messages: spanMessages(messages) }); diff --git a/packages/openai-agents/src/__tests__/handler.test.ts b/packages/openai-agents/src/__tests__/handler.test.ts index 21f02a2a..01487b13 100644 --- a/packages/openai-agents/src/__tests__/handler.test.ts +++ b/packages/openai-agents/src/__tests__/handler.test.ts @@ -606,7 +606,6 @@ describe('createOpenAIAgentsHandler', () => { modelName: 'test-model', providerName: 'TestProvider', }, - __ldValue: { model: { name: 'test-model' }, _ldMeta: { enabled: true, variationKey: 'variation-a' } }, }; it('emits the feature_flag correlation event on the invoke_agent root', async () => { diff --git a/packages/openai-agents/src/handler.ts b/packages/openai-agents/src/handler.ts index bb126a02..1a7888a7 100644 --- a/packages/openai-agents/src/handler.ts +++ b/packages/openai-agents/src/handler.ts @@ -619,7 +619,7 @@ export function createOpenAIAgentsHandlerInternal({ return trace.getTracer(TRACER_NAME).startActiveSpan('invoke_agent', async (span) => { span.setAttribute('gen_ai.operation.name', 'invoke_agent'); setModelIdentityAttributes(span, 'openai', config.model.name); - setLdSpanAttributes(span, variables, captureContent); + setLdSpanAttributes(span, variables); // Explicit rather than a bare `context.active()`: the active context only carries this // span while an OTel ContextManager is registered, so a host app that installs its own // TracerProvider without one would otherwise get a flat trace. @@ -678,7 +678,7 @@ export function createOpenAIAgentsHandlerInternal({ const span = trace.getTracer(TRACER_NAME).startSpan('invoke_agent'); span.setAttribute('gen_ai.operation.name', 'invoke_agent'); setModelIdentityAttributes(span, 'openai', config.model.name); - setLdSpanAttributes(span, variables, captureContent); + setLdSpanAttributes(span, variables); const parentContext = trace.setSpan(context.active(), span); // A consumer that `break`s out of `for await`, or throws inside the loop body, makes this diff --git a/packages/openai-messages/src/handler.ts b/packages/openai-messages/src/handler.ts index e0fe7116..577acbae 100644 --- a/packages/openai-messages/src/handler.ts +++ b/packages/openai-messages/src/handler.ts @@ -506,7 +506,7 @@ function createOpenAIMessagesHandlerInternal({ captureContent = false }: Content return trace.getTracer(TRACER_NAME).startActiveSpan('invoke_agent', async (span) => { span.setAttribute('gen_ai.operation.name', 'invoke_agent'); setModelIdentityAttributes(span, 'openai', config.model.name); - setLdSpanAttributes(span, variables, captureContent); + setLdSpanAttributes(span, variables); // Explicit rather than a bare `context.active()`: the active context only carries this // span while an OTel ContextManager is registered, so a host app that installs its own // TracerProvider without one would otherwise get a flat trace. @@ -652,7 +652,7 @@ function createOpenAIMessagesHandlerInternal({ captureContent = false }: Content const span = trace.getTracer(TRACER_NAME).startSpan('invoke_agent'); span.setAttribute('gen_ai.operation.name', 'invoke_agent'); setModelIdentityAttributes(span, 'openai', config.model.name); - setLdSpanAttributes(span, variables, captureContent); + setLdSpanAttributes(span, variables); const parentContext = trace.setSpan(context.active(), span); // A consumer that `break`s out of `for await`, or throws inside the loop body, makes this diff --git a/packages/typesafe/src/handler.ts b/packages/typesafe/src/handler.ts index 9e85c889..ba327c42 100644 --- a/packages/typesafe/src/handler.ts +++ b/packages/typesafe/src/handler.ts @@ -84,7 +84,7 @@ export function createTypesafeHandler(options: TypesafeHandlerOptions = {}): Pro const modelName = config.model?.name || 'jev'; const root = tracer.startSpan('invoke_agent'); const chat = tracer.startSpan(`chat ${modelName}`, undefined, trace.setSpan(context.active(), root)); - setLdSpanAttributes(root, variables, captureContent); + setLdSpanAttributes(root, variables); setModelIdentityAttributes(root, 'typesafe', modelName); setModelIdentityAttributes(chat, 'typesafe', modelName); if (captureContent) { diff --git a/packages/vercel-agents/src/handler.ts b/packages/vercel-agents/src/handler.ts index 9c3c0008..a1fc2830 100644 --- a/packages/vercel-agents/src/handler.ts +++ b/packages/vercel-agents/src/handler.ts @@ -276,7 +276,7 @@ export function createVercelAgentsHandlerInternal(options: VercelAgentsOptions = trace.getTracer(TRACER_NAME).startActiveSpan('invoke_agent', async (rootSpan) => { rootSpan.setAttribute('gen_ai.operation.name', 'invoke_agent'); setModelIdentityAttributes(rootSpan, servingProvider(configRep), configRep.model.name); - setLdSpanAttributes(rootSpan, variables, captureContent); + setLdSpanAttributes(rootSpan, variables); let chatSpan: Span | undefined; try { const messages = buildMessages(configRep, userInput, variables, history); @@ -318,7 +318,7 @@ export function createVercelAgentsHandlerInternal(options: VercelAgentsOptions = const rootSpan = trace.getTracer(TRACER_NAME).startSpan('invoke_agent'); rootSpan.setAttribute('gen_ai.operation.name', 'invoke_agent'); setModelIdentityAttributes(rootSpan, servingProvider(configRep), configRep.model.name); - setLdSpanAttributes(rootSpan, variables, captureContent); + setLdSpanAttributes(rootSpan, variables); let chatSpan: Span | undefined; let iterator: AsyncIterator | undefined; let completed = false; diff --git a/packages/vercel-messages/src/evaluate.ts b/packages/vercel-messages/src/evaluate.ts index 3f138a91..7b6ddcab 100644 --- a/packages/vercel-messages/src/evaluate.ts +++ b/packages/vercel-messages/src/evaluate.ts @@ -110,7 +110,7 @@ export async function vercelEvaluate { span.setAttribute('gen_ai.operation.name', 'evaluate'); setModelIdentityAttributes(span, servingProvider(config), config.model.name); - setLdSpanAttributes(span, { __ld: trackData, ldContext: { ...context } }, captureContent); + setLdSpanAttributes(span, { __ld: trackData, ldContext: { ...context } }); const payload = JSON.stringify({ state, questions: options.questions }); setInputContentAttributes(span, captureContent, { messages: [textMessage('user', payload)], diff --git a/packages/vercel-messages/src/handler.ts b/packages/vercel-messages/src/handler.ts index 47222049..9d7891f3 100644 --- a/packages/vercel-messages/src/handler.ts +++ b/packages/vercel-messages/src/handler.ts @@ -266,7 +266,7 @@ function createVercelMessagesHandlerInternal(options: VercelMessagesOptions = {} trace.getTracer(TRACER_NAME).startActiveSpan('invoke_agent', async (rootSpan) => { rootSpan.setAttribute('gen_ai.operation.name', 'invoke_agent'); setModelIdentityAttributes(rootSpan, servingProvider(configRep), configRep.model.name); - setLdSpanAttributes(rootSpan, variables, captureContent); + setLdSpanAttributes(rootSpan, variables); let chatSpan: Span | undefined; try { const system = systemInstructions(configRep, variables); @@ -316,7 +316,7 @@ function createVercelMessagesHandlerInternal(options: VercelMessagesOptions = {} const rootSpan = trace.getTracer(TRACER_NAME).startSpan('invoke_agent'); rootSpan.setAttribute('gen_ai.operation.name', 'invoke_agent'); setModelIdentityAttributes(rootSpan, servingProvider(configRep), configRep.model.name); - setLdSpanAttributes(rootSpan, variables, captureContent); + setLdSpanAttributes(rootSpan, variables); let chatSpan: Span | undefined; let chatEnded = false; let rootEnded = false; From bda9e7b4b6faa885a50476de49902bdc99612387 Mon Sep 17 00:00:00 2001 From: Alexis Georges Date: Wed, 28 Oct 2026 17:58:46 -0400 Subject: [PATCH 05/11] test(client): cover the capture-on feature_flag value end to end Run config().invoke with a handler built by createHandler with captureContent on and check that the root feature_flag event's feature_flag.result.value is the JSON of the raw variation extractVariation returned. --- packages/client/src/__tests__/client.test.ts | 38 ++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/packages/client/src/__tests__/client.test.ts b/packages/client/src/__tests__/client.test.ts index c43b937a..9530759d 100644 --- a/packages/client/src/__tests__/client.test.ts +++ b/packages/client/src/__tests__/client.test.ts @@ -27,6 +27,7 @@ import { config } from '../client.js'; import { runJudges } from '../judges.js'; import { extractVariation, getClient } from '../lifecycle.js'; import type { ProviderHandler } from '../types.js'; +import { createHandler, setLdSpanAttributes } from '../utils.js'; // ─── Fixtures ───────────────────────────────────────────────────────────────── @@ -820,3 +821,40 @@ describe('config().stream() — multi-handler routing', () => { await expect(collectStream(gen)).rejects.toThrow(/OpenAI/i); }); }); + +// ─── config().invoke — feature_flag.result.value with content capture on ───── + +describe('config().invoke — capture-on handler', () => { + beforeEach(() => { + vi.clearAllMocks(); + (getClient as ReturnType).mockReturnValue({ track: mockTrack }); + (runJudges as ReturnType).mockResolvedValue({}); + }); + + it('writes the raw variation JSON as feature_flag.result.value on the root feature_flag event', async () => { + const rawVariation = { + ...mockConfig, + _ldMeta: { enabled: true, variationKey: 'v1', version: 1, mode: 'messages' }, + }; + (extractVariation as ReturnType).mockResolvedValue({ + config: mockConfig, + meta: mockMeta, + value: rawVariation, + }); + const span = { setAttribute: vi.fn(), addEvent: vi.fn() }; + const handler = createHandler( + ['OpenAI', 'messages'], + async (_config, _input, _tools, variables) => { + setLdSpanAttributes(span as any, variables); + return { output: 'ok', usage: { input_tokens: 1, output_tokens: 1 } }; + }, + undefined, + true, + ); + + await config({ key: 'my-flag', handler }).invoke('hi', mockContext); + + const event = span.addEvent.mock.calls.find((c: unknown[]) => c[0] === 'feature_flag'); + expect((event?.[1] as Record)['feature_flag.result.value']).toBe(JSON.stringify(rawVariation)); + }); +}); From 8398f4e1ddc53b5265108a5d91c9b87162ebdda7 Mon Sep 17 00:00:00 2001 From: Alexis Georges Date: Thu, 29 Oct 2026 11:43:04 -0400 Subject: [PATCH 06/11] feat(client): carry the judge variation on JudgeTask for feature_flag.result.value A deferred judge (buildJudgeTasks, then runJudge in a worker) wrote the variation key as feature_flag.result.value even when the judge handler captures content, because the task had no raw variation to pass. JudgeTask now carries it as judgeValue, and runJudge hands it to the core as the evaluated value, which writes it only for a capture-on handler. --- packages/client/etc/ai-server.api.md | 1 + packages/client/src/__tests__/client.test.ts | 71 +++++++++++++++++++- packages/client/src/judges.ts | 5 +- packages/client/src/types.ts | 5 ++ 4 files changed, 80 insertions(+), 2 deletions(-) diff --git a/packages/client/etc/ai-server.api.md b/packages/client/etc/ai-server.api.md index 71180621..42ae92f8 100644 --- a/packages/client/etc/ai-server.api.md +++ b/packages/client/etc/ai-server.api.md @@ -408,6 +408,7 @@ export type JudgeTask = { configKey: string; judgeConfig: AiConfigRep; judgeMeta: LDVariationMeta; + judgeValue?: unknown; actualOutput: string; userInput?: string; trajectory?: string; diff --git a/packages/client/src/__tests__/client.test.ts b/packages/client/src/__tests__/client.test.ts index 9530759d..9fbee755 100644 --- a/packages/client/src/__tests__/client.test.ts +++ b/packages/client/src/__tests__/client.test.ts @@ -24,7 +24,7 @@ vi.mock('../judges.js', async (importOriginal) => { }); import { config } from '../client.js'; -import { runJudges } from '../judges.js'; +import { buildJudgeTasks, runJudge, runJudges } from '../judges.js'; import { extractVariation, getClient } from '../lifecycle.js'; import type { ProviderHandler } from '../types.js'; import { createHandler, setLdSpanAttributes } from '../utils.js'; @@ -858,3 +858,72 @@ describe('config().invoke — capture-on handler', () => { expect((event?.[1] as Record)['feature_flag.result.value']).toBe(JSON.stringify(rawVariation)); }); }); + +// ─── buildJudgeTasks / runJudge — feature_flag.result.value on the deferred judge span ───── + +describe('deferred judge feature_flag.result.value', () => { + const judgeConfig = { + model: { name: 'gpt-4o' }, + provider: { name: 'OpenAI' }, + instructions: 'You are a judge.', + evaluationMetricKey: 'judge-metric', + }; + const judgeMeta = { enabled: true, variationKey: 'judge-v1', version: 2, mode: 'messages' as const }; + const rawJudgeVariation = { ...judgeConfig, _ldMeta: { ...judgeMeta } }; + const parentConfig = { + ...mockConfig, + judgeConfiguration: { judges: [{ key: 'judge-flag', samplingRate: 1 }] }, + }; + + beforeEach(() => { + vi.clearAllMocks(); + (getClient as ReturnType).mockReturnValue({ track: mockTrack }); + (extractVariation as ReturnType).mockResolvedValue({ + config: judgeConfig, + meta: judgeMeta, + value: rawJudgeVariation, + }); + }); + + const judgeHandler = (span: { setAttribute: unknown; addEvent: unknown }, capture: boolean) => + createHandler( + ['OpenAI', 'messages'], + async (_config, _input, _tools, variables) => { + setLdSpanAttributes(span as any, variables); + return { output: '{"score":0.9,"reasoning":"ok"}', usage: { input_tokens: 1, output_tokens: 1 } }; + }, + undefined, + capture, + ); + + const featureFlagValue = (span: { addEvent: ReturnType }) => { + const event = span.addEvent.mock.calls.find((c: unknown[]) => c[0] === 'feature_flag'); + return (event?.[1] as Record)['feature_flag.result.value']; + }; + + it.each([ + ['on', true, JSON.stringify(rawJudgeVariation)], + ['off', false, JSON.stringify('judge-v1')], + ])('with capture %s, the task carries the raw judge variation and its run writes the expected value', async (_label, capture, expected) => { + const span = { setAttribute: vi.fn(), addEvent: vi.fn() }; + const handler = judgeHandler(span, capture); + const tasks = await buildJudgeTasks({ + config: parentConfig, + userContext: mockContext, + handler, + handlers: [handler], + llmResponse: 'hello', + baseTrackData: { runId: 'r', configKey: 'flag', variationKey: 'v1', version: 1, modelName: '', providerName: '' }, + }); + + expect(tasks).toHaveLength(1); + expect(tasks[0].judgeValue).toEqual(rawJudgeVariation); + // A worker receives the task through structured clone. + const task = structuredClone(tasks[0]); + + const result = await runJudge(task, [handler]); + + expect(result?.score).toBe(0.9); + expect(featureFlagValue(span)).toBe(expected); + }); +}); diff --git a/packages/client/src/judges.ts b/packages/client/src/judges.ts index 4b2ef594..a87c6a64 100644 --- a/packages/client/src/judges.ts +++ b/packages/client/src/judges.ts @@ -324,7 +324,7 @@ export const buildJudgeTasksInternal = async ({ const resolved = await resolveJudge(judge.key, userContext); if (!resolved) continue; - const { config: judgeConfig, meta: judgeMeta } = resolved; + const { config: judgeConfig, meta: judgeMeta, value: judgeValue } = resolved; const judgeProvider = judgeConfig.provider?.name; const judgeMode = normalizeMode(judgeMeta.mode); @@ -351,6 +351,7 @@ export const buildJudgeTasksInternal = async ({ configKey: judge.key, judgeConfig: withoutOutputFormat(judgeConfig, judge.key), judgeMeta, + judgeValue, actualOutput: llmResponse, userInput, trajectory, @@ -384,6 +385,7 @@ export const runJudge = async (task: JudgeTask, handlers: ProviderHandler[]): Pr const { judgeConfig, judgeMeta, + judgeValue, actualOutput, userInput, trajectory, @@ -428,6 +430,7 @@ export const runJudge = async (task: JudgeTask, handlers: ProviderHandler[]): Pr response_to_evaluate: actualOutput, ...(userInput ? { input: userInput } : {}), }, + evaluatedValue: judgeValue, }); const judgeResponse = typeof rawResponse === 'string' ? rawResponse : JSON.stringify(rawResponse); diff --git a/packages/client/src/types.ts b/packages/client/src/types.ts index d3a0845f..63822634 100644 --- a/packages/client/src/types.ts +++ b/packages/client/src/types.ts @@ -657,6 +657,11 @@ export type JudgeTask = { judgeConfig: AiConfigRep; /** Variation metadata for the judge config. */ judgeMeta: VariationMeta; + /** + * The judge variation exactly as the LaunchDarkly SDK returned it. Written as + * `feature_flag.result.value` on the judge span when the judge's handler captures content. + */ + judgeValue?: unknown; /** The LLM response to evaluate. */ actualOutput: string; /** From de89d4a4397b9b35106a976de2ab72ec7ac1d2ab Mon Sep 17 00:00:00 2001 From: Alexis Georges Date: Thu, 29 Oct 2026 11:45:55 -0400 Subject: [PATCH 07/11] refactor(client): resolve content capture for a call through one function captureContentFor returns a handler's own captureContent when it states one, otherwise the process-wide default (off). The evaluated-variation gate in executeAndTrack and executeAndStream now goes through it, so a later process-wide setting changes one place. Behavior is unchanged. --- packages/client/src/content.ts | 14 ++++++++++++++ packages/client/src/tracking.ts | 5 +++-- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/packages/client/src/content.ts b/packages/client/src/content.ts index a8842e4e..8dc0f30d 100644 --- a/packages/client/src/content.ts +++ b/packages/client/src/content.ts @@ -1,4 +1,5 @@ import type { Span } from '@opentelemetry/api'; +import type { ProviderHandler } from './types.js'; /** * Conversation content on spans, per LaunchDarkly's "Richer LLM spans" proposal. @@ -59,6 +60,19 @@ export interface ContentCaptureOptions { captureContent?: boolean; } +/** Content capture when nothing states a preference. Conversation content is PII, so it is off. */ +const DEFAULT_CAPTURE_CONTENT = false; + +/** + * Content capture for a call made through `handler`: its own `captureContent` when it states one, + * otherwise the process-wide default. Pass `undefined` for a span that has no single handler, such + * as the graph root. + * + * Internal: not exported from an entry point. + */ +export const captureContentFor = (handler: ProviderHandler | undefined): boolean => + typeof handler?.captureContent === 'boolean' ? handler.captureContent : DEFAULT_CAPTURE_CONTENT; + /** A typed piece of one message, mirroring the `parts` union in the GenAI JSON schemas. */ export type SpanMessagePart = | { type: 'text'; content: string } diff --git a/packages/client/src/tracking.ts b/packages/client/src/tracking.ts index 732e666c..e373208e 100644 --- a/packages/client/src/tracking.ts +++ b/packages/client/src/tracking.ts @@ -1,3 +1,4 @@ +import { captureContentFor } from './content.js'; import { getClient } from './lifecycle.js'; import { exposedToolKeys, HANDOFF_TOOL_PREFIX, renderTrajectory, TrajectoryRecorder } from './trajectory.js'; import type { @@ -260,7 +261,7 @@ export const executeAndTrack = async ({ ...variables, ldContext: { ...userContext }, __ld: trackData, - ...(evaluatedValue !== undefined && handler.captureContent === true ? { __ldValue: evaluatedValue } : {}), + ...(evaluatedValue !== undefined && captureContentFor(handler) ? { __ldValue: evaluatedValue } : {}), }, history, ); @@ -338,7 +339,7 @@ export async function* executeAndStream({ ...variables, ldContext: { ...userContext }, __ld: trackData, - ...(evaluatedValue !== undefined && handler.captureContent === true ? { __ldValue: evaluatedValue } : {}), + ...(evaluatedValue !== undefined && captureContentFor(handler) ? { __ldValue: evaluatedValue } : {}), }; const startTime = Date.now(); From 4ad0f344614283d151b29a694c52c75641ca13b0 Mon Sep 17 00:00:00 2001 From: Alexis Georges Date: Thu, 29 Oct 2026 11:45:55 -0400 Subject: [PATCH 08/11] feat(client): write the graph variation as feature_flag.result.value on the graph root The launchdarkly.graph span wrote the variation key as feature_flag.result.value even when content capture is on, because the raw graph variation never reached it. buildGraph now returns the variation it fetched, and graph().stream (which invoke() walks) passes it as __ldValue when captureContentFor(undefined) is true. The root has no single handler, so it follows the process-wide default. --- packages/client/src/__tests__/graph.test.ts | 43 ++++++++++++++++++++- packages/client/src/graph.ts | 30 ++++++++++---- packages/client/src/utils.ts | 3 +- 3 files changed, 67 insertions(+), 9 deletions(-) diff --git a/packages/client/src/__tests__/graph.test.ts b/packages/client/src/__tests__/graph.test.ts index c6a86fb3..5da26d6b 100644 --- a/packages/client/src/__tests__/graph.test.ts +++ b/packages/client/src/__tests__/graph.test.ts @@ -5,12 +5,23 @@ import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vites // ─── Hoisted mocks ──────────────────────────────────────────────────────────── -const { mockTrack, mockVariation, mockExtractVariation } = vi.hoisted(() => ({ +const { mockTrack, mockVariation, mockExtractVariation, captureDefault } = vi.hoisted(() => ({ mockTrack: vi.fn(), mockVariation: vi.fn(), mockExtractVariation: vi.fn(), + // Stands in for the process-wide content capture default, which is off on this branch. + captureDefault: { value: false }, })); +vi.mock('../content.js', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + captureContentFor: (handler: { captureContent?: boolean } | undefined) => + typeof handler?.captureContent === 'boolean' ? handler.captureContent : captureDefault.value, + }; +}); + vi.mock('../lifecycle.js', () => ({ getClient: vi.fn().mockReturnValue({ track: mockTrack, variation: mockVariation }), initClient: vi.fn().mockResolvedValue(undefined), @@ -667,6 +678,36 @@ describe('graph() conversation id', () => { expect(flagEvents[0].attributes?.['feature_flag.key']).toBe('graph-flag'); }); + it.each([ + ['invoke', false], + ['stream', false], + ['invoke', true], + ['stream', true], + ] as const)('writes feature_flag.result.value on the launchdarkly.graph span on %s with default capture %s', async (mode, capture) => { + setupTwoNodeGraph(); + const rawGraphVariation = { + root: 'root-node', + edges: { 'root-node': [{ key: 'leaf-node' }] }, + _ldMeta: { variationKey: 'graph-v1', version: 3 }, + }; + mockVariation.mockResolvedValue(rawGraphVariation); + captureDefault.value = capture; + try { + const g = graph('graph-flag', { handlers: [makeStreamingHandler(['ok'])] }); + if (mode === 'invoke') await g.invoke('hi', mockContext); + else await collectStream(g.stream('hi', mockContext)); + } finally { + captureDefault.value = false; + } + + const graphSpan = exporter.getFinishedSpans().find((s) => s.name === 'launchdarkly.graph'); + const flagEvent = graphSpan?.events.find((e) => e.name === 'feature_flag'); + expect(flagEvent?.attributes?.['feature_flag.result.variant']).toBe('graph-v1'); + expect(flagEvent?.attributes?.['feature_flag.result.value']).toBe( + capture ? JSON.stringify(rawGraphVariation) : JSON.stringify('graph-v1'), + ); + }); + it('gives each invocation with the same context its own run id', async () => { setupTwoNodeGraph(); const g = graph('graph-flag', { handlers: [makeStreamingHandler(['ok'])] }); diff --git a/packages/client/src/graph.ts b/packages/client/src/graph.ts index 6ab22f85..27b42640 100644 --- a/packages/client/src/graph.ts +++ b/packages/client/src/graph.ts @@ -1,5 +1,6 @@ import type { Context, Span } from '@opentelemetry/api'; import { context as otelContext, SpanStatusCode, trace } from '@opentelemetry/api'; +import { captureContentFor } from './content.js'; import { bindConversationId, bindSpanContext } from './conversation.js'; import { runJudges } from './judges.js'; import { extractVariation, getClient, initClient } from './lifecycle.js'; @@ -92,7 +93,7 @@ const disabledDefinition = (key: string): GraphDefinition => ({ const fetchGraphVariation = async ( key: string, context: LDContext, -): Promise<{ enabled: boolean; topology?: GraphTopology; meta: VariationMeta }> => { +): Promise<{ enabled: boolean; topology?: GraphTopology; meta: VariationMeta; value: unknown }> => { await initClient(); const variation: unknown = await getClient().variation(key, context, {}); @@ -103,10 +104,10 @@ const fetchGraphVariation = async ( const parsed = GraphTopologySchema.safeParse(variation); if (!parsed.success || !parsed.data.root) { - return { enabled: false, meta }; + return { enabled: false, meta, value: variation }; } - return { enabled: true, topology: parsed.data, meta }; + return { enabled: true, topology: parsed.data, meta, value: variation }; }; /** @@ -124,6 +125,8 @@ const buildGraph = async ( ): Promise<{ def: GraphDefinition; graphTrackData: TrackData; + /** The graph flag variation exactly as the LaunchDarkly SDK returned it. */ + graphValue: unknown; streamRoute: ( node: GraphNode, input?: string, @@ -131,7 +134,7 @@ const buildGraph = async ( runTrackData?: TrackData, ) => AsyncGenerator; }> => { - const { enabled, topology, meta } = await fetchGraphVariation(key, context); + const { enabled, topology, meta, value: graphValue } = await fetchGraphVariation(key, context); const graphTrackData: TrackData = { runId: crypto.randomUUID(), @@ -153,6 +156,7 @@ const buildGraph = async ( return { def: disabledDefinition(key), graphTrackData, + graphValue, streamRoute: disabledStreamRoute, }; } @@ -201,6 +205,7 @@ const buildGraph = async ( return { def: disabledDefinition(key), graphTrackData, + graphValue, streamRoute: disabledStreamRoute, }; } @@ -584,7 +589,7 @@ const buildGraph = async ( reverseTraverse, }; - return { def, graphTrackData, streamRoute }; + return { def, graphTrackData, graphValue, streamRoute }; }; /** @@ -721,7 +726,12 @@ export const graphInternal = (key: string, options: GraphOptions): GraphCaller = ); } - const { def, graphTrackData: builtTrackData, streamRoute } = await buildGraph(key, context, resolvedOptions); + const { + def, + graphTrackData: builtTrackData, + graphValue, + streamRoute, + } = await buildGraph(key, context, resolvedOptions); if (!def.enabled) { throw new Error(`Agent graph "${key}" is disabled`); } @@ -730,7 +740,13 @@ export const graphInternal = (key: string, options: GraphOptions): GraphCaller = const graphTrackData: TrackData = { ...builtTrackData, runId: crypto.randomUUID() }; const span = trace.getTracer('@launchdarkly/ai-server').startSpan('launchdarkly.graph', undefined, callerContext); - setLdSpanAttributes(span, { __ld: graphTrackData, ldContext: context }); + // The root span has no single handler, so the raw graph variation follows the process-wide + // content capture default. Without it, feature_flag.result.value falls back to the variation key. + setLdSpanAttributes(span, { + __ld: graphTrackData, + ldContext: context, + ...(graphValue !== undefined && captureContentFor(undefined) ? { __ldValue: graphValue } : {}), + }); const spanContext = trace.setSpan(callerContext, span); const ended = new Set(); diff --git a/packages/client/src/utils.ts b/packages/client/src/utils.ts index 5da90009..f01913f1 100644 --- a/packages/client/src/utils.ts +++ b/packages/client/src/utils.ts @@ -479,7 +479,8 @@ function contextIdentityFromLdContext(ldContext: unknown): ContextIdentity | und * feature_flag.result.value is `JSON.stringify` of `variables.__ldValue`, the * raw variation (as the OTel TracingHook writes it with `includeValue: true`), * when present. The core sets `__ldValue` only for handlers with - * `captureContent` on. Otherwise it is `JSON.stringify(variationKey)` when the + * `captureContent` on, and on the graph root only when the process-wide + * default captures content. Otherwise it is `JSON.stringify(variationKey)` when the * key is non-empty, and omitted when there is neither. * The observability backend projects these event attrs to span-level attrs, * satisfying the query: events.name=feature_flag AND From 1fb76904c206179e9ff43416c243abc9dd5f951a Mon Sep 17 00:00:00 2001 From: Alexis Georges Date: Thu, 29 Oct 2026 11:46:19 -0400 Subject: [PATCH 09/11] docs: say where deferred judges and the graph root get feature_flag.result.value --- AGENTS.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/AGENTS.md b/AGENTS.md index 16899d80..935573d6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -641,6 +641,10 @@ key (for example `"\"opus\""`), and it is left off when there is neither. The core client sets `__ldValue`, beside `__ld`, only when the handler that runs has `captureContent` on (`ProviderHandler.captureContent`, the same gate judge reasoning uses), so the raw config never reaches a capture-off handler. +A deferred judge carries its variation on `JudgeTask.judgeValue`, and +`runJudge` applies the same gate with the judge's handler. The +`launchdarkly.graph` root span has no single handler, so it gets the graph +flag's raw variation only when the process-wide default captures content. It is not in `TrackData`, because `TrackData` is the `$ld:ai:*` event payload. When `variables.ldContext` has a usable identity, that event also carries `feature_flag.context.id` and `feature_flag.contextKeys`, From 995a2cc09f40d07226582f6efbbe055cb0c24754 Mon Sep 17 00:00:00 2001 From: Alexis Georges Date: Thu, 29 Oct 2026 15:26:03 -0400 Subject: [PATCH 10/11] fix(client): never pass a caller-supplied __ldValue to a handler executeAndTrack and executeAndStream spread the caller's variables before adding __ldValue, so a capture-off handler still received a caller-supplied __ldValue and wrote it as feature_flag.result.value. Both now build the handler's variables in one place that sets __ldValue only for a capture-on handler and removes it otherwise, as __ld is already overwritten. --- .../client/src/__tests__/tracking.test.ts | 18 ++++++++ packages/client/src/tracking.ts | 42 +++++++++++-------- 2 files changed, 42 insertions(+), 18 deletions(-) diff --git a/packages/client/src/__tests__/tracking.test.ts b/packages/client/src/__tests__/tracking.test.ts index 759ea1c8..46703026 100644 --- a/packages/client/src/__tests__/tracking.test.ts +++ b/packages/client/src/__tests__/tracking.test.ts @@ -221,6 +221,24 @@ describe('executeAndTrack', () => { expect(handler.mock.calls[0][3]).not.toHaveProperty('__ldValue'); }); + it.each([ + ['capture off', false, undefined], + ['capture on', true, 'evaluated'], + ])('never passes a caller-supplied __ldValue to a handler with %s', async (_label, capture, expected) => { + const handler = makeHandler('ok'); + handler.captureContent = capture; + await executeAndTrack({ + configKey: 'my-flag', + config: execConfig as any, + meta: execMeta, + userContext: execContext, + handler, + variables: { __ldValue: 'from-caller' }, + evaluatedValue: 'evaluated', + }); + expect(handler.mock.calls[0][3].__ldValue).toBe(expected); + }); + it('does not set __ldValue when evaluatedValue is absent', async () => { const handler = makeHandler('ok'); handler.captureContent = true; diff --git a/packages/client/src/tracking.ts b/packages/client/src/tracking.ts index 4e6faa4e..8e17e73e 100644 --- a/packages/client/src/tracking.ts +++ b/packages/client/src/tracking.ts @@ -237,6 +237,27 @@ export const makeRunTrackData = ({ environmentId: tryGetEnvironmentId(), }); +/** + * The variables a handler receives. `ldContext`, `__ld` and `__ldValue` are SDK-owned, so a + * caller-supplied value never survives: `__ldValue` is the evaluated variation for a handler + * that captures content, and absent otherwise. + */ +const handlerVariables = ( + variables: Record | undefined, + userContext: LDContext, + trackData: TrackData, + handler: ProviderHandler, + evaluatedValue: unknown, +): Record => { + const merged: Record = { ...variables, ldContext: { ...userContext }, __ld: trackData }; + if (evaluatedValue !== undefined && captureContentFor(handler)) { + merged.__ldValue = evaluatedValue; + } else { + delete merged.__ldValue; + } + return merged; +}; + export const executeAndTrack = async ({ configKey, config, @@ -271,23 +292,13 @@ export const executeAndTrack = async ({ const trackData = makeRunTrackData({ configKey, config, meta, graphKey }); const { tools: trackedToolHandlers, recorder } = instrumentTools(config, toolHandlers, userContext, trackData); + const mergedVariables = handlerVariables(variables, userContext, trackData, handler, evaluatedValue); const startTime = Date.now(); let response: Awaited>; try { - response = await handler( - config, - userInput, - trackedToolHandlers, - { - ...variables, - ldContext: { ...userContext }, - __ld: trackData, - ...(evaluatedValue !== undefined && captureContentFor(handler) ? { __ldValue: evaluatedValue } : {}), - }, - history, - ); + response = await handler(config, userInput, trackedToolHandlers, mergedVariables, history); } catch (err) { getClient().track('$ld:ai:duration:total', userContext, trackData, Date.now() - startTime); getClient().track('$ld:ai:generation:error', userContext, trackData, 1); @@ -358,12 +369,7 @@ export async function* executeAndStream({ const trackData = makeRunTrackData({ configKey, config, meta, graphKey }); const { tools: trackedToolHandlers, recorder } = instrumentTools(config, toolHandlers, userContext, trackData); - const mergedVariables = { - ...variables, - ldContext: { ...userContext }, - __ld: trackData, - ...(evaluatedValue !== undefined && captureContentFor(handler) ? { __ldValue: evaluatedValue } : {}), - }; + const mergedVariables = handlerVariables(variables, userContext, trackData, handler, evaluatedValue); const startTime = Date.now(); let fullText = ''; From 869636ad4f00fcc1df0311905f9a2655463f5d10 Mon Sep 17 00:00:00 2001 From: Alexis Georges Date: Thu, 29 Oct 2026 15:37:00 -0400 Subject: [PATCH 11/11] test(client): pass captureContent to createHandler as an option #111 replaced createHandler's positional stream and captureContent arguments with an options object, and the positional form now throws. The two client tests this branch added still used it. --- packages/client/src/__tests__/client.test.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/packages/client/src/__tests__/client.test.ts b/packages/client/src/__tests__/client.test.ts index 80bbfbdb..d8ad997b 100644 --- a/packages/client/src/__tests__/client.test.ts +++ b/packages/client/src/__tests__/client.test.ts @@ -949,8 +949,7 @@ describe('config().invoke — capture-on handler', () => { setLdSpanAttributes(span as any, variables); return { output: 'ok', usage: { input_tokens: 1, output_tokens: 1 } }; }, - undefined, - true, + { captureContent: true }, ); await config({ key: 'my-flag', handler }).invoke('hi', mockContext); @@ -993,8 +992,7 @@ describe('deferred judge feature_flag.result.value', () => { setLdSpanAttributes(span as any, variables); return { output: '{"score":0.9,"reasoning":"ok"}', usage: { input_tokens: 1, output_tokens: 1 } }; }, - undefined, - capture, + { captureContent: capture }, ); const featureFlagValue = (span: { addEvent: ReturnType }) => {