Skip to content

Commit 8ec5fe4

Browse files
andreiborzaclaude
andauthored
feat(server-utils): Instrument the OpenAI decisions API as gen_ai.decide spans (#25285)
## What OpenAI `decisions.create` calls now create `decide <model>` spans with op `gen_ai.decide`, recorded in the same shape as Jev calls so Sentry renders them the same way: - `input` is recorded as `state`, images included as they are. - Questions and answers are keyed by question name, as the conventions define. - Choice and score question options go in `criteria`, like Jev: choices as `{ [value]: description }`, score levels as their labels. - Choice and score `probabilities` are keyed by option or score value, and score labels go in a `legend`. - `predicate` answers keep OpenAI's `{ type, probability }`, which Sentry renders as yes/no. | | OpenAI sends / returns | Recorded on the span | |---|---|---| | Evaluated input | `input: "I want a refund"` | `state: "I want a refund"` | | Questions | `[{ name: "wants_refund", type: "predicate", ... }]` | `{ wants_refund: { type: "predicate", ... } }` | | Choice question | `choices: [{ value: "billing", description: "Charges and refunds" }, { value: "other" }]` | `criteria: { billing: "Charges and refunds", other: "" }` | | Score question | `levels: [{ label: "low" }, { label: "high", description: "Blocks the customer" }]` | `criteria: ["low", "high"]`, `descriptions: { "1": "Blocks the customer" }` | | Answers | `[{ name: "wants_refund", type: "predicate", ... }]` | `{ wants_refund: { type: "predicate", ... } }` | | Predicate answer | `{ type: "predicate", probability: 0.97 }` | unchanged | | Choice answer | `choice: true`, `probabilities: [{ value: true, probability: 0.91 }, ...]` | `choice: "true"`, `probabilities: { "true": 0.91, ... }` | | Score answer | `probabilities: [{ value: 1, label: "high", probability: 0.95 }, ...]` | `probabilities: { "1": 0.95, ... }`, `legend: { "1": "high", ... }` | ### OpenAI decisions have a few things Jev does not - `predicate` questions and answers instead of `noul`, with the yes probability in `probability` - Score levels can have descriptions, recorded in `descriptions` keyed by score value - A question can come back as a refusal, `{ type: "refusal" }`, with no probability or reason - The input can hold images as base64 data URLs. ## Why OpenAI added a Decisions API that works like Jev, and these calls had no span. Related: #25253 --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
1 parent ca4c983 commit 8ec5fe4

10 files changed

Lines changed: 476 additions & 34 deletions

File tree

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
import * as Sentry from '@sentry/node';
2+
import { loggingTransport } from '@sentry-internal/node-integration-tests';
3+
4+
Sentry.init({
5+
dsn: 'https://public@dsn.ingest.sentry.io/1337',
6+
release: '1.0',
7+
tracesSampleRate: 1.0,
8+
transport: loggingTransport,
9+
});
Lines changed: 115 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,115 @@
1+
import * as Sentry from '@sentry/node';
2+
import express from 'express';
3+
import OpenAI from 'openai';
4+
5+
function startMockServer() {
6+
const app = express();
7+
app.use(express.json({ limit: '1mb' }));
8+
9+
app.post('/openai/decisions', (req, res) => {
10+
const { model } = req.body;
11+
12+
if (model === 'error-model') {
13+
res.status(500).set('x-request-id', 'mock-request-error').end('Internal server error');
14+
return;
15+
}
16+
17+
// Shaped like a real `gpt-6-luna` response. Decisions bills input tokens only.
18+
res.send({
19+
model: 'gpt-6-luna',
20+
answers: [
21+
{ type: 'predicate', name: 'wants_refund', probability: 0.97 },
22+
{
23+
type: 'choice',
24+
name: 'department',
25+
choice: 'billing',
26+
probabilities: [
27+
{ value: 'billing', probability: 0.82 },
28+
{ value: 'other', probability: 0.18 },
29+
],
30+
confidence: 0.82,
31+
},
32+
{
33+
type: 'score',
34+
name: 'urgency',
35+
score: 0.95,
36+
probabilities: [
37+
{ value: 0, label: 'low', probability: 0.05 },
38+
{ value: 1, label: 'high', probability: 0.95 },
39+
],
40+
confidence: 0.93,
41+
},
42+
],
43+
usage: {
44+
input_tokens: 382,
45+
input_tokens_details: { cached_tokens: 256, cache_write_tokens: 64 },
46+
output_tokens: 0,
47+
output_tokens_details: { reasoning_tokens: 0 },
48+
total_tokens: 382,
49+
},
50+
});
51+
});
52+
53+
return new Promise(resolve => {
54+
const server = app.listen(0, () => {
55+
resolve(server);
56+
});
57+
});
58+
}
59+
60+
const questions = [
61+
{ type: 'predicate', name: 'wants_refund', instructions: 'Is a refund requested?' },
62+
{
63+
type: 'choice',
64+
name: 'department',
65+
instructions: 'Which team should handle this?',
66+
choices: [{ value: 'billing', description: 'Charges and refunds' }, { value: 'other' }],
67+
},
68+
{
69+
type: 'score',
70+
name: 'urgency',
71+
instructions: 'How urgent is this ticket?',
72+
levels: [{ label: 'low' }, { label: 'high', description: 'Blocks the customer' }],
73+
},
74+
];
75+
76+
async function run() {
77+
const server = await startMockServer();
78+
79+
await Sentry.startSpan({ op: 'function', name: 'main' }, async () => {
80+
const client = new OpenAI({
81+
baseURL: `http://localhost:${server.address().port}/openai`,
82+
apiKey: 'mock-api-key',
83+
});
84+
85+
await client.decisions.create({
86+
model: 'gpt-6-luna',
87+
input: 'I was charged twice and want a refund.',
88+
questions,
89+
});
90+
91+
await client.decisions.create({
92+
model: 'gpt-6-luna',
93+
input: [
94+
{
95+
role: 'user',
96+
content: [
97+
{ type: 'input_text', text: 'Is the package damaged?' },
98+
{ type: 'input_image', image_url: 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUg==' },
99+
],
100+
},
101+
],
102+
questions: [{ type: 'predicate', name: 'damaged', instructions: 'Is there visible damage?' }],
103+
});
104+
105+
try {
106+
await client.decisions.create({ model: 'error-model', input: 'Error test', questions });
107+
} catch {
108+
// Error is expected and handled
109+
}
110+
});
111+
112+
server.close();
113+
}
114+
115+
run();

‎dev-packages/node-integration-tests/suites/tracing/openai/v7/test.ts‎

Lines changed: 105 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,25 @@
11
import type { SerializedStreamedSpanContainer } from '@sentry/core';
22
import { afterAll, describe, expect } from 'vitest';
33
import {
4+
GEN_AI_INPUT_MESSAGES,
45
GEN_AI_OPERATION_NAME,
6+
GEN_AI_OUTPUT_MESSAGES,
57
GEN_AI_PROVIDER_NAME,
68
GEN_AI_REQUEST_MODEL,
79
GEN_AI_RESPONSE_FINISH_REASONS,
810
GEN_AI_RESPONSE_ID,
911
GEN_AI_RESPONSE_MODEL,
1012
GEN_AI_RESPONSE_STREAMING,
13+
GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS,
14+
GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS,
1115
GEN_AI_USAGE_INPUT_TOKENS,
1216
GEN_AI_USAGE_OUTPUT_TOKENS,
17+
GEN_AI_USAGE_REASONING_OUTPUT_TOKENS,
1318
GEN_AI_USAGE_TOTAL_TOKENS,
1419
SENTRY_OP,
1520
SENTRY_ORIGIN,
1621
} from '@sentry/conventions/attributes';
22+
import { GEN_AI_DECIDE } from '@sentry/conventions/op';
1723
import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../../utils/runner';
1824
import { supports } from '../../../../utils/index';
1925

@@ -158,4 +164,103 @@ describe.runIf(supports({ min: 22 }))('OpenAI integration (V7)', () => {
158164
},
159165
},
160166
);
167+
168+
// The decisions API (`openai` >= 7.30.0) returns typed answers, so it is recorded like a Jev call.
169+
createEsmAndCjsTests(
170+
__dirname,
171+
'scenario-decisions.mjs',
172+
'instrument-with-recording.mjs',
173+
(createRunner, test) => {
174+
test('instruments the decisions API as decide spans', async () => {
175+
const runner = createRunner().ignore('event');
176+
const spansPromise = runner.collectStreamedSpansUntilSegment('main');
177+
178+
await runner.start().completed();
179+
180+
const spans = (await spansPromise).filter(span => span.attributes[SENTRY_OP]?.value === GEN_AI_DECIDE);
181+
expect(spans).toHaveLength(3);
182+
183+
// Both successful spans are named `decide gpt-6-luna` and may arrive in any order, so match on the input.
184+
const inputOf = (span: (typeof spans)[number]): string => String(span.attributes[GEN_AI_INPUT_MESSAGES]?.value);
185+
const textSpan = spans.find(span => inputOf(span).includes('charged twice'));
186+
const imageSpan = spans.find(span => inputOf(span).includes('input_image'));
187+
expect(textSpan!.name).toBe('decide gpt-6-luna');
188+
expect(textSpan!.attributes).toMatchObject({
189+
[SENTRY_ORIGIN]: { value: 'auto.ai.openai' },
190+
[GEN_AI_OPERATION_NAME]: { value: 'decide' },
191+
[GEN_AI_PROVIDER_NAME]: { value: 'openai' },
192+
[GEN_AI_REQUEST_MODEL]: { value: 'gpt-6-luna' },
193+
[GEN_AI_RESPONSE_MODEL]: { value: 'gpt-6-luna' },
194+
// OpenAI's input tokens already include the cached ones, so they are not added again.
195+
[GEN_AI_USAGE_INPUT_TOKENS]: { value: 382 },
196+
[GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS]: { value: 256 },
197+
[GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS]: { value: 64 },
198+
[GEN_AI_USAGE_OUTPUT_TOKENS]: { value: 0 },
199+
[GEN_AI_USAGE_REASONING_OUTPUT_TOKENS]: { value: 0 },
200+
[GEN_AI_USAGE_TOTAL_TOKENS]: { value: 382 },
201+
});
202+
expect(JSON.parse(textSpan!.attributes[GEN_AI_INPUT_MESSAGES]!.value as string)).toEqual([
203+
{
204+
type: 'evaluation',
205+
state: 'I was charged twice and want a refund.',
206+
questions: {
207+
wants_refund: { type: 'predicate', instructions: 'Is a refund requested?' },
208+
department: {
209+
type: 'choice',
210+
instructions: 'Which team should handle this?',
211+
// `other` has no description, so it gets an empty one.
212+
criteria: { billing: 'Charges and refunds', other: '' },
213+
},
214+
urgency: {
215+
type: 'score',
216+
instructions: 'How urgent is this ticket?',
217+
criteria: ['low', 'high'],
218+
// Keyed by score value; `low` has no description, so it is left out.
219+
descriptions: { 1: 'Blocks the customer' },
220+
},
221+
},
222+
},
223+
]);
224+
expect(JSON.parse(textSpan!.attributes[GEN_AI_OUTPUT_MESSAGES]!.value as string)).toEqual([
225+
{
226+
type: 'evaluation',
227+
answers: {
228+
wants_refund: { type: 'predicate', probability: 0.97 },
229+
department: {
230+
type: 'choice',
231+
choice: 'billing',
232+
probabilities: { billing: 0.82, other: 0.18 },
233+
confidence: 0.82,
234+
},
235+
urgency: {
236+
type: 'score',
237+
score: 0.95,
238+
probabilities: { 0: 0.05, 1: 0.95 },
239+
legend: { 0: 'low', 1: 'high' },
240+
confidence: 0.93,
241+
},
242+
},
243+
},
244+
]);
245+
246+
expect(JSON.parse(imageSpan!.attributes[GEN_AI_INPUT_MESSAGES]!.value as string)[0].state).toEqual([
247+
{
248+
role: 'user',
249+
content: [
250+
{ type: 'input_text', text: 'Is the package damaged?' },
251+
{ type: 'input_image', image_url: 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUg==' },
252+
],
253+
},
254+
]);
255+
256+
const errorSpan = spans.find(span => span.name === 'decide error-model');
257+
expect(errorSpan!.status).not.toBe('ok');
258+
});
259+
},
260+
{
261+
additionalDependencies: {
262+
openai: '7.30.0',
263+
},
264+
},
265+
);
161266
});

‎packages/server-utils/src/ai/openai/constants.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,8 @@ export const OPENAI_METHOD_REGISTRY = {
1212
// Conversations API - for conversation state management
1313
// https://platform.openai.com/docs/guides/conversation-state
1414
'conversations.create': { operation: 'chat' },
15+
// https://developers.openai.com/api/docs/guides/decisions
16+
'decisions.create': { operation: 'decide' },
1517
} as const satisfies InstrumentedMethodRegistry;
1618
export const RESPONSES_TOOL_CALL_EVENT_TYPES = [
1719
'response.output_item.added',

‎packages/server-utils/src/ai/openai/index.ts‎

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ import {
3131
import { OPENAI_METHOD_REGISTRY } from './constants';
3232
import { instrumentStream } from './streaming';
3333
import type { ChatCompletionChunk, OpenAiOptions, OpenAIStream, ResponseStreamingEvent } from './types';
34+
import { getDecisionInputMessages } from './mapping';
3435
import { addResponseAttributes, extractRequestParameters } from './utils';
3536

3637
/**
@@ -73,12 +74,15 @@ export function extractRequestAttributes(
7374
if (args.length > 0 && typeof args[0] === 'object' && args[0] !== null) {
7475
const params = args[0] as Record<string, unknown>;
7576

76-
const availableTools = recordInputs ? extractAvailableTools(params) : undefined;
77-
if (availableTools) {
78-
attributes[GEN_AI_TOOL_DEFINITIONS] = availableTools;
77+
if (recordInputs) {
78+
attributes[GEN_AI_TOOL_DEFINITIONS] = extractAvailableTools(params);
7979
}
8080

8181
Object.assign(attributes, extractRequestParameters(params));
82+
83+
if (recordInputs && operationName === 'decide') {
84+
attributes[GEN_AI_INPUT_MESSAGES] = getDecisionInputMessages(params);
85+
}
8286
} else {
8387
attributes[GEN_AI_REQUEST_MODEL] = 'unknown';
8488
}
@@ -88,6 +92,11 @@ export function extractRequestAttributes(
8892

8993
// Extract and record AI request inputs, if present. This is intentionally separate from response attributes.
9094
export function addRequestAttributes(span: Span, params: Record<string, unknown>, operationName: string): void {
95+
// A decision's input is recorded when its span starts, in `extractRequestAttributes`.
96+
if (operationName === 'decide') {
97+
return;
98+
}
99+
91100
// Store embeddings input on a separate attribute
92101
if (operationName === 'embeddings' && 'input' in params) {
93102
const input = params.input;
@@ -203,7 +212,7 @@ function instrumentMethod<T extends unknown[], R>(
203212
}
204213

205214
return originalResult.then(result => {
206-
addResponseAttributes(span, result, options.recordOutputs);
215+
addResponseAttributes(span, result, options.recordOutputs, operationName);
207216
return result;
208217
});
209218
});

0 commit comments

Comments
 (0)