|
1 | 1 | import type { SerializedStreamedSpanContainer } from '@sentry/core'; |
2 | 2 | import { afterAll, describe, expect } from 'vitest'; |
3 | 3 | import { |
| 4 | + GEN_AI_INPUT_MESSAGES, |
4 | 5 | GEN_AI_OPERATION_NAME, |
| 6 | + GEN_AI_OUTPUT_MESSAGES, |
5 | 7 | GEN_AI_PROVIDER_NAME, |
6 | 8 | GEN_AI_REQUEST_MODEL, |
7 | 9 | GEN_AI_RESPONSE_FINISH_REASONS, |
8 | 10 | GEN_AI_RESPONSE_ID, |
9 | 11 | GEN_AI_RESPONSE_MODEL, |
10 | 12 | GEN_AI_RESPONSE_STREAMING, |
| 13 | + GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS, |
| 14 | + GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS, |
11 | 15 | GEN_AI_USAGE_INPUT_TOKENS, |
12 | 16 | GEN_AI_USAGE_OUTPUT_TOKENS, |
| 17 | + GEN_AI_USAGE_REASONING_OUTPUT_TOKENS, |
13 | 18 | GEN_AI_USAGE_TOTAL_TOKENS, |
14 | 19 | SENTRY_OP, |
15 | 20 | SENTRY_ORIGIN, |
16 | 21 | } from '@sentry/conventions/attributes'; |
| 22 | +import { GEN_AI_DECIDE } from '@sentry/conventions/op'; |
17 | 23 | import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../../utils/runner'; |
18 | 24 | import { supports } from '../../../../utils/index'; |
19 | 25 |
|
@@ -158,4 +164,103 @@ describe.runIf(supports({ min: 22 }))('OpenAI integration (V7)', () => { |
158 | 164 | }, |
159 | 165 | }, |
160 | 166 | ); |
| 167 | + |
| 168 | + // The decisions API (`openai` >= 7.30.0) returns typed answers, so it is recorded like a Jev call. |
| 169 | + createEsmAndCjsTests( |
| 170 | + __dirname, |
| 171 | + 'scenario-decisions.mjs', |
| 172 | + 'instrument-with-recording.mjs', |
| 173 | + (createRunner, test) => { |
| 174 | + test('instruments the decisions API as decide spans', async () => { |
| 175 | + const runner = createRunner().ignore('event'); |
| 176 | + const spansPromise = runner.collectStreamedSpansUntilSegment('main'); |
| 177 | + |
| 178 | + await runner.start().completed(); |
| 179 | + |
| 180 | + const spans = (await spansPromise).filter(span => span.attributes[SENTRY_OP]?.value === GEN_AI_DECIDE); |
| 181 | + expect(spans).toHaveLength(3); |
| 182 | + |
| 183 | + // Both successful spans are named `decide gpt-6-luna` and may arrive in any order, so match on the input. |
| 184 | + const inputOf = (span: (typeof spans)[number]): string => String(span.attributes[GEN_AI_INPUT_MESSAGES]?.value); |
| 185 | + const textSpan = spans.find(span => inputOf(span).includes('charged twice')); |
| 186 | + const imageSpan = spans.find(span => inputOf(span).includes('input_image')); |
| 187 | + expect(textSpan!.name).toBe('decide gpt-6-luna'); |
| 188 | + expect(textSpan!.attributes).toMatchObject({ |
| 189 | + [SENTRY_ORIGIN]: { value: 'auto.ai.openai' }, |
| 190 | + [GEN_AI_OPERATION_NAME]: { value: 'decide' }, |
| 191 | + [GEN_AI_PROVIDER_NAME]: { value: 'openai' }, |
| 192 | + [GEN_AI_REQUEST_MODEL]: { value: 'gpt-6-luna' }, |
| 193 | + [GEN_AI_RESPONSE_MODEL]: { value: 'gpt-6-luna' }, |
| 194 | + // OpenAI's input tokens already include the cached ones, so they are not added again. |
| 195 | + [GEN_AI_USAGE_INPUT_TOKENS]: { value: 382 }, |
| 196 | + [GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS]: { value: 256 }, |
| 197 | + [GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS]: { value: 64 }, |
| 198 | + [GEN_AI_USAGE_OUTPUT_TOKENS]: { value: 0 }, |
| 199 | + [GEN_AI_USAGE_REASONING_OUTPUT_TOKENS]: { value: 0 }, |
| 200 | + [GEN_AI_USAGE_TOTAL_TOKENS]: { value: 382 }, |
| 201 | + }); |
| 202 | + expect(JSON.parse(textSpan!.attributes[GEN_AI_INPUT_MESSAGES]!.value as string)).toEqual([ |
| 203 | + { |
| 204 | + type: 'evaluation', |
| 205 | + state: 'I was charged twice and want a refund.', |
| 206 | + questions: { |
| 207 | + wants_refund: { type: 'predicate', instructions: 'Is a refund requested?' }, |
| 208 | + department: { |
| 209 | + type: 'choice', |
| 210 | + instructions: 'Which team should handle this?', |
| 211 | + // `other` has no description, so it gets an empty one. |
| 212 | + criteria: { billing: 'Charges and refunds', other: '' }, |
| 213 | + }, |
| 214 | + urgency: { |
| 215 | + type: 'score', |
| 216 | + instructions: 'How urgent is this ticket?', |
| 217 | + criteria: ['low', 'high'], |
| 218 | + // Keyed by score value; `low` has no description, so it is left out. |
| 219 | + descriptions: { 1: 'Blocks the customer' }, |
| 220 | + }, |
| 221 | + }, |
| 222 | + }, |
| 223 | + ]); |
| 224 | + expect(JSON.parse(textSpan!.attributes[GEN_AI_OUTPUT_MESSAGES]!.value as string)).toEqual([ |
| 225 | + { |
| 226 | + type: 'evaluation', |
| 227 | + answers: { |
| 228 | + wants_refund: { type: 'predicate', probability: 0.97 }, |
| 229 | + department: { |
| 230 | + type: 'choice', |
| 231 | + choice: 'billing', |
| 232 | + probabilities: { billing: 0.82, other: 0.18 }, |
| 233 | + confidence: 0.82, |
| 234 | + }, |
| 235 | + urgency: { |
| 236 | + type: 'score', |
| 237 | + score: 0.95, |
| 238 | + probabilities: { 0: 0.05, 1: 0.95 }, |
| 239 | + legend: { 0: 'low', 1: 'high' }, |
| 240 | + confidence: 0.93, |
| 241 | + }, |
| 242 | + }, |
| 243 | + }, |
| 244 | + ]); |
| 245 | + |
| 246 | + expect(JSON.parse(imageSpan!.attributes[GEN_AI_INPUT_MESSAGES]!.value as string)[0].state).toEqual([ |
| 247 | + { |
| 248 | + role: 'user', |
| 249 | + content: [ |
| 250 | + { type: 'input_text', text: 'Is the package damaged?' }, |
| 251 | + { type: 'input_image', image_url: 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUg==' }, |
| 252 | + ], |
| 253 | + }, |
| 254 | + ]); |
| 255 | + |
| 256 | + const errorSpan = spans.find(span => span.name === 'decide error-model'); |
| 257 | + expect(errorSpan!.status).not.toBe('ok'); |
| 258 | + }); |
| 259 | + }, |
| 260 | + { |
| 261 | + additionalDependencies: { |
| 262 | + openai: '7.30.0', |
| 263 | + }, |
| 264 | + }, |
| 265 | + ); |
161 | 266 | }); |
0 commit comments