diff --git a/packages/app/cypress/e2e/gpu-compare-agentic-detail.cy.ts b/packages/app/cypress/e2e/gpu-compare-agentic-detail.cy.ts index a5fd8f12..b95af169 100644 --- a/packages/app/cypress/e2e/gpu-compare-agentic-detail.cy.ts +++ b/packages/app/cypress/e2e/gpu-compare-agentic-detail.cy.ts @@ -1,4 +1,4 @@ -import { unlockAgenticGate } from '../support/e2e'; +import { interceptDerivedAgenticMetrics, unlockAgenticGate } from '../support/e2e'; // --------------------------------------------------------------------------- // Spec-scoped fixture helpers @@ -147,6 +147,9 @@ describe('GPU comparison agentic point detail', () => { ); request.reply({ body: result }); }); + // The agentic default x-axis mode (OSL / E2EL) fetches derived metrics on + // mount; without values every point drops out of the (remapped) data set. + interceptDerivedAgenticMetrics(); cy.visit('/inference?g_model=DeepSeek-V4-Pro&i_seq=agentic-traces&i_prec=fp4', { onBeforeLoad(win) { @@ -206,6 +209,7 @@ describe('GPU comparison agentic point detail', () => { 'agenticAvailability', ); cy.intercept('GET', '/api/v1/benchmarks*', { body: agenticBenchmarks }).as('agenticBenchmarks'); + interceptDerivedAgenticMetrics(); cy.visit( '/inference?g_model=DeepSeek-V4-Pro&i_seq=agentic-traces&i_prec=fp4&i_gpus=b200_sglang,b200_vllm&i_dates=2026-06-12&i_dstart=2026-06-12&i_dend=2026-06-12', diff --git a/packages/app/cypress/e2e/overlay-legend-remove.cy.ts b/packages/app/cypress/e2e/overlay-legend-remove.cy.ts index d0bc6e12..b8cef8c3 100644 --- a/packages/app/cypress/e2e/overlay-legend-remove.cy.ts +++ b/packages/app/cypress/e2e/overlay-legend-remove.cy.ts @@ -9,7 +9,7 @@ * appeared to do nothing. The legend toggle already had the overlay-aware * split (`unifiedToggle`); the X now shares it (`handleRemoveHwType`). */ -import { unlockAgenticGate } from '../support/e2e'; +import { interceptDerivedAgenticMetrics, unlockAgenticGate } from '../support/e2e'; import { countVisible, interceptOverlayRun, @@ -20,6 +20,10 @@ import { describe('Official legend X works while an unofficial overlay is loaded', () => { before(() => { interceptOverlayRun(); + // The agentic default mode is OSL / E2EL (which suppresses overlays and + // fetches derived metrics) — stub the fetch, then switch to Interactivity + // where the overlay renders. + interceptDerivedAgenticMetrics(); cy.visit(`/inference?unofficialrun=${OVERLAY_RUN_ID}&i_seq=agentic-traces&i_pctl=p90`, { onBeforeLoad(win) { win.localStorage.setItem('inferencex-star-modal-dismissed', String(Date.now())); @@ -27,6 +31,7 @@ describe('Official legend X works while an unofficial overlay is loaded', () => }, }); cy.wait('@unofficialRun'); + cy.get('[data-testid="x-axis-mode-interactivity"]').click(); cy.get('[data-testid="chart-figure"]').should('have.length.at.least', 1); cy.get('[data-testid="inference-chart-display"] svg .unofficial-overlay-pt').should( 'have.length', diff --git a/packages/app/cypress/e2e/overlay-optimal-only.cy.ts b/packages/app/cypress/e2e/overlay-optimal-only.cy.ts index c7c7a88a..9d4f4293 100644 --- a/packages/app/cypress/e2e/overlay-optimal-only.cy.ts +++ b/packages/app/cypress/e2e/overlay-optimal-only.cy.ts @@ -10,7 +10,7 @@ * dashed roofline (the monotone spline between C=8 and C=2 passes within * ~0.5% of it) while the official twin was hidden. */ -import { unlockAgenticGate } from '../support/e2e'; +import { interceptDerivedAgenticMetrics, unlockAgenticGate } from '../support/e2e'; import { countVisible, interceptOverlayRun, @@ -21,6 +21,10 @@ import { describe('Overlay points respect Optimal Only (agentic interactivity)', () => { before(() => { interceptOverlayRun(); + // The agentic default mode is OSL / E2EL (which suppresses overlays and + // fetches derived metrics) — stub the fetch, then switch to the + // Interactivity mode this suite is about. + interceptDerivedAgenticMetrics(); cy.visit(`/inference?unofficialrun=${OVERLAY_RUN_ID}&i_seq=agentic-traces&i_pctl=p90`, { onBeforeLoad(win) { win.localStorage.setItem('inferencex-star-modal-dismissed', String(Date.now())); @@ -28,6 +32,7 @@ describe('Overlay points respect Optimal Only (agentic interactivity)', () => { }, }); cy.wait('@unofficialRun'); + cy.get('[data-testid="x-axis-mode-interactivity"]').click(); cy.get('[data-testid="chart-figure"]').should('have.length.at.least', 1); cy.get('[data-testid="x-axis-mode-interactivity"]').should( 'have.attr', diff --git a/packages/app/cypress/e2e/ttft-x-axis-toggle.cy.ts b/packages/app/cypress/e2e/ttft-x-axis-toggle.cy.ts index 5023877e..4be1a6f1 100644 --- a/packages/app/cypress/e2e/ttft-x-axis-toggle.cy.ts +++ b/packages/app/cypress/e2e/ttft-x-axis-toggle.cy.ts @@ -1,26 +1,4 @@ -import { unlockAgenticGate } from '../support/e2e'; - -const interceptDerivedMetrics = () => { - cy.intercept('GET', '/api/v1/derived-agentic-metrics*', (request) => { - const ids = new URL(request.url).searchParams.get('ids')?.split(',').filter(Boolean) ?? []; - request.reply({ - body: Object.fromEntries( - ids.map((id, index) => [ - id, - { - id: Number(id), - normalized_session_time_s: 60 + index, - p90_prefill_tps_per_user: 100 + index, - p75_normalized_e2e_400_s: 8 + index, - p90_normalized_e2e_400_s: 12 + index, - p75_osl_per_e2el: 40 + index, - p90_osl_per_e2el: 25 + index, - }, - ]), - ), - }); - }).as('derivedAgenticMetrics'); -}; +import { interceptDerivedAgenticMetrics, unlockAgenticGate } from '../support/e2e'; // This spec exercises the agentic x-axis modes, which only exist when the // selected model resolves to the Agentic Traces scenario. The default e2e @@ -148,10 +126,9 @@ const interceptFixedSequenceData = () => { describe('X-Axis Mode Toggle (inference chart)', () => { before(() => { interceptAgenticData(); - // Stub derived metrics before the visit: if the agentic DEFAULT x-axis mode - // is (or becomes) a derived mode that fetches /derived-agentic-metrics on - // mount, the chart must not sit on its loading skeleton. - interceptDerivedMetrics(); + // The agentic default mode is OSL / E2EL, which fetches derived metrics + // on first render — stub them before the visit. + interceptDerivedAgenticMetrics(); cy.visit('/inference?i_seq=agentic-traces', { onBeforeLoad(win) { win.localStorage.setItem('inferencex-star-modal-dismissed', String(Date.now())); @@ -162,14 +139,32 @@ describe('X-Axis Mode Toggle (inference chart)', () => { cy.get('[data-testid="chart-figure"]').should('have.length.at.least', 1); }); - it('shows Interactivity by default for the agentic view', () => { + it('shows OSL / E2EL by default for the agentic view, as the leftmost option', () => { cy.get('[data-testid="scenario-selector"]').should('contain.text', 'Agentic Traces'); cy.get('[data-testid="x-axis-mode-ttft"]').should('be.visible'); cy.get('[data-testid="x-axis-mode-e2e"]').should('be.visible'); - cy.get('[data-testid="x-axis-mode-normalized-e2e"]').should('be.visible'); - cy.get('[data-testid="x-axis-mode-interactivity"]') + cy.get('[data-testid="x-axis-mode-interactivity"]').should('be.visible'); + cy.get('[data-testid="x-axis-mode-osl-e2el"]') .should('be.visible') .and('have.attr', 'aria-selected', 'true'); + // OSL / E2EL leads the mode list for agentic. + cy.get('[data-testid="x-axis-mode-buttons"] [role="tab"]') + .first() + .should('have.attr', 'data-testid', 'x-axis-mode-osl-e2el'); + cy.get('[data-testid="chart-figure"] h2').should('contain.text', 'P90 OSL / E2EL'); + cy.get('[data-testid="chart-figure"] svg').should( + 'contain.text', + 'P90 OSL / E2EL (tok/s/user)', + ); + }); + + it('switches to Interactivity and updates the heading', () => { + cy.get('[data-testid="x-axis-mode-interactivity"]').click(); + cy.get('[data-testid="x-axis-mode-interactivity"]').should( + 'have.attr', + 'aria-selected', + 'true', + ); cy.get('[data-testid="chart-figure"] h2').should('contain.text', 'Interactivity'); }); @@ -230,6 +225,9 @@ describe('X-Axis Mode Toggle (inference chart)', () => { it('honors explicit label URL overrides for the agentic view', () => { interceptAgenticData(); + // Fresh page load → fresh React Query cache → the default OSL / E2EL + // mode refetches derived metrics. + interceptDerivedAgenticMetrics(); cy.visit('/inference?i_seq=agentic-traces&i_label=0&i_advlabel=0&i_linelabel=1', { onBeforeLoad(win) { win.localStorage.setItem('inferencex-star-modal-dismissed', String(Date.now())); @@ -255,30 +253,27 @@ describe('X-Axis Mode Toggle (inference chart)', () => { cy.get('[data-testid="chart-figure"] svg').should('contain.text', 'P90 End-to-end Latency (s)'); }); - it('switches to request-level normalized E2E at 400 output tokens', () => { - interceptDerivedMetrics(); - cy.get('[data-testid="x-axis-mode-normalized-e2e"]').click(); - cy.wait('@derivedAgenticMetrics'); - cy.get('[data-testid="x-axis-mode-normalized-e2e"]').should( - 'have.attr', - 'aria-selected', - 'true', - ); - cy.get('[data-testid="chart-figure"] h2').should( - 'contain.text', - 'P90 Normalized E2E @ 400 output tokens', - ); + it('switches back to request-level OSL / E2EL', () => { + // No cy.wait here: the derived metrics were fetched (and stubbed) on the + // initial default-mode load and are still fresh in the React Query cache + // (staleTime 5 min), so re-entering the mode fires no new request. + cy.get('[data-testid="x-axis-mode-osl-e2el"]').click(); + cy.get('[data-testid="x-axis-mode-osl-e2el"]').should('have.attr', 'aria-selected', 'true'); + cy.get('[data-testid="chart-figure"] h2').should('contain.text', 'P90 OSL / E2EL'); cy.get('[data-testid="chart-figure"] svg').should( 'contain.text', - 'P90 Normalized E2E @ 400 output tokens (s)', + 'P90 OSL / E2EL (tok/s/user)', ); cy.get('[data-testid="percentile-selector"]').click(); cy.contains('[role="option"]', 'p75').click(); - cy.get('[data-testid="chart-figure"] h2').should( - 'contain.text', - 'P75 Normalized E2E @ 400 output tokens', - ); + cy.get('[data-testid="chart-figure"] h2').should('contain.text', 'P75 OSL / E2EL'); + + // The percentile selector is shared page state for the whole suite — + // restore the p90 default so later tests assert against a known value. + cy.get('[data-testid="percentile-selector"]').click(); + cy.contains('[role="option"]', 'p90').click(); + cy.get('[data-testid="chart-figure"] h2').should('contain.text', 'P90 OSL / E2EL'); }); it('switches back to Interactivity', () => { @@ -289,7 +284,18 @@ describe('X-Axis Mode Toggle (inference chart)', () => { 'true', ); cy.get('[data-testid="chart-figure"] h2').should('contain.text', 'Interactivity'); - // Percentile was switched to p75 in the previous test — the axis label follows. + cy.get('[data-testid="chart-figure"] svg').should( + 'contain.text', + 'P90 Interactivity (tok/s/user)', + ); + }); + + it('follows the percentile selector in the Interactivity axis label', () => { + // Select p75 here rather than inheriting it from another test — the axis + // label must track the selector on its own. + cy.get('[data-testid="x-axis-mode-interactivity"]').click(); + cy.get('[data-testid="percentile-selector"]').click(); + cy.contains('[role="option"]', 'p75').click(); cy.get('[data-testid="chart-figure"] svg').should( 'contain.text', 'P75 Interactivity (tok/s/user)', @@ -297,6 +303,33 @@ describe('X-Axis Mode Toggle (inference chart)', () => { }); }); +describe('X-axis mode URL param', () => { + // Regression: the reconcile effect used to run before availability resolved + // the sequence. It recorded the fixed-seq placeholder kind, then treated the + // switch to agentic as a user-driven kind change and clobbered the + // URL-restored mode with the agentic default (OSL / E2EL). + it('keeps a URL-restored mode through the agentic sequence resolving', () => { + interceptAgenticData(); + interceptDerivedAgenticMetrics(); + cy.visit('/inference?i_seq=agentic-traces&i_xmode=interactivity', { + onBeforeLoad(win) { + win.localStorage.setItem('inferencex-star-modal-dismissed', String(Date.now())); + unlockAgenticGate(win); + }, + }); + cy.get('[data-testid="scenario-selector"]').should('contain.text', 'Agentic Traces'); + cy.get('[data-testid="x-axis-mode-interactivity"]').should( + 'have.attr', + 'aria-selected', + 'true', + ); + // Assert on the rendered chart too: the clobber happened one tick after + // the buttons first painted, so a button-only check could pass too early. + cy.get('[data-testid="chart-figure"] h2').should('contain.text', 'Interactivity'); + cy.get('[data-testid="x-axis-mode-osl-e2el"]').should('have.attr', 'aria-selected', 'false'); + }); +}); + describe('Default scenario', () => { it('bare /inference defaults to 8K / 1K even when the model has agentic data', () => { // Availability contains BOTH agentic and fixed-seq rows for the default @@ -411,9 +444,8 @@ const interceptAgenticDataWithOverlay = () => { describe('X-Axis Mode Toggle — overlay path (finding #8 regression guard)', () => { before(() => { interceptAgenticDataWithOverlay(); - // Same as the main suite: keep the visit independent of the agentic default - // x-axis mode (a derived default fetches /derived-agentic-metrics on mount). - interceptDerivedMetrics(); + // Default agentic mode is OSL / E2EL → derived metrics fetch on mount. + interceptDerivedAgenticMetrics(); cy.visit(`/inference?unofficialrun=${OVERLAY_RUN_ID}&i_seq=agentic-traces`, { onBeforeLoad(win) { win.localStorage.setItem('inferencex-star-modal-dismissed', String(Date.now())); @@ -463,18 +495,14 @@ describe('X-Axis Mode Toggle — overlay path (finding #8 regression guard)', () }); }); - it('normalized-e2e mode shows suppression banner for unofficial-run overlays', () => { - interceptDerivedMetrics(); - cy.get('[data-testid="x-axis-mode-normalized-e2e"]').click(); - cy.get('[data-testid="x-axis-mode-normalized-e2e"]').should( - 'have.attr', - 'aria-selected', - 'true', - ); + it('osl-e2el mode shows suppression banner for unofficial-run overlays', () => { + // Derived metrics are cached from the initial default-mode load. + cy.get('[data-testid="x-axis-mode-osl-e2el"]').click(); + cy.get('[data-testid="x-axis-mode-osl-e2el"]').should('have.attr', 'aria-selected', 'true'); // The suppression message appears because isUnofficialRun is true and the - // mode is 'normalized-e2e' (documented in ChartDisplay.tsx ~line 640). + // mode is 'osl-e2el' (documented in ChartDisplay.tsx). cy.contains( - 'Normalized E2E requires persisted per-request traces, so unofficial-run overlays are unavailable for this experimental view.', + 'OSL / E2EL requires persisted per-request traces, so unofficial-run overlays are unavailable for this experimental view.', ).should('be.visible'); }); }); diff --git a/packages/app/cypress/support/e2e.ts b/packages/app/cypress/support/e2e.ts index 8d03599e..3ee554b3 100644 --- a/packages/app/cypress/support/e2e.ts +++ b/packages/app/cypress/support/e2e.ts @@ -37,3 +37,30 @@ export function unlockAgenticGate(win: Window): void { // agentic surfaces are public regardless. } } + +/** + * Stub `/api/v1/derived-agentic-metrics` with deterministic per-id values. + * + * OSL / E2EL is the DEFAULT x-axis mode for agentic scenarios, so any spec + * that loads the agentic view fires this fetch on mount — without a stub the + * fixture server has no DB and the chart sits on its loading skeleton until + * the query errors out. Values are index-stable so axis positions are + * deterministic. Register BEFORE `cy.visit`. + */ +export function interceptDerivedAgenticMetrics(): void { + cy.intercept('GET', '/api/v1/derived-agentic-metrics*', (request) => { + const ids = new URL(request.url).searchParams.get('ids')?.split(',').filter(Boolean) ?? []; + request.reply({ + body: Object.fromEntries( + ids.map((id, index) => [ + id, + { + id: Number(id), + p75_osl_per_e2el: 40 + index, + p90_osl_per_e2el: 25 + index, + }, + ]), + ), + }); + }).as('derivedAgenticMetrics'); +} diff --git a/packages/app/src/app/api/v1/derived-agentic-metrics/route.ts b/packages/app/src/app/api/v1/derived-agentic-metrics/route.ts index 373eb370..541fcfc6 100644 --- a/packages/app/src/app/api/v1/derived-agentic-metrics/route.ts +++ b/packages/app/src/app/api/v1/derived-agentic-metrics/route.ts @@ -35,12 +35,9 @@ const getCachedDerivedAgenticMetrics = cachedQuery( * * Returns per-id derived metrics computed live from the stored aiperf * profile_export.jsonl blobs: - * - normalized_session_time_s: mean across sessions of session e2e time - * (Σ per-turn request_latency) rescaled by mean_load / session_load. - * - p90_prefill_tps_per_user: P90 of per-turn prefill TPS/user (ISL / TTFT) - * across every turn in every session. - * - p75/p90_normalized_e2e_400_s: percentile of per-request - * TTFT + 399 × observed ITL. + * - p75/p90_osl_per_e2el: slow-tail OSL / E2E-latency in tok/s/user, + * computed as 1 / pXX(per-request E2EL/OSL) across every turn in every + * session — plain interactivity charged for the prefill wait. * * Ids without a trace_replay blob or with unparseable records are omitted. */ diff --git a/packages/app/src/components/inference/InferenceContext.tsx b/packages/app/src/components/inference/InferenceContext.tsx index 6d5a2b93..03fe1c44 100644 --- a/packages/app/src/components/inference/InferenceContext.tsx +++ b/packages/app/src/components/inference/InferenceContext.tsx @@ -611,7 +611,8 @@ export function InferenceProvider({ // Reconcile the x-axis mode with the scenario kind: // - On mount with no `i_xmode` URL param: snap to the kind's natural default - // (interactivity for both agentic and fixed-sequence scenarios). The state was initialized + // (OSL / E2EL for agentic — the "north star" e2e-interactivity view — + // and interactivity for fixed-sequence scenarios). The state was initialized // to a SSR-stable constant so server and client render the same DOM; this // effect fixes it up after hydration. // - When the user later switches sequence kinds: snap to the new kind's @@ -619,6 +620,11 @@ export function InferenceProvider({ // doesn't carry over). const lastSeqKindRef = useRef | null>(null); useEffect(() => { + // Wait for availability to resolve the sequence. Before it does, + // `effectiveSequence` is a fixed-seq placeholder; recording that kind here + // would make the later switch to agentic look like a user-driven kind + // change and clobber a URL-restored `i_xmode` with the kind's default. + if (!sequenceResolved) return; const kind = sequenceKind(effectiveSequence); const isInitialMount = lastSeqKindRef.current === null; const isAgenticOnlyMode = isAgenticOnlyXAxisMode(selectedXAxisMode); @@ -642,8 +648,8 @@ export function InferenceProvider({ // — fall through to the default snap below. return; } - handleSetXAxisMode('interactivity'); - }, [effectiveSequence, selectedXAxisMode, handleSetXAxisMode]); + handleSetXAxisMode(kind === 'agentic' ? 'osl-e2el' : 'interactivity'); + }, [sequenceResolved, effectiveSequence, selectedXAxisMode, handleSetXAxisMode]); // Reconcile selectedE2eXAxisMetric whenever the mode, sequence kind, or // agentic percentile changes. For fixed-seq the JSONB only carries diff --git a/packages/app/src/components/inference/hooks/useChartData.test.ts b/packages/app/src/components/inference/hooks/useChartData.test.ts index fe045212..e0012863 100644 --- a/packages/app/src/components/inference/hooks/useChartData.test.ts +++ b/packages/app/src/components/inference/hooks/useChartData.test.ts @@ -1,10 +1,13 @@ import { describe, it, expect } from 'vitest'; +import chartDefinitions from '@/components/inference/inference-chart-config.json'; + import { applyAgenticPercentileToXLabel, buildComparisonDates, dedupeRowsToLatestPerConfig, filterByGPU, + derivedModeRoofline, flipRooflineDirection, } from './useChartData'; @@ -200,3 +203,42 @@ describe('flipRooflineDirection', () => { } }); }); + +describe('derived higher-is-better x-axis rooflines', () => { + // The OSL / E2EL mode renders on the e2e chart definition (lower-x-is-better) + // but its x-axis is higher-is-better, like interactivity. ChartDisplay + // therefore mirrors each configured e2e corner horizontally rather than + // hardcoding one corner — hardcoding `upper_left` inverted the frontier for + // cost and joules metrics, whose good direction is a LOWER corner. + it('mirroring the e2e corner reproduces the interactivity corner for every y-metric', () => { + const defs = chartDefinitions as Record[]; + const e2e = defs.find((d) => d.chartType === 'e2e')!; + const interactivity = defs.find((d) => d.chartType === 'interactivity')!; + + const rooflineKeys = Object.keys(e2e).filter((k) => k.endsWith('_roofline')); + expect(rooflineKeys.length).toBeGreaterThan(0); + + for (const key of rooflineKeys) { + const e2eCorner = e2e[key] as Parameters[0]; + expect( + derivedModeRoofline(e2eCorner, true), + `${key}: mirrored e2e corner must match the interactivity chart`, + ).toBe(interactivity[key]); + } + }); + + it('covers the cost metrics Bugbot flagged, not just throughput', () => { + const defs = chartDefinitions as Record[]; + const e2e = defs.find((d) => d.chartType === 'e2e')!; + // Throughput wants an upper corner, cost a lower one — a single hardcoded + // corner cannot serve both. + expect(derivedModeRoofline(e2e.y_tpPerGpu_roofline as 'upper_right', true)).toBe('upper_left'); + expect(derivedModeRoofline(e2e.y_costh_roofline as 'lower_left', true)).toBe('lower_right'); + expect(derivedModeRoofline(e2e.y_jTotal_roofline as 'lower_left', true)).toBe('lower_right'); + }); + + it('leaves the corner alone for a lower-is-better derived metric', () => { + expect(derivedModeRoofline('upper_right', false)).toBe('upper_right'); + expect(derivedModeRoofline(undefined, true)).toBeUndefined(); + }); +}); diff --git a/packages/app/src/components/inference/hooks/useChartData.ts b/packages/app/src/components/inference/hooks/useChartData.ts index c125e9e9..0af4412e 100644 --- a/packages/app/src/components/inference/hooks/useChartData.ts +++ b/packages/app/src/components/inference/hooks/useChartData.ts @@ -41,22 +41,9 @@ import { * the single definition — InferenceContext (URL/state) and ChartDisplay * (buttons, derived-metric remapping) import it from here. */ -export type XAxisMode = - | 'ttft' - | 'e2e' - | 'normalized-e2e' - | 'interactivity' - | 'session-time' - | 'prefill-tps'; - -export const X_AXIS_MODES: readonly XAxisMode[] = [ - 'ttft', - 'e2e', - 'normalized-e2e', - 'interactivity', - 'session-time', - 'prefill-tps', -]; +export type XAxisMode = 'ttft' | 'e2e' | 'interactivity' | 'osl-e2el'; + +export const X_AXIS_MODES: readonly XAxisMode[] = ['ttft', 'e2e', 'interactivity', 'osl-e2el']; /** * Modes whose x metric is derived from persisted per-request traces — @@ -64,14 +51,14 @@ export const X_AXIS_MODES: readonly XAxisMode[] = [ * trace_replay blob to derive them from). */ export function isAgenticOnlyXAxisMode(mode: XAxisMode): boolean { - return mode === 'normalized-e2e' || mode === 'session-time' || mode === 'prefill-tps'; + return mode === 'osl-e2el'; } /** * Compute the set of benchmark_results.id values that sit on the * (e2e_latency, y) Pareto frontier within each (hwKey, precision, date) * group. Used to restrict the non-e2e xmode charts (ttft, interactivity, - * session-time, prefill-tps) so they show *only* the points that win on + * osl-e2el) so they show *only* the points that win on * end-to-end latency — preventing benchmark-hacking where a config tops * one axis while tanking the other. * @@ -126,7 +113,7 @@ export function filterByGPU( }); } -type RooflineDirection = 'upper_left' | 'upper_right' | 'lower_left' | 'lower_right'; +export type RooflineDirection = 'upper_left' | 'upper_right' | 'lower_left' | 'lower_right'; const FLIP_MAP: Record = { upper_left: 'upper_right', upper_right: 'upper_left', @@ -139,6 +126,22 @@ export function flipRooflineDirection(dir: RooflineDirection): RooflineDirection return FLIP_MAP[dir]; } +/** + * Roofline corner for a trace-derived x-axis mode. Derived modes render on the + * e2e chart definition, whose corners assume lower-x-is-better; when the + * derived metric is higher-is-better (OSL / E2EL) the corner mirrors + * horizontally. This keeps the y-metric's own good direction — throughput + * lands on an upper corner, cost and joules on a lower one — where hardcoding + * a single corner inverted the frontier for the cost metrics. + */ +export function derivedModeRoofline( + configuredE2eCorner: RooflineDirection | undefined, + higherXIsBetter: boolean, +): RooflineDirection | undefined { + if (!configuredE2eCorner || !higherXIsBetter) return configuredE2eCorner; + return flipRooflineDirection(configuredE2eCorner); +} + // Statistic words that may already prefix an x-axis label (from chart config // or the TTFT override label). Trailing whitespace is consumed so a replace // never doubles the separator space. @@ -220,7 +223,7 @@ export function useChartData( /** * Current x-axis mode. When set to anything other than 'e2e', the displayed * data is filtered to the (e2e-latency, y) Pareto frontier so the ttft / - * interactivity / session-time / prefill-tps charts show only points that + * interactivity / osl-e2el charts show only points that * also win on end-to-end latency — preventing benchmark-hacking where a * config tops one metric while tanking the other. The 'e2e' mode is the * source of truth and keeps the full point set. diff --git a/packages/app/src/components/inference/types.ts b/packages/app/src/components/inference/types.ts index 8bdc0433..96e6eb0a 100644 --- a/packages/app/src/components/inference/types.ts +++ b/packages/app/src/components/inference/types.ts @@ -258,7 +258,7 @@ export interface InferenceData extends Partial void; + selectedXAxisMode: 'ttft' | 'e2e' | 'interactivity' | 'osl-e2el'; + setSelectedXAxisMode: (mode: 'ttft' | 'e2e' | 'interactivity' | 'osl-e2el') => void; scaleType: 'auto' | 'linear' | 'log'; setScaleType: (type: 'auto' | 'linear' | 'log') => void; /** Coarse vendor / framework / agg-disagg / mtp-stp filters applied to the chart point set. */ diff --git a/packages/app/src/components/inference/ui/ChartDisplay.tsx b/packages/app/src/components/inference/ui/ChartDisplay.tsx index 76d3e6f4..9ca3545a 100644 --- a/packages/app/src/components/inference/ui/ChartDisplay.tsx +++ b/packages/app/src/components/inference/ui/ChartDisplay.tsx @@ -1,8 +1,5 @@ 'use client'; -import { - DISPLAY_MODEL_TO_DB, - NORMALIZED_E2E_OUTPUT_TOKENS, -} from '@semianalysisai/inferencex-constants'; +import { DISPLAY_MODEL_TO_DB } from '@semianalysisai/inferencex-constants'; import { track } from '@/lib/analytics'; import dynamic from 'next/dynamic'; import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; @@ -63,7 +60,12 @@ import { useDerivedAgenticMetrics, type DerivedAgenticMetric, } from '@/hooks/api/use-derived-agentic-metrics'; -import { isAgenticOnlyXAxisMode, type XAxisMode } from '@/components/inference/hooks/useChartData'; +import { + derivedModeRoofline, + isAgenticOnlyXAxisMode, + type RooflineDirection, + type XAxisMode, +} from '@/components/inference/hooks/useChartData'; import { isPersistedBenchmarkId } from '@/lib/benchmark-id'; import { useTrendData } from '@/components/inference/hooks/useTrendData'; import { getHardwareConfig, hardwareKeyMatchesAnyBase } from '@/lib/constants'; @@ -95,8 +97,8 @@ const STRINGS = { sourceUnofficial: 'Source: UNOFFICIAL', sourceOfficial: 'Source: SemiAnalysis InferenceX™', updated: 'Updated:', - normalizedE2eDisclaimer: - 'Normalized E2E requires persisted per-request traces, so unofficial-run overlays are unavailable for this experimental view.', + oslE2elDisclaimer: + 'OSL / E2EL requires persisted per-request traces, so unofficial-run overlays are unavailable for this experimental view.', selectDateRange: 'Select a date range or add a run to view GPU comparison', performanceOverTime: 'Performance Over Time', performanceOverTimeDesc: @@ -114,8 +116,8 @@ const STRINGS = { sourceUnofficial: '来源:非官方', sourceOfficial: '来源:SemiAnalysis InferenceX™', updated: '更新时间:', - normalizedE2eDisclaimer: - 'Normalized E2E 需要持久化的逐请求 trace 数据,因此该实验性视图不支持非官方运行覆盖。', + oslE2elDisclaimer: + 'OSL / E2EL 需要持久化的逐请求 trace 数据,因此该实验性视图不支持非官方运行覆盖。', selectDateRange: '请选择日期范围或添加运行以查看 GPU 对比', performanceOverTime: '性能趋势', performanceOverTimeDesc: '双击散点图上的数据点以追踪配置随时间的变化。', @@ -145,13 +147,13 @@ function zhHeading(configured: string): string { return `vs. ${pctl ? `${pctl} ` : ''}${subjectZh}`; } +// OSL / E2EL leads: it's the agentic default (and agentic-only, so the +// filter below drops it for fixed-seq, leaving Interactivity first there). const X_AXIS_MODE_BUTTONS: { value: XAxisMode; label: string; labelZh: string }[] = [ + { value: 'osl-e2el', label: 'OSL / E2EL', labelZh: 'OSL / E2EL' }, { value: 'interactivity', label: 'Interactivity', labelZh: '交互性' }, { value: 'e2e', label: 'E2E Latency', labelZh: '端到端延迟' }, { value: 'ttft', label: 'TTFT', labelZh: 'TTFT' }, - { value: 'normalized-e2e', label: 'Normalized E2E', labelZh: 'Normalized E2E' }, - { value: 'session-time', label: 'Session Time', labelZh: '会话时长' }, - { value: 'prefill-tps', label: 'Prefill TPS / user', labelZh: 'Prefill TPS / user' }, ]; /** @@ -168,7 +170,13 @@ interface DerivedXModeSpec { heading: (percentileLabel: string) => string; /** Chinese heading suffix; omit to reuse the English one. */ headingZh?: (percentileLabel: string) => string; - rooflineCorner: 'upper_right' | 'upper_left'; + /** + * True when higher x is better. Derived modes render on the e2e chart + * definition, whose corners assume lower-x-is-better, so a higher-is-better + * metric mirrors each configured corner horizontally instead of hardcoding + * one corner for every y-metric (which inverted cost/joules frontiers). + */ + higherXIsBetter: boolean; /** Pull the raw metric for this mode off the derived-metrics payload. */ value: (m: DerivedAgenticMetric | undefined, percentile: string) => number | null | undefined; /** Convert the raw metric to the plotted x value. */ @@ -176,30 +184,15 @@ interface DerivedXModeSpec { } const DERIVED_X_MODE_SPECS: Partial> = { - 'session-time': { - xLabel: () => 'Mean Normalized Session Time (min)', - xLabelZh: () => '平均归一化会话时长(min)', - heading: () => 'vs. Mean Normalized Session Time', - headingZh: () => 'vs. 平均归一化会话时长', - rooflineCorner: 'upper_right', - value: (m) => m?.normalized_session_time_s, - toX: (raw) => raw / 60, - }, - 'normalized-e2e': { - xLabel: (pctl) => `${pctl} Normalized E2E @ ${NORMALIZED_E2E_OUTPUT_TOKENS} output tokens (s)`, - xLabelZh: (pctl) => `${pctl} Normalized E2E @ ${NORMALIZED_E2E_OUTPUT_TOKENS} 输出 token(s)`, - heading: (pctl) => `vs. ${pctl} Normalized E2E @ ${NORMALIZED_E2E_OUTPUT_TOKENS} output tokens`, - headingZh: (pctl) => `vs. ${pctl} Normalized E2E @ ${NORMALIZED_E2E_OUTPUT_TOKENS} 输出 token`, - rooflineCorner: 'upper_right', - value: (m, percentile) => - percentile === 'p75' ? m?.p75_normalized_e2e_400_s : m?.p90_normalized_e2e_400_s, - toX: (raw) => raw, - }, - 'prefill-tps': { - xLabel: () => 'P90 Prefill TPS per user (tok/s)', - heading: () => 'vs. P90 Prefill TPS / user', - rooflineCorner: 'upper_left', - value: (m) => m?.p90_prefill_tps_per_user, + // "E2E interactivity": per-request OSL / E2E latency — the rate at which + // the user receives output tokens INCLUDING the prefill wait. Slow-tail + // percentiles (pXX = 1/pXX(E2EL/OSL)), matching the *_intvty convention. + 'osl-e2el': { + xLabel: (pctl) => `${pctl} OSL / E2EL (tok/s/user)`, + heading: (pctl) => `vs. ${pctl} OSL / E2EL`, + headingZh: (pctl) => `vs. ${pctl} OSL / 端到端延迟`, + higherXIsBetter: true, + value: (m, percentile) => (percentile === 'p75' ? m?.p75_osl_per_e2el : m?.p90_osl_per_e2el), toX: (raw) => raw, }, }; @@ -662,12 +655,16 @@ export default function ChartDisplay() { locale === 'zh' && derivedSpec.xLabelZh ? derivedSpec.xLabelZh : derivedSpec.xLabel; const xLabel = xLabelFn(selectedPercentile.toUpperCase()); return visibleGraphs.map((graph) => { + const rooflineKey = `${selectedYAxisMetric}_roofline` as keyof typeof graph.chartDefinition; + const corner = derivedModeRoofline( + graph.chartDefinition[rooflineKey] as RooflineDirection | undefined, + derivedSpec.higherXIsBetter, + ); const chartDefinition = { ...graph.chartDefinition, x_label: xLabel, y_latency_limit: undefined, - [`${selectedYAxisMetric}_roofline` as keyof typeof graph.chartDefinition]: - derivedSpec.rooflineCorner, + ...(corner ? { [rooflineKey]: corner } : {}), }; const data = graph.data .map((point) => { @@ -845,9 +842,9 @@ export default function ChartDisplay() { )}

- {isUnofficialRun && selectedXAxisMode === 'normalized-e2e' && ( + {isUnofficialRun && selectedXAxisMode === 'osl-e2el' && (

- {t.normalizedE2eDisclaimer} + {t.oslE2elDisclaimer}

)} diff --git a/packages/app/src/components/inference/utils.test.ts b/packages/app/src/components/inference/utils.test.ts index 35e48ad2..f24b4d5e 100644 --- a/packages/app/src/components/inference/utils.test.ts +++ b/packages/app/src/components/inference/utils.test.ts @@ -10,8 +10,8 @@ import { describe('selectUnofficialOverlayForMode', () => { const overlays = { e2e: { id: 'e2e' }, interactivity: { id: 'interactivity' } }; - it('suppresses raw unofficial E2E data for normalized E2E mode', () => { - expect(selectUnofficialOverlayForMode('normalized-e2e', 'e2e', overlays)).toBeNull(); + it('suppresses raw unofficial E2E data for OSL / E2EL mode', () => { + expect(selectUnofficialOverlayForMode('osl-e2el', 'e2e', overlays)).toBeNull(); }); it('preserves matching unofficial overlays for supported modes', () => { diff --git a/packages/app/src/components/inference/utils.ts b/packages/app/src/components/inference/utils.ts index 2b8074d9..62244d4d 100644 --- a/packages/app/src/components/inference/utils.ts +++ b/packages/app/src/components/inference/utils.ts @@ -11,16 +11,17 @@ import { resolveXAxisField } from '@/components/inference/utils/resolveXAxisFiel import type { ChartDefinition, InferenceData, YAxisMetricKey } from './types'; /** - * Select the matching unofficial-run overlay for a chart mode. Normalized E2E + * Select the matching unofficial-run overlay for a chart mode. OSL / E2EL * is intentionally excluded: unofficial benchmark rows do not include the - * persisted per-request trace needed to normalize before taking percentiles. + * persisted per-request trace needed to derive the per-request ratio + * percentiles. */ export function selectUnofficialOverlayForMode( xAxisMode: string, chartType: 'e2e' | 'interactivity', overlays: { e2e: T | null; interactivity: T | null }, ): T | null { - if (xAxisMode === 'normalized-e2e') return null; + if (xAxisMode === 'osl-e2el') return null; return overlays[chartType]; } diff --git a/packages/app/src/components/inference/utils/e2eFrontier.ts b/packages/app/src/components/inference/utils/e2eFrontier.ts index 588afdd3..a7c8ca29 100644 --- a/packages/app/src/components/inference/utils/e2eFrontier.ts +++ b/packages/app/src/components/inference/utils/e2eFrontier.ts @@ -2,7 +2,7 @@ * @file e2eFrontier.ts * @description Shared seed for the anti-benchmark-hacking roofline restriction. * - * On the non-e2e xmode charts (interactivity, ttft, session-time, prefill-tps) + * On the non-e2e xmode charts (interactivity, ttft, osl-e2el) * the roofline is restricted to the configs that ALSO win on end-to-end latency, * so a config can't top interactivity while tanking decode (or vice versa). Both * the official path (`useChartData` → benchmark ids → `isOnE2eFrontier`) and the diff --git a/packages/app/src/hooks/api/use-derived-agentic-metrics.ts b/packages/app/src/hooks/api/use-derived-agentic-metrics.ts index 388563d9..737c41b8 100644 --- a/packages/app/src/hooks/api/use-derived-agentic-metrics.ts +++ b/packages/app/src/hooks/api/use-derived-agentic-metrics.ts @@ -2,16 +2,11 @@ import { bulkIdsFetcher, useBulkIdsQuery } from './benchmark-id-query'; export interface DerivedAgenticMetric { id: number; - /** Mean across sessions of e2e time (Σ per-turn request_latency) rescaled - * by mean_load / session_load. Null when the JSONL had no usable records. */ - normalized_session_time_s: number | null; - /** P90 of per-turn ISL/TTFT across every turn in every session. - * Null when no prefill rates could be computed. */ - p90_prefill_tps_per_user: number | null; - /** P75 normalized per-request E2E at a fixed 400-token output length. */ - p75_normalized_e2e_400_s: number | null; - /** P90 normalized per-request E2E at a fixed 400-token output length. */ - p90_normalized_e2e_400_s: number | null; + /** Slow-tail P75 OSL/E2EL in tok/s/user — 1 / p75(per-request E2EL/OSL). + * Null when the JSONL had no usable records. */ + p75_osl_per_e2el: number | null; + /** Slow-tail P90 OSL/E2EL in tok/s/user — 1 / p90(per-request E2EL/OSL). */ + p90_osl_per_e2el: number | null; } export type DerivedAgenticMetricMap = Record; @@ -43,9 +38,9 @@ async function fetchDerivedAgenticMetrics( } /** - * Fetch per-id derived agentic metrics (session time + p90 prefill TPS/user) + * Fetch per-id derived agentic metrics (slow-tail OSL/E2EL tok/s/user) * computed live from the stored aiperf profile_export.jsonl. Used to drive - * the "Session Time" and "Prefill TPS/user" chart variants. + * the "OSL / E2EL" chart variant. * * Ids without a trace_replay blob (older or non-aiperf agentic runs) are * silently omitted from the response. diff --git a/packages/app/src/lib/benchmark-id.ts b/packages/app/src/lib/benchmark-id.ts index b1ccb8bc..6b1030df 100644 --- a/packages/app/src/lib/benchmark-id.ts +++ b/packages/app/src/lib/benchmark-id.ts @@ -8,7 +8,7 @@ * * A bare `typeof id === 'number'` check is NOT enough: `NaN` and `0` are both * `number` yet neither is a real row. Passing them to the id-keyed endpoints - * (`/api/v1/derived-agentic-metrics?ids=…`, `…?id=…`) yields a 400 (the routes + * (`/api/v1/agentic-aggregates?ids=…`, `…?id=…`) yields a 400 (the routes * filter to `Number.isFinite(n) && n > 0`), and building an * `/inference/agentic/` link out of one points at a non-existent row. * diff --git a/packages/constants/src/agentic.ts b/packages/constants/src/agentic.ts deleted file mode 100644 index 42eab306..00000000 --- a/packages/constants/src/agentic.ts +++ /dev/null @@ -1,2 +0,0 @@ -/** Fixed output length used by the experimental normalized-E2E chart metric. */ -export const NORMALIZED_E2E_OUTPUT_TOKENS = 400; diff --git a/packages/constants/src/index.ts b/packages/constants/src/index.ts index 7d3d6783..e767e500 100644 --- a/packages/constants/src/index.ts +++ b/packages/constants/src/index.ts @@ -1,4 +1,3 @@ -export * from './agentic'; export * from './framework-aliases'; export * from './github'; export * from './gpu-keys'; diff --git a/packages/db/src/backfill-aggregate-stats.ts b/packages/db/src/backfill-aggregate-stats.ts index 8aaa1214..e7981b92 100644 --- a/packages/db/src/backfill-aggregate-stats.ts +++ b/packages/db/src/backfill-aggregate-stats.ts @@ -94,12 +94,16 @@ async function main(): Promise { } let stats: AggregateStats; - if (row.aggregate_stats?.version === 3) { + // v3 onwards → current is a profile-only change (the server-derived fields + // haven't changed since v3), so skip re-reading the huge server blob and + // carry its KV/prefix distributions forward. + const storedVersion = row.aggregate_stats?.version; + if (storedVersion !== undefined && storedVersion >= 3 && storedVersion < STATS_VERSION) { const profileStats = await computeAggregateStats({ profileBlob: row.profile_export_jsonl_gz, serverBlob: null, }); - stats = mergeProfileStatsUpgrade(row.aggregate_stats, profileStats); + stats = mergeProfileStatsUpgrade(row.aggregate_stats!, profileStats); } else { const [serverRow] = await sql<{ server_metrics_json_gz: Buffer | null }[]>` select server_metrics_json_gz diff --git a/packages/db/src/etl/compute-aggregate-stats.test.ts b/packages/db/src/etl/compute-aggregate-stats.test.ts index 7b745c09..b08f46b5 100644 --- a/packages/db/src/etl/compute-aggregate-stats.test.ts +++ b/packages/db/src/etl/compute-aggregate-stats.test.ts @@ -66,12 +66,10 @@ describe('computeAggregateStats', () => { expect(stats.osl).toBeNull(); expect(stats.kvCacheUtil).toBeNull(); expect(stats.prefixCacheHitRate).toBeNull(); - expect(stats.normalizedSessionTimeS).toBeNull(); - expect(stats.p90PrefillTpsPerUser).toBeNull(); - expect(stats.normalizedE2e400).toBeNull(); + expect(stats.e2elPerOsl).toBeNull(); }); - it('computes ISL/OSL percentiles + derived metrics from the profile blob', async () => { + it('computes ISL/OSL percentiles + the E2EL/OSL ratio bundle from the profile blob', async () => { const profileBlob = makeProfileBlob([ { isl: 100, osl: 50, rl: 1000, ttft: 100 }, { isl: 200, osl: 75, rl: 2000, ttft: 200 }, @@ -88,15 +86,11 @@ describe('computeAggregateStats', () => { expect(stats.kvCacheUtil).toBeNull(); expect(stats.prefixCacheHitRate).toBeNull(); - // Derived: prefill TPS per turn = isl / (ttft/1000) = 1000 for each, so p90 = 1000. - expect(stats.p90PrefillTpsPerUser).toBeCloseTo(1000, 6); - // Normalized session time: T̃_i = T_i × (mean_load / load_i), then mean. - // loads = [150, 275, 400], mean_load = 275 - // scaled times (s) = [1×275/150, 2×275/275, 3×275/400] = [1.8333, 2, 2.0625] - // mean ≈ 1.9653 - expect(stats.normalizedSessionTimeS).toBeCloseTo(1.9653, 3); - expect(stats.normalizedE2e400?.n).toBe(3); - expect(stats.normalizedE2e400?.p90).toBeGreaterThan(0); + // Per-request E2EL/OSL ratios (s/tok): 1/50=0.02, 2/75≈0.02667, 3/100=0.03. + expect(stats.e2elPerOsl?.n).toBe(3); + expect(stats.e2elPerOsl?.p50).toBeCloseTo(2 / 75, 6); + // p90 of 3 values (linear interpolation): pos=1.8 → 0.02667 + 0.8×(0.03-0.02667) + expect(stats.e2elPerOsl?.p90).toBeCloseTo(2 / 75 + 0.8 * (0.03 - 2 / 75), 6); }); it('computes KV util + prefix hit rate from the server blob alone', async () => { @@ -112,9 +106,7 @@ describe('computeAggregateStats', () => { // Profile-derived metrics absent. expect(stats.isl).toBeNull(); expect(stats.osl).toBeNull(); - expect(stats.normalizedSessionTimeS).toBeNull(); - expect(stats.p90PrefillTpsPerUser).toBeNull(); - expect(stats.normalizedE2e400).toBeNull(); + expect(stats.e2elPerOsl).toBeNull(); }); it('tolerates a malformed profile blob by leaving its metrics null', async () => { @@ -123,9 +115,7 @@ describe('computeAggregateStats', () => { const stats = await computeAggregateStats({ profileBlob: garbage, serverBlob: null }); expect(stats.isl).toBeNull(); expect(stats.osl).toBeNull(); - expect(stats.normalizedSessionTimeS).toBeNull(); - expect(stats.p90PrefillTpsPerUser).toBeNull(); - expect(stats.normalizedE2e400).toBeNull(); + expect(stats.e2elPerOsl).toBeNull(); // Version still set so the row is considered "computed". expect(stats.version).toBe(STATS_VERSION); }); @@ -145,8 +135,34 @@ describe('mergeProfileStatsUpgrade', () => { const merged = mergeProfileStatsUpgrade(existing, profile); expect(merged.version).toBe(STATS_VERSION); expect(merged.isl?.mean).toBe(100); - expect(merged.normalizedE2e400?.p90).toBeGreaterThan(0); + expect(merged.e2elPerOsl?.p90).toBeCloseTo(2.08 / 100, 6); expect(merged.kvCacheUtil).toEqual(existing.kvCacheUtil); expect(merged.prefixCacheHitRate).toEqual(existing.prefixCacheHitRate); }); + + it('drops legacy derived fields when upgrading a pre-v6 bundle', async () => { + // A stale bundle from an older STATS_VERSION carries since-retired fields — + // the merged result must not resurrect them. + const legacy = { + version: STATS_VERSION - 1, + isl: null, + osl: null, + kvCacheUtil: { mean: 0.4, p50: 0.4, p75: 0.5, p90: 0.6, p99: 0.7, n: 3 }, + prefixCacheHitRate: null, + normalizedSessionTimeS: 999, + p90PrefillTpsPerUser: 999, + normalizedE2e400: { mean: 1, p50: 1, p75: 1, p90: 2, p99: 3, n: 5 }, + }; + const profile = await computeAggregateStats({ + profileBlob: makeProfileBlob([{ isl: 100, osl: 100, rl: 2080, ttft: 100 }]), + serverBlob: null, + }); + + const merged = mergeProfileStatsUpgrade(legacy, profile); + expect(merged.version).toBe(STATS_VERSION); + expect(merged.kvCacheUtil).toEqual(legacy.kvCacheUtil); + expect(merged).not.toHaveProperty('normalizedSessionTimeS'); + expect(merged).not.toHaveProperty('p90PrefillTpsPerUser'); + expect(merged).not.toHaveProperty('normalizedE2e400'); + }); }); diff --git a/packages/db/src/etl/compute-aggregate-stats.ts b/packages/db/src/etl/compute-aggregate-stats.ts index f0385f6e..1ede7cfb 100644 --- a/packages/db/src/etl/compute-aggregate-stats.ts +++ b/packages/db/src/etl/compute-aggregate-stats.ts @@ -29,12 +29,25 @@ export interface AggregateStats { osl: MetricPercentiles | null; kvCacheUtil: MetricPercentiles | null; prefixCacheHitRate: MetricPercentiles | null; - /** Mean of (per-session e2e time × mean_load / session_load) across sessions. */ - normalizedSessionTimeS: number | null; - /** P90 of per-turn ISL/TTFT pooled across every session's turns. */ - p90PrefillTpsPerUser: number | null; - /** Per-request normalized E2E distribution at a fixed 400-token OSL. */ - normalizedE2e400: MetricPercentiles | null; + /** + * Per-request E2E latency / OSL (seconds per output token) percentiles. + * The read path inverts to plot the slow-tail "OSL / E2EL" x-axis metric + * (tok/s/user): pXX OSL/E2EL = 1 / pXX(E2EL/OSL). + */ + e2elPerOsl: MetricPercentiles | null; +} + +/** + * The subset of an older-version bundle a profile-only upgrade carries + * forward. Pre-v6 bundles also carry since-retired derived fields + * (normalizedSessionTimeS, p90PrefillTpsPerUser, normalizedE2e400) — spreading + * `profile` first drops them from the merged result. + */ +interface ProfileUpgradeCarryover { + isl: MetricPercentiles | null; + osl: MetricPercentiles | null; + kvCacheUtil: MetricPercentiles | null; + prefixCacheHitRate: MetricPercentiles | null; } /** @@ -43,17 +56,13 @@ export interface AggregateStats { * while preserving its already-computed KV/cache distributions. */ export function mergeProfileStatsUpgrade( - existing: Omit & { - normalizedE2e400?: MetricPercentiles | null; - }, + existing: ProfileUpgradeCarryover, profile: AggregateStats, ): AggregateStats { return { ...profile, isl: profile.isl ?? existing.isl, osl: profile.osl ?? existing.osl, - normalizedSessionTimeS: profile.normalizedSessionTimeS ?? existing.normalizedSessionTimeS, - p90PrefillTpsPerUser: profile.p90PrefillTpsPerUser ?? existing.p90PrefillTpsPerUser, kvCacheUtil: existing.kvCacheUtil, prefixCacheHitRate: existing.prefixCacheHitRate, }; @@ -92,9 +101,7 @@ export async function computeAggregateStats(args: { }): Promise { let islPct: MetricPercentiles | null = null; let oslPct: MetricPercentiles | null = null; - let normalized: number | null = null; - let prefillP90: number | null = null; - let normalizedE2e400: MetricPercentiles | null = null; + let e2elPerOsl: MetricPercentiles | null = null; if (args.profileBlob) { try { @@ -102,10 +109,7 @@ export async function computeAggregateStats(args: { const { isl, osl } = extractIslOsl(jsonl); islPct = percentilesOf(isl); oslPct = percentilesOf(osl); - const derived = computeDerivedFromBlob(jsonl); - normalized = derived.normalized_session_time_s; - prefillP90 = derived.p90_prefill_tps_per_user; - normalizedE2e400 = derived.normalized_e2e_400; + e2elPerOsl = computeDerivedFromBlob(jsonl).e2el_per_osl; } catch { // ignore malformed blob — leave nulls } @@ -136,8 +140,6 @@ export async function computeAggregateStats(args: { osl: oslPct, kvCacheUtil: kvPct, prefixCacheHitRate: prefixPct, - normalizedSessionTimeS: normalized, - p90PrefillTpsPerUser: prefillP90, - normalizedE2e400, + e2elPerOsl, }; } diff --git a/packages/db/src/queries/agentic-aggregates.test.ts b/packages/db/src/queries/agentic-aggregates.test.ts index 0c4dbc89..566ca65f 100644 --- a/packages/db/src/queries/agentic-aggregates.test.ts +++ b/packages/db/src/queries/agentic-aggregates.test.ts @@ -129,9 +129,7 @@ interface WrittenStats { osl: unknown; kvCacheUtil: { mean: number } | null; prefixCacheHitRate: unknown; - normalizedSessionTimeS: number | null; - p90PrefillTpsPerUser: number | null; - normalizedE2e400: unknown; + e2elPerOsl: { p90: number; n: number } | null; } /** Capture SQL template text + bound values for the write-back assertions. */ @@ -228,10 +226,14 @@ describe('getAgenticAggregates write-back', () => { expect(written.version).toBe(STATS_VERSION); // Server field FRESHLY recomputed (0.25), not the stale 0.9 carried forward. expect(written.kvCacheUtil?.mean).toBeCloseTo(0.25, 6); - // Derived fields FRESHLY recomputed (not the stale 999s). - expect(written.normalizedSessionTimeS).toBeCloseTo(3, 6); - expect(written.p90PrefillTpsPerUser).toBeCloseTo(200, 6); - expect(written.normalizedE2e400).not.toBeNull(); + // Derived ratio bundle FRESHLY recomputed from the two turns: + // ratios (s/tok) = [1.0/50, 2.0/50] = [0.02, 0.04] → p90 = 0.038. + expect(written.e2elPerOsl?.n).toBe(2); + expect(written.e2elPerOsl?.p90).toBeCloseTo(0.038, 8); + // Retired legacy fields must not survive the recompute. + expect(written).not.toHaveProperty('normalizedSessionTimeS'); + expect(written).not.toHaveProperty('p90PrefillTpsPerUser'); + expect(written).not.toHaveProperty('normalizedE2e400'); expect(written.isl).not.toBeNull(); }); diff --git a/packages/db/src/queries/agentic-aggregates.ts b/packages/db/src/queries/agentic-aggregates.ts index 4917dc4c..44abd8ab 100644 --- a/packages/db/src/queries/agentic-aggregates.ts +++ b/packages/db/src/queries/agentic-aggregates.ts @@ -330,9 +330,7 @@ export async function getAgenticAggregates( osl: oslPct, kvCacheUtil: null, prefixCacheHitRate: null, - normalizedSessionTimeS: derived.normalized_session_time_s, - p90PrefillTpsPerUser: derived.p90_prefill_tps_per_user, - normalizedE2e400: derived.normalized_e2e_400, + e2elPerOsl: derived.e2el_per_osl, }, }); } catch { @@ -404,8 +402,6 @@ interface AggregateStatsRow { osl: MetricPercentiles | null; kvCacheUtil: MetricPercentiles | null; prefixCacheHitRate: MetricPercentiles | null; - normalizedSessionTimeS: number | null; - p90PrefillTpsPerUser: number | null; } /** @@ -419,9 +415,7 @@ interface FullAggregateStats { osl: MetricPercentiles | null; kvCacheUtil: MetricPercentiles | null; prefixCacheHitRate: MetricPercentiles | null; - normalizedSessionTimeS: number | null; - p90PrefillTpsPerUser: number | null; - normalizedE2e400: MetricPercentiles | null; + e2elPerOsl: MetricPercentiles | null; } function blankAggregate(id: number): AgenticAggregate { diff --git a/packages/db/src/queries/agentic-shared.ts b/packages/db/src/queries/agentic-shared.ts index d8673a07..a18344fc 100644 --- a/packages/db/src/queries/agentic-shared.ts +++ b/packages/db/src/queries/agentic-shared.ts @@ -31,8 +31,16 @@ import type { DbClient } from '../connection.js'; * * v5: reject osl <= 0 in extractTurn to exclude cancelled/empty-output turns * whose decode-interval math would explode normalized E2E to thousands of seconds. + * + * v6: drop the retired per-point derived metrics (normalizedSessionTimeS, + * p90PrefillTpsPerUser, normalizedE2e400) along with the experimental chart + * modes they fed. + * + * v7: add `e2elPerOsl` — percentiles of per-request E2E latency divided by + * OSL (seconds per output token), the inverse of the "OSL / E2EL" x-axis + * metric. */ -export const STATS_VERSION = 5; +export const STATS_VERSION = 7; interface ProfileRecord { metadata?: { benchmark_phase?: string }; diff --git a/packages/db/src/queries/derived-agentic-metrics.test.ts b/packages/db/src/queries/derived-agentic-metrics.test.ts index a39de670..d556296a 100644 --- a/packages/db/src/queries/derived-agentic-metrics.test.ts +++ b/packages/db/src/queries/derived-agentic-metrics.test.ts @@ -25,51 +25,39 @@ function rec( } describe('computeDerivedFromBlob', () => { - it('returns nulls when no usable records', () => { + it('returns null when no usable records', () => { const out = computeDerivedFromBlob(''); - expect(out.normalized_session_time_s).toBeNull(); - expect(out.p90_prefill_tps_per_user).toBeNull(); - expect(out.normalized_e2e_400).toBeNull(); + expect(out.e2el_per_osl).toBeNull(); }); - it('normalizes each request to 400 output tokens before taking percentiles', () => { + it('computes per-request E2EL/OSL ratios pooled across sessions', () => { + // Ratios (s per output token): 1.0/50 = 0.02, 4.0/100 = 0.04. const jsonl = [ - // Both requests have TTFT=2s and ITL=20ms, despite very different OSL/E2E. - rec('s1', 0, { isl: 100, osl: 100, ttft_ms: 2000, latency_ms: 3980 }), - rec('s2', 0, { isl: 100, osl: 1000, ttft_ms: 2000, latency_ms: 21_980 }), + rec('s1', 0, { isl: 100, osl: 50, ttft_ms: 500, latency_ms: 1000 }), + rec('s2', 0, { isl: 200, osl: 100, ttft_ms: 1000, latency_ms: 4000 }), ].join('\n'); const out = computeDerivedFromBlob(jsonl); - // 2s TTFT + 399 × 20ms ITL = 9.98s for both requests. - expect(out.normalized_e2e_400?.n).toBe(2); - expect(out.normalized_e2e_400?.p75).toBeCloseTo(9.98, 8); - expect(out.normalized_e2e_400?.p90).toBeCloseTo(9.98, 8); + expect(out.e2el_per_osl?.n).toBe(2); + expect(out.e2el_per_osl?.p50).toBeCloseTo(0.03, 8); + // p90 of [0.02, 0.04]: pos = 0.9 → 0.02 + 0.9 × 0.02 = 0.038. + expect(out.e2el_per_osl?.p90).toBeCloseTo(0.038, 8); + expect(out.e2el_per_osl?.p75).toBeCloseTo(0.035, 8); }); - it('rescales single-session time and computes P90 prefill', () => { - // One session, two turns. load = (100+50) + (200+50) = 400. - // Single session ⇒ mean_load = load_i ⇒ T̃ = T = (1000+2000) ms = 3.0 s. + it('OSL cancels out of the ratio when TTFT and ITL are identical', () => { + // Both requests: TTFT=2s, ITL=20ms — very different OSL/E2E, but the + // per-token ratio only differs through the TTFT amortization term + // (TTFT/OSL), which is the intended second-order OSL sensitivity. const jsonl = [ - rec('s1', 0, { isl: 100, osl: 50, ttft_ms: 500, latency_ms: 1000 }), - rec('s1', 1, { isl: 200, osl: 50, ttft_ms: 1000, latency_ms: 2000 }), + rec('s1', 0, { isl: 100, osl: 100, ttft_ms: 2000, latency_ms: 3980 }), + rec('s2', 0, { isl: 100, osl: 1000, ttft_ms: 2000, latency_ms: 21_980 }), ].join('\n'); - const out = computeDerivedFromBlob(jsonl); - expect(out.normalized_session_time_s).toBeCloseTo(3, 6); - // Prefill TPS per turn: 100/0.5=200, 200/1.0=200 → global P90 = 200. - expect(out.p90_prefill_tps_per_user).toBeCloseTo(200, 6); - }); - it('rescales times across sessions with unequal load', () => { - // s1: 1 turn, load = 100, T = 1s - // s2: 1 turn, load = 300, T = 3s - // mean_load = 200; T̃_1 = 1 * 200/100 = 2; T̃_2 = 3 * 200/300 = 2 - // Mean T̃ = 2.0 - const jsonl = [ - rec('s1', 0, { isl: 90, osl: 10, ttft_ms: 500, latency_ms: 1000 }), - rec('s2', 0, { isl: 270, osl: 30, ttft_ms: 500, latency_ms: 3000 }), - ].join('\n'); const out = computeDerivedFromBlob(jsonl); - expect(out.normalized_session_time_s).toBeCloseTo(2, 6); + expect(out.e2el_per_osl?.n).toBe(2); + // 3.98/100 = 0.0398 ≈ ITL + TTFT/OSL = 0.0198 + 0.02 + expect(out.e2el_per_osl?.p50).toBeCloseTo((0.0398 + 0.02198) / 2, 8); }); it('drops records missing required fields and skips non-profiling phase', () => { @@ -96,63 +84,35 @@ describe('computeDerivedFromBlob', () => { }), ]; const out = computeDerivedFromBlob(lines.join('\n')); - expect(out.normalized_session_time_s).toBeCloseTo(1, 6); - expect(out.p90_prefill_tps_per_user).toBeCloseTo(200, 6); + expect(out.e2el_per_osl?.n).toBe(1); + expect(out.e2el_per_osl?.p90).toBeCloseTo(0.02, 8); }); - it('p90 across turns: 10-turn session picks the right rank', () => { - // Prefill rates 100..1000 (per turn isl/ttft); p90 of 10 values (linear) = 910. - const turns = Array.from({ length: 10 }, (_, i) => - rec('s1', i, { - isl: (i + 1) * 100, // 100, 200, ..., 1000 tokens - osl: 10, - ttft_ms: 1000, // 1 second → rates: 100..1000 tps - latency_ms: 1500, - }), - ); - const out = computeDerivedFromBlob(turns.join('\n')); - expect(out.p90_prefill_tps_per_user).toBeCloseTo(910, 6); - }); - - it('excludes osl=0 (cancelled/empty-output) turns from normalized E2E', () => { - // Two normal turns + one cancelled turn (osl=0, latency=30s, ttft=1s). - // - // The cancelled turn must be excluded because observedDecodeIntervals collapses - // to max(0-1,1)=1, making itlMs=(30000-1000)/1=29000ms and normalizedMs explode - // to ~11 572 s — roughly 386× the real scale. (Pre-fix behavior for reference; - // this number is intentionally not asserted below to avoid enshrining the bug.) - // - // Normal turn A: isl=100, osl=50, ttft=500ms, latency=1000ms - // observedDecodeIntervals = max(49,1) = 49 - // itlMs = (1000-500)/49 - // normalizedMs = 500 + 399*(500/49) - // - // Normal turn B: isl=200, osl=100, ttft=1000ms, latency=3000ms - // observedDecodeIntervals = max(99,1) = 99 - // itlMs = (3000-1000)/99 - // normalizedMs = 1000 + 399*(2000/99) - const normA = (500 + (399 * 500) / 49) / 1000; // seconds - const normB = (1000 + (399 * 2000) / 99) / 1000; // seconds - + it('excludes osl=0 (cancelled/empty-output) turns from the ratio distribution', () => { const jsonl = [ rec('s1', 0, { isl: 100, osl: 50, ttft_ms: 500, latency_ms: 1000 }), - rec('s1', 1, { isl: 200, osl: 100, ttft_ms: 1000, latency_ms: 3000 }), - // Cancelled / empty-output turn — osl=0 must be rejected by extractTurn. + // Cancelled / empty-output turn — osl=0 must be rejected by extractTurn + // (the ratio would divide by zero). rec('s2', 0, { isl: 150, osl: 0, ttft_ms: 1000, latency_ms: 30000 }), ].join('\n'); const out = computeDerivedFromBlob(jsonl); + expect(out.e2el_per_osl?.n).toBe(1); + expect(out.e2el_per_osl?.p90).toBeCloseTo(0.02, 8); + }); - // Only the 2 normal turns contribute; osl=0 record is silently excluded. - expect(out.normalized_e2e_400?.n).toBe(2); - - // p90 of [normA, normB] sorted ascending (normA < normB): - // pos = 1*0.9 = 0.9; result = normA + (normB - normA)*0.9 - const expectedP90 = normA + (normB - normA) * 0.9; - expect(out.normalized_e2e_400?.p90).toBeCloseTo(expectedP90, 6); - - // Sanity: p90 should be single-digit seconds, not thousands. - expect(out.normalized_e2e_400!.p90).toBeLessThan(20); + it('p90 across turns: 10-turn population picks the right rank', () => { + // Ratios 0.01..0.10 s/tok; p90 of 10 values (linear) = 0.091. + const turns = Array.from({ length: 10 }, (_, i) => + rec('s1', i, { + isl: 100, + osl: 100, + ttft_ms: 100, + latency_ms: (i + 1) * 1000, // 1s..10s → ratios 0.01..0.10 + }), + ); + const out = computeDerivedFromBlob(turns.join('\n')); + expect(out.e2el_per_osl?.p90).toBeCloseTo(0.091, 8); }); }); @@ -172,9 +132,10 @@ function mockSql(queue: unknown[][]): { describe('getDerivedAgenticMetrics write-back', () => { it('self-heals aggregate_stats from the profile blob, carrying server fields forward', async () => { + // Two turns with ratios 0.02 and 0.04 s/tok → p90 = 0.038, p75 = 0.035. const jsonl = [ rec('s1', 0, { isl: 100, osl: 50, ttft_ms: 500, latency_ms: 1000 }), - rec('s1', 1, { isl: 200, osl: 50, ttft_ms: 1000, latency_ms: 2000 }), + rec('s1', 1, { isl: 200, osl: 100, ttft_ms: 1000, latency_ms: 4000 }), ].join('\n'); const blob = gzipSync(Buffer.from(jsonl)); @@ -201,9 +162,9 @@ describe('getDerivedAgenticMetrics write-back', () => { const result = await getDerivedAgenticMetrics(sql, [7]); - // Response is the freshly recomputed value, not the stale 999s. - expect(result[7]?.normalized_session_time_s).toBeCloseTo(3, 6); - expect(result[7]?.p90_prefill_tps_per_user).toBeCloseTo(200, 6); + // Response is the freshly recomputed slow-tail inverse (tok/s/user). + expect(result[7]?.p90_osl_per_e2el).toBeCloseTo(1 / 0.038, 6); + expect(result[7]?.p75_osl_per_e2el).toBeCloseTo(1 / 0.035, 6); // 3 calls: stats read, blob read, write-back UPDATE. expect(calls).toHaveLength(3); @@ -219,18 +180,47 @@ describe('getDerivedAgenticMetrics write-back', () => { isl: unknown; osl: unknown; kvCacheUtil: unknown; - normalizedSessionTimeS: number | null; - p90PrefillTpsPerUser: number | null; + e2elPerOsl: { p75: number; p90: number; n: number } | null; } const [written, traceReplayId] = calls[2]!.values as [WrittenStats, number]; expect(traceReplayId).toBe(870); expect(written.version).toBe(STATS_VERSION); - expect(written.normalizedSessionTimeS).toBeCloseTo(3, 6); - expect(written.p90PrefillTpsPerUser).toBeCloseTo(200, 6); + expect(written.e2elPerOsl?.n).toBe(2); + expect(written.e2elPerOsl?.p90).toBeCloseTo(0.038, 8); expect(written.isl).not.toBeNull(); expect(written.osl).not.toBeNull(); // Server-derived field carried forward from the stale row (not re-read). expect(written.kvCacheUtil).toEqual(staleServerKv); + // Retired legacy fields must not survive the heal. + expect(written).not.toHaveProperty('normalizedSessionTimeS'); + expect(written).not.toHaveProperty('p90PrefillTpsPerUser'); + expect(written).not.toHaveProperty('normalizedE2e400'); + }); + + it('does not stamp the bundle when there are no server fields to preserve', async () => { + // A row with null (or pre-v3) stats has no kvCacheUtil / prefixCacheHitRate + // for this route to carry forward, and it cannot recompute them — it never + // reads the server blob. Stamping the current version anyway would look + // complete downstream: the backfill's candidate query skips matching + // versions and agentic-aggregates takes the fast path, so those fields + // would stay null forever. The response is still served from the blob. + const jsonl = rec('s1', 0, { isl: 100, osl: 50, ttft_ms: 500, latency_ms: 1000 }); + const blob = gzipSync(Buffer.from(jsonl)); + + const { sql, calls } = mockSql([ + // fetchAggregateStatsRows — no stored bundle at all + [{ benchmark_result_id: 7, stats: null }], + // fallback profile-blob query + [{ benchmark_result_id: 7, trace_replay_id: 870, blob }], + ]); + + const result = await getDerivedAgenticMetrics(sql, [7]); + + // Caller still gets the freshly computed metric (1 / 0.02 s-per-token). + expect(result[7]?.p90_osl_per_e2el).toBeCloseTo(50, 6); + // Stats read + blob read only — no write-back UPDATE. + expect(calls).toHaveLength(2); + expect(calls.some((c) => c.text.includes('update agentic_trace_replay'))).toBe(false); }); it('takes the fast path (no blob read, no write-back) when stats are current', async () => { @@ -240,17 +230,34 @@ describe('getDerivedAgenticMetrics write-back', () => { osl: null, kvCacheUtil: null, prefixCacheHitRate: null, - normalizedSessionTimeS: 1.5, - p90PrefillTpsPerUser: 42, - normalizedE2e400: { mean: 1, p50: 1, p75: 1, p90: 2, p99: 3, n: 5 }, + e2elPerOsl: { mean: 0.03, p50: 0.03, p75: 0.04, p90: 0.05, p99: 0.06, n: 5 }, }; const { sql, calls } = mockSql([[{ benchmark_result_id: 7, stats: currentStats }]]); const result = await getDerivedAgenticMetrics(sql, [7]); - expect(result[7]?.normalized_session_time_s).toBe(1.5); - expect(result[7]?.p90_normalized_e2e_400_s).toBe(2); + expect(result[7]?.p75_osl_per_e2el).toBeCloseTo(25, 6); + expect(result[7]?.p90_osl_per_e2el).toBeCloseTo(20, 6); // Only the stats read — no fallback blob query, no write-back. expect(calls).toHaveLength(1); }); + + it('maps a missing ratio bundle to nulls on the fast path', async () => { + const currentStats = { + version: STATS_VERSION, + isl: null, + osl: null, + kvCacheUtil: null, + prefixCacheHitRate: null, + e2elPerOsl: null, + }; + const { sql } = mockSql([ + [{ benchmark_result_id: 7, stats: currentStats }], + // fallback blob query fires for no ids → but guard returns before; keep + // the queue empty-safe anyway. + ]); + + const result = await getDerivedAgenticMetrics(sql, [7]); + expect(result[7]).toEqual({ id: 7, p75_osl_per_e2el: null, p90_osl_per_e2el: null }); + }); }); diff --git a/packages/db/src/queries/derived-agentic-metrics.ts b/packages/db/src/queries/derived-agentic-metrics.ts index 626ab9c7..cfb3dedb 100644 --- a/packages/db/src/queries/derived-agentic-metrics.ts +++ b/packages/db/src/queries/derived-agentic-metrics.ts @@ -1,34 +1,33 @@ /** * Live-computed per-point metrics derived from the stored aiperf * `profile_export.jsonl` blob. These aren't precomputed in the metrics JSONB - * because they require grouping by `conversation_id` and aggregating per - * session — work that's cheap once per agentic point but adds up to be - * meaningful only when actually plotted. + * because they require a full pass over the per-request records — work that's + * cheap once per agentic point but adds up to be meaningful only when + * actually plotted. * - * - normalized_session_time_s: per the "Mean Normalized Session Time" proposal - * (https://gist.github.com/xinli-sw/115d370c17f6d1b977878b68530981fa). Sum of - * per-turn `request_latency` per session (inter-turn tool/thinking gaps are - * inherently excluded since we only sum the active GPU time, not wallclock). - * Each session's time is rescaled by `mean_load / session_load`, where load - * is Σ(ISL+OSL) across turns. The plotted value is the mean across sessions. + * - OSL / E2EL ("e2e interactivity"): per-request output_sequence_length + * divided by request_latency — the rate at which the user receives output + * tokens INCLUDING the prefill wait, per + * https://semianalysis.slack.com/archives/C0AV4T40BT3/p1782432266626969. + * Algebraically `OSL / (TTFT + decode_time) ≈ 1 / (ITL + TTFT/OSL)`: plain + * interactivity with a penalty that grows when TTFT is large relative to + * the output produced, so prefill-delaying can't inflate the metric the way + * it can with 1/TPOT. * - * - p90_prefill_tps_per_user: per the same gist's "Prefill" Pareto chart. - * Per turn: prefill_tps = ISL / TTFT_seconds. Single P90 across every turn - * in every session — the per-session percentile + cross-session mean - * sandwich was discarded because it just dampens tail behavior. + * Percentiles follow the slow-tail convention the mapper enforces for + * `*_intvty` (1/p(ITL), not p(1/ITL)): we store percentiles of the + * per-request E2EL/OSL ratio (seconds per output token) and the read path + * inverts, so `p90 OSL/E2EL = 1 / p90(E2EL/OSL)` is the 90th-percentile + * WORST request's effective token rate. */ import { gunzipSync } from 'node:zlib'; -import { NORMALIZED_E2E_OUTPUT_TOKENS } from '@semianalysisai/inferencex-constants'; - import type { DbClient } from '../connection.js'; import { extractIslOsl, fetchAggregateStatsRows, - meanOf, percentilesOf, - quantile, readNum, STATS_VERSION, writeBackTraceReplayJsonb, @@ -38,14 +37,10 @@ import { export interface DerivedAgenticMetric { /** benchmark_results.id this entry belongs to. */ id: number; - /** Mean normalized session time in seconds. */ - normalized_session_time_s: number | null; - /** P90 of per-turn prefill tps/user (ISL / TTFT) across every turn in every session. */ - p90_prefill_tps_per_user: number | null; - /** P75 normalized per-request E2E at a fixed 400-token output length. */ - p75_normalized_e2e_400_s: number | null; - /** P90 normalized per-request E2E at a fixed 400-token output length. */ - p90_normalized_e2e_400_s: number | null; + /** Slow-tail P75 OSL/E2EL in tok/s/user — 1 / p75(per-request E2EL/OSL). */ + p75_osl_per_e2el: number | null; + /** Slow-tail P90 OSL/E2EL in tok/s/user — 1 / p90(per-request E2EL/OSL). */ + p90_osl_per_e2el: number | null; } export type DerivedAgenticMetricMap = Record; @@ -66,9 +61,7 @@ interface StoredAggregateStats { osl: MetricPercentiles | null; kvCacheUtil: MetricPercentiles | null; prefixCacheHitRate: MetricPercentiles | null; - normalizedSessionTimeS: number | null; - p90PrefillTpsPerUser: number | null; - normalizedE2e400: MetricPercentiles | null; + e2elPerOsl: MetricPercentiles | null; } /** @@ -113,17 +106,22 @@ function extractTurn(rec: ProfileRecord): TurnFields | null { return { request_latency_ms: rl, ttft_ms: tt, isl, osl }; } +/** 1/x for a positive stored ratio; null when the bundle/percentile is absent. */ +function invertRatio(v: number | null | undefined): number | null { + return typeof v === 'number' && Number.isFinite(v) && v > 0 ? 1 / v : null; +} + /** - * Parse one point's JSONL and return the two derived metrics. Returns - * `{ session_time: null, prefill: null }` if the blob has no usable records. + * Parse one point's JSONL and return the per-request E2EL/OSL ratio + * percentiles (seconds per output token). Every profiling-phase turn with + * complete, positive fields contributes one sample — the distribution pools + * turns across all sessions so a percentile sees the full request population. + * Returns `{ e2el_per_osl: null }` if the blob has no usable records. */ export function computeDerivedFromBlob(jsonl: string): { - normalized_session_time_s: number | null; - p90_prefill_tps_per_user: number | null; - normalized_e2e_400: MetricPercentiles | null; + e2el_per_osl: MetricPercentiles | null; } { - // Group records by conversation_id, filter to the profiling phase. - const bySession = new Map(); + const ratios: number[] = []; for (const line of jsonl.split('\n')) { if (!line) continue; let rec: ProfileRecord; @@ -133,87 +131,11 @@ export function computeDerivedFromBlob(jsonl: string): { continue; } if (rec.metadata?.benchmark_phase && rec.metadata.benchmark_phase !== 'profiling') continue; - const sid = rec.metadata?.conversation_id; - if (!sid) continue; const turn = extractTurn(rec); if (!turn) continue; - let list = bySession.get(sid); - if (!list) { - list = []; - bySession.set(sid, list); - } - list.push(turn); - } - if (bySession.size === 0) { - return { - normalized_session_time_s: null, - p90_prefill_tps_per_user: null, - normalized_e2e_400: null, - }; - } - - // Per-session aggregates for session time; per-turn prefill rates pool into - // a single global array so the percentile sees the full distribution. - const sessionTimesS: number[] = []; - const sessionLoads: number[] = []; - const allPrefillRates: number[] = []; - const allNormalizedE2eS: number[] = []; - for (const turns of bySession.values()) { - let timeMs = 0; - let load = 0; - for (const t of turns) { - timeMs += t.request_latency_ms; - load += t.isl + t.osl; - const ttftSec = t.ttft_ms / 1000; - if (ttftSec > 0) allPrefillRates.push(t.isl / ttftSec); - - // Keep the observed TTFT, then project the request's mean decode - // interval to a fixed output length. Do this per request before taking - // percentiles so long original outputs do not dominate the tail. - const observedDecodeIntervals = Math.max(t.osl - 1, 1); - const itlMs = (t.request_latency_ms - t.ttft_ms) / observedDecodeIntervals; - const normalizedMs = t.ttft_ms + (NORMALIZED_E2E_OUTPUT_TOKENS - 1) * itlMs; - if ( - Number.isFinite(itlMs) && - itlMs >= 0 && - Number.isFinite(normalizedMs) && - normalizedMs > 0 - ) { - allNormalizedE2eS.push(normalizedMs / 1000); - } - } - if (load > 0) { - sessionTimesS.push(timeMs / 1000); - sessionLoads.push(load); - } - } - - // Normalized session time: T̃_i = T_i × (mean_load / load_i), then mean. - let normalized: number | null = null; - if (sessionTimesS.length > 0) { - const meanLoad = meanOf(sessionLoads); - if (meanLoad > 0) { - const scaled: number[] = []; - for (let i = 0; i < sessionTimesS.length; i++) { - const ti = sessionTimesS[i]!; - const li = sessionLoads[i]!; - if (li > 0) scaled.push(ti * (meanLoad / li)); - } - normalized = scaled.length > 0 ? meanOf(scaled) : null; - } + ratios.push(turn.request_latency_ms / 1000 / turn.osl); } - - let prefill: number | null = null; - if (allPrefillRates.length > 0) { - allPrefillRates.sort((a, b) => a - b); - prefill = quantile(allPrefillRates, 0.9); - } - - return { - normalized_session_time_s: normalized, - p90_prefill_tps_per_user: prefill, - normalized_e2e_400: percentilesOf(allNormalizedE2eS), - }; + return { e2el_per_osl: percentilesOf(ratios) }; } export async function getDerivedAgenticMetrics( @@ -224,8 +146,8 @@ export async function getDerivedAgenticMetrics( const result: DerivedAgenticMetricMap = {}; - // Fast path: read the pre-computed values out of `aggregate_stats`. The - // ingest pipeline computes both metrics in the same pass that produces the + // Fast path: read the pre-computed ratio bundle out of `aggregate_stats`. + // The ingest pipeline computes it in the same pass that produces the // percentile bundles, so a single SQL round-trip covers most ids without // touching the gzipped profile blob. const statsRows = await fetchAggregateStatsRows(sql, benchmarkResultIds); @@ -240,10 +162,8 @@ export async function getDerivedAgenticMetrics( if (row.stats && Number(row.stats.version) === STATS_VERSION) { result[id] = { id, - normalized_session_time_s: row.stats.normalizedSessionTimeS ?? null, - p90_prefill_tps_per_user: row.stats.p90PrefillTpsPerUser ?? null, - p75_normalized_e2e_400_s: row.stats.normalizedE2e400?.p75 ?? null, - p90_normalized_e2e_400_s: row.stats.normalizedE2e400?.p90 ?? null, + p75_osl_per_e2el: invertRatio(row.stats.e2elPerOsl?.p75), + p90_osl_per_e2el: invertRatio(row.stats.e2elPerOsl?.p90), }; } else { idsNeedingBlob.push(id); @@ -282,34 +202,40 @@ export async function getDerivedAgenticMetrics( const id = Number(row.benchmark_result_id); try { const jsonl = gunzipSync(row.blob).toString('utf8'); - const { normalized_session_time_s, p90_prefill_tps_per_user, normalized_e2e_400 } = - computeDerivedFromBlob(jsonl); + const { e2el_per_osl } = computeDerivedFromBlob(jsonl); result[id] = { id, - normalized_session_time_s, - p90_prefill_tps_per_user, - p75_normalized_e2e_400_s: normalized_e2e_400?.p75 ?? null, - p90_normalized_e2e_400_s: normalized_e2e_400?.p90 ?? null, + p75_osl_per_e2el: invertRatio(e2el_per_osl?.p75), + p90_osl_per_e2el: invertRatio(e2el_per_osl?.p90), }; // Self-heal the shared `aggregate_stats` bundle. We only have the profile - // blob here, so recompute the profile-derived fields (isl/osl + the three - // derived metrics) and carry the stale row's server-derived fields + // blob here, so recompute the profile-derived fields (isl/osl + the + // ratio bundle) and carry the stale row's server-derived fields // forward untouched — the profile-only upgrade the backfill CLI also // performs. Fire-and-forget, best-effort (no-ops on a read-only replica). - const { isl, osl } = extractIslOsl(jsonl); + // + // Only stamp the bundle when the stale row actually HAS server-derived + // fields to carry forward. Writing nulls at the current version would + // look complete to everyone downstream: the backfill skips the row + // (its candidate query matches on version) and the agentic-aggregates + // route takes the fast path, so kvCacheUtil / prefixCacheHitRate would + // stay null forever. Leaving the row stale instead costs one repeat + // parse and lets a reader that CAN see the server blob heal it fully. const prior = staleStatsById.get(id) ?? null; - const merged: StoredAggregateStats = { - version: STATS_VERSION, - isl: percentilesOf(isl), - osl: percentilesOf(osl), - kvCacheUtil: prior?.kvCacheUtil ?? null, - prefixCacheHitRate: prior?.prefixCacheHitRate ?? null, - normalizedSessionTimeS: normalized_session_time_s, - p90PrefillTpsPerUser: p90_prefill_tps_per_user, - normalizedE2e400: normalized_e2e_400, - }; - writeBackTraceReplayJsonb(sql, 'aggregate_stats', Number(row.trace_replay_id), merged); + const canPreserveServerFields = Boolean(prior?.kvCacheUtil || prior?.prefixCacheHitRate); + if (canPreserveServerFields) { + const { isl, osl } = extractIslOsl(jsonl); + const merged: StoredAggregateStats = { + version: STATS_VERSION, + isl: percentilesOf(isl), + osl: percentilesOf(osl), + kvCacheUtil: prior?.kvCacheUtil ?? null, + prefixCacheHitRate: prior?.prefixCacheHitRate ?? null, + e2elPerOsl: e2el_per_osl, + }; + writeBackTraceReplayJsonb(sql, 'aggregate_stats', Number(row.trace_replay_id), merged); + } } catch { // Skip malformed blobs silently — frontend treats missing ids as "no data". }