From 91bd8a78fa439eb0fc1789fd505ec52556111080 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Wed, 29 Jul 2026 16:53:15 -0500 Subject: [PATCH 1/7] feat(agentic): remove Normalized E2E / Session Time / Prefill TPS x-axis modes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Retire the three experimental agentic x-axis modes and everything that existed solely to serve them, leaving Interactivity / E2E Latency / TTFT. - chart: drop the three mode buttons, DERIVED_X_MODE_SPECS and the derived-metric fetch/remap plumbing in ChartDisplay; the remaining modes apply to both scenario kinds, so the agentic-only button filter (and its `mounted` SSR guard) goes too - API/hook/db: delete /api/v1/derived-agentic-metrics, the useDerivedAgenticMetrics hook, and queries/derived-agentic-metrics.ts - aggregate_stats v6: drop normalizedSessionTimeS, p90PrefillTpsPerUser and normalizedE2e400; mergeProfileStatsUpgrade no longer carries them forward, and the backfill's profile-only fast path now covers every v3+ bundle instead of only v3 - drop the NORMALIZED_E2E_OUTPUT_TOKENS constant and the overlay suppression helper that existed only for Normalized E2E 中文：移除三个实验性智能体 X 轴模式（Normalized E2E、会话时长、 Prefill TPS / user）及其专属实现，仅保留交互性 / 端到端延迟 / TTFT。 同时删除对应的 API 路由、React Query hook 与数据库查询模块； aggregate_stats 升级至 v6，去掉三个已废弃字段，backfill 的 profile-only 快速路径扩展至所有 v3 及以上版本。 Co-Authored-By: Claude Fable 5 --- .../app/cypress/e2e/ttft-x-axis-toggle.cy.ts | 71 --- .../src/app/api/v1/agentic-cache-keys.test.ts | 6 - .../api/v1/derived-agentic-metrics/route.ts | 51 -- .../components/inference/InferenceContext.tsx | 46 +- .../inference/hooks/useChartData.ts | 39 +- .../app/src/components/inference/types.ts | 17 +- .../components/inference/ui/ChartDisplay.tsx | 577 +++++++----------- .../src/components/inference/utils.test.ts | 21 +- .../app/src/components/inference/utils.ts | 14 - .../components/inference/utils/e2eFrontier.ts | 2 +- .../api/use-derived-agentic-metrics.test.ts | 13 - .../hooks/api/use-derived-agentic-metrics.ts | 55 -- packages/app/src/lib/benchmark-id.ts | 2 +- packages/constants/src/agentic.ts | 2 - packages/constants/src/index.ts | 1 - packages/db/src/backfill-aggregate-stats.ts | 8 +- .../src/etl/compute-aggregate-stats.test.ts | 49 +- .../db/src/etl/compute-aggregate-stats.ts | 41 +- .../db/src/queries/agentic-aggregates.test.ts | 11 +- packages/db/src/queries/agentic-aggregates.ts | 17 +- packages/db/src/queries/agentic-shared.ts | 7 +- .../queries/derived-agentic-metrics.test.ts | 256 -------- .../db/src/queries/derived-agentic-metrics.ts | 318 ---------- 23 files changed, 297 insertions(+), 1327 deletions(-) delete mode 100644 packages/app/src/app/api/v1/derived-agentic-metrics/route.ts delete mode 100644 packages/app/src/hooks/api/use-derived-agentic-metrics.test.ts delete mode 100644 packages/app/src/hooks/api/use-derived-agentic-metrics.ts delete mode 100644 packages/constants/src/agentic.ts delete mode 100644 packages/db/src/queries/derived-agentic-metrics.test.ts delete mode 100644 packages/db/src/queries/derived-agentic-metrics.ts diff --git a/packages/app/cypress/e2e/ttft-x-axis-toggle.cy.ts b/packages/app/cypress/e2e/ttft-x-axis-toggle.cy.ts index 5023877ea..f3a897b1b 100644 --- a/packages/app/cypress/e2e/ttft-x-axis-toggle.cy.ts +++ b/packages/app/cypress/e2e/ttft-x-axis-toggle.cy.ts @@ -1,27 +1,5 @@ import { unlockAgenticGate } from '../support/e2e'; -const interceptDerivedMetrics = () => { - cy.intercept('GET', '/api/v1/derived-agentic-metrics*', (request) => { - const ids = new URL(request.url).searchParams.get('ids')?.split(',').filter(Boolean) ?? []; - request.reply({ - body: Object.fromEntries( - ids.map((id, index) => [ - id, - { - id: Number(id), - normalized_session_time_s: 60 + index, - p90_prefill_tps_per_user: 100 + index, - p75_normalized_e2e_400_s: 8 + index, - p90_normalized_e2e_400_s: 12 + index, - p75_osl_per_e2el: 40 + index, - p90_osl_per_e2el: 25 + index, - }, - ]), - ), - }); - }).as('derivedAgenticMetrics'); -}; - // This spec exercises the agentic x-axis modes, which only exist when the // selected model resolves to the Agentic Traces scenario. The default e2e // fixtures (cypress/fixtures/api/*.json) have NO agentic rows for any model, and @@ -148,10 +126,6 @@ const interceptFixedSequenceData = () => { describe('X-Axis Mode Toggle (inference chart)', () => { before(() => { interceptAgenticData(); - // Stub derived metrics before the visit: if the agentic DEFAULT x-axis mode - // is (or becomes) a derived mode that fetches /derived-agentic-metrics on - // mount, the chart must not sit on its loading skeleton. - interceptDerivedMetrics(); cy.visit('/inference?i_seq=agentic-traces', { onBeforeLoad(win) { win.localStorage.setItem('inferencex-star-modal-dismissed', String(Date.now())); @@ -166,7 +140,6 @@ describe('X-Axis Mode Toggle (inference chart)', () => { cy.get('[data-testid="scenario-selector"]').should('contain.text', 'Agentic Traces'); cy.get('[data-testid="x-axis-mode-ttft"]').should('be.visible'); cy.get('[data-testid="x-axis-mode-e2e"]').should('be.visible'); - cy.get('[data-testid="x-axis-mode-normalized-e2e"]').should('be.visible'); cy.get('[data-testid="x-axis-mode-interactivity"]') .should('be.visible') .and('have.attr', 'aria-selected', 'true'); @@ -255,32 +228,6 @@ describe('X-Axis Mode Toggle (inference chart)', () => { cy.get('[data-testid="chart-figure"] svg').should('contain.text', 'P90 End-to-end Latency (s)'); }); - it('switches to request-level normalized E2E at 400 output tokens', () => { - interceptDerivedMetrics(); - cy.get('[data-testid="x-axis-mode-normalized-e2e"]').click(); - cy.wait('@derivedAgenticMetrics'); - cy.get('[data-testid="x-axis-mode-normalized-e2e"]').should( - 'have.attr', - 'aria-selected', - 'true', - ); - cy.get('[data-testid="chart-figure"] h2').should( - 'contain.text', - 'P90 Normalized E2E @ 400 output tokens', - ); - cy.get('[data-testid="chart-figure"] svg').should( - 'contain.text', - 'P90 Normalized E2E @ 400 output tokens (s)', - ); - - cy.get('[data-testid="percentile-selector"]').click(); - cy.contains('[role="option"]', 'p75').click(); - cy.get('[data-testid="chart-figure"] h2').should( - 'contain.text', - 'P75 Normalized E2E @ 400 output tokens', - ); - }); - it('switches back to Interactivity', () => { cy.get('[data-testid="x-axis-mode-interactivity"]').click(); cy.get('[data-testid="x-axis-mode-interactivity"]').should( @@ -411,9 +358,6 @@ const interceptAgenticDataWithOverlay = () => { describe('X-Axis Mode Toggle — overlay path (finding #8 regression guard)', () => { before(() => { interceptAgenticDataWithOverlay(); - // Same as the main suite: keep the visit independent of the agentic default - // x-axis mode (a derived default fetches /derived-agentic-metrics on mount). - interceptDerivedMetrics(); cy.visit(`/inference?unofficialrun=${OVERLAY_RUN_ID}&i_seq=agentic-traces`, { onBeforeLoad(win) { win.localStorage.setItem('inferencex-star-modal-dismissed', String(Date.now())); @@ -462,19 +406,4 @@ describe('X-Axis Mode Toggle — overlay path (finding #8 regression guard)', () expect(total).to.be.greaterThan(0); }); }); - - it('normalized-e2e mode shows suppression banner for unofficial-run overlays', () => { - interceptDerivedMetrics(); - cy.get('[data-testid="x-axis-mode-normalized-e2e"]').click(); - cy.get('[data-testid="x-axis-mode-normalized-e2e"]').should( - 'have.attr', - 'aria-selected', - 'true', - ); - // The suppression message appears because isUnofficialRun is true and the - // mode is 'normalized-e2e' (documented in ChartDisplay.tsx ~line 640). - cy.contains( - 'Normalized E2E requires persisted per-request traces, so unofficial-run overlays are unavailable for this experimental view.', - ).should('be.visible'); - }); }); diff --git a/packages/app/src/app/api/v1/agentic-cache-keys.test.ts b/packages/app/src/app/api/v1/agentic-cache-keys.test.ts index 3a6287cf6..7177f9960 100644 --- a/packages/app/src/app/api/v1/agentic-cache-keys.test.ts +++ b/packages/app/src/app/api/v1/agentic-cache-keys.test.ts @@ -28,17 +28,12 @@ import { STATS_VERSION } from '@semianalysisai/inferencex-db/queries/agentic-agg import { REQUEST_TIMELINE_VERSION } from '@semianalysisai/inferencex-db/etl/compute-request-timeline'; import { TRACE_SERVER_METRICS_VERSION } from '@semianalysisai/inferencex-db/queries/trace-server-metrics'; -import { CACHE_KEY_PREFIX as derivedAgenticMetricsKey } from './derived-agentic-metrics/route'; import { CACHE_KEY_PREFIX as agenticAggregatesKey } from './agentic-aggregates/route'; import { CACHE_KEY_PREFIX as requestTimelineKey } from './request-timeline/route'; import { CACHE_KEY_PREFIX as traceServerMetricsKey } from './trace-server-metrics/route'; import { CACHE_KEY_PREFIX as traceHistogramsKey } from './trace-histograms/route'; describe('agentic blob-cache keys are version-derived', () => { - it('derived-agentic-metrics key embeds STATS_VERSION', () => { - expect(derivedAgenticMetricsKey).toBe(`derived-agentic-metrics-v${STATS_VERSION}`); - }); - it('agentic-aggregates key embeds STATS_VERSION', () => { expect(agenticAggregatesKey).toBe(`agentic-aggregates-v${STATS_VERSION}`); }); @@ -57,7 +52,6 @@ describe('agentic blob-cache keys are version-derived', () => { it('every key actually contains a version segment (no unversioned literals)', () => { for (const key of [ - derivedAgenticMetricsKey, agenticAggregatesKey, requestTimelineKey, traceServerMetricsKey, diff --git a/packages/app/src/app/api/v1/derived-agentic-metrics/route.ts b/packages/app/src/app/api/v1/derived-agentic-metrics/route.ts deleted file mode 100644 index 373eb370d..000000000 --- a/packages/app/src/app/api/v1/derived-agentic-metrics/route.ts +++ /dev/null @@ -1,51 +0,0 @@ -import { STATS_VERSION } from '@semianalysisai/inferencex-db/queries/agentic-aggregates'; -import { getDb } from '@semianalysisai/inferencex-db/connection'; - -import { - getDerivedAgenticMetrics, - type DerivedAgenticMetricMap, -} from '@semianalysisai/inferencex-db/queries/derived-agentic-metrics'; - -import { cachedQuery } from '@/lib/api-cache'; - -import { idsQueryRoute } from '../id-routes'; - -export const dynamic = 'force-dynamic'; - -// blobOnly: the response is one entry per id with two numbers, but the -// derivation work parses thousands of JSONL records per blob — cache the -// computed result so a chart-refresh hits the warm path. -// -// The cache key is derived from STATS_VERSION (the payload governs the derived -// metrics read out of `aggregate_stats`). blobSet is write-once and nothing -// purges post-backfill, so a hand-written version string would serve stale -// data forever after a bump — deriving the key from the constant means a -// STATS_VERSION bump automatically rolls the cache namespace. -/** Version-derived blob-cache key namespace (exported for the key-derivation test). */ -export const CACHE_KEY_PREFIX = `derived-agentic-metrics-v${STATS_VERSION}`; - -const getCachedDerivedAgenticMetrics = cachedQuery( - (ids: number[]): Promise => getDerivedAgenticMetrics(getDb(), ids), - CACHE_KEY_PREFIX, - { blobOnly: true }, -); - -/** - * GET /api/v1/derived-agentic-metrics?ids=1,2,3 - * - * Returns per-id derived metrics computed live from the stored aiperf - * profile_export.jsonl blobs: - * - normalized_session_time_s: mean across sessions of session e2e time - * (Σ per-turn request_latency) rescaled by mean_load / session_load. - * - p90_prefill_tps_per_user: P90 of per-turn prefill TPS/user (ISL / TTFT) - * across every turn in every session. - * - p75/p90_normalized_e2e_400_s: percentile of per-request - * TTFT + 399 × observed ITL. - * - * Ids without a trace_replay blob or with unparseable records are omitted. - */ -export const GET = idsQueryRoute({ - maxIds: 200, - logLabel: 'derived agentic metrics', - fetch: getCachedDerivedAgenticMetrics, -}); diff --git a/packages/app/src/components/inference/InferenceContext.tsx b/packages/app/src/components/inference/InferenceContext.tsx index 6d5a2b935..8b837ca5a 100644 --- a/packages/app/src/components/inference/InferenceContext.tsx +++ b/packages/app/src/components/inference/InferenceContext.tsx @@ -60,12 +60,7 @@ import { } from '@/lib/exclusion'; import { filterRunsByModel, getDisplayLabel } from '@/lib/utils'; -import { - isAgenticOnlyXAxisMode, - useChartData, - X_AXIS_MODES, - type XAxisMode, -} from './hooks/useChartData'; +import { useChartData, X_AXIS_MODES, type XAxisMode } from './hooks/useChartData'; import { resolveComparisonEntries } from './utils/comparisonEntry'; import { comparisonDefaultGroup, @@ -610,40 +605,23 @@ export function InferenceProvider({ }, [labelScenarioKind, sequenceResolved, getUrlParam]); // Reconcile the x-axis mode with the scenario kind: - // - On mount with no `i_xmode` URL param: snap to the kind's natural default - // (interactivity for both agentic and fixed-sequence scenarios). The state was initialized - // to a SSR-stable constant so server and client render the same DOM; this - // effect fixes it up after hydration. - // - When the user later switches sequence kinds: snap to the new kind's - // natural default (the prior selection was for a different kind, so it - // doesn't carry over). + // - On mount with no `i_xmode` URL param: snap to the natural default + // (interactivity for both agentic and fixed-sequence scenarios). The state + // was initialized to a SSR-stable constant so server and client render the + // same DOM; this effect fixes it up after hydration. + // - When the user later switches sequence kinds: snap back to that default + // (the prior selection was for a different kind, so it doesn't carry over). const lastSeqKindRef = useRef | null>(null); useEffect(() => { const kind = sequenceKind(effectiveSequence); const isInitialMount = lastSeqKindRef.current === null; - const isAgenticOnlyMode = isAgenticOnlyXAxisMode(selectedXAxisMode); - // On a stale render where kind hasn't changed, bail unless the current - // mode is agentic-only and we just landed on a fixed-seq scenario — in - // that case force the snap so the chart doesn't try to plot trace-derived - // metrics against rows that have no trace_replay. - if (!isInitialMount && lastSeqKindRef.current === kind) { - if (kind === 'fixed-seq' && isAgenticOnlyMode) { - handleSetXAxisMode('interactivity'); - } - return; - } + // Stale render where the kind hasn't changed — nothing to reconcile. + if (!isInitialMount && lastSeqKindRef.current === kind) return; lastSeqKindRef.current = kind; - if ( - isInitialMount && - xAxisModeFromUrlRef.current && - !(kind === 'fixed-seq' && isAgenticOnlyMode) - ) { - // URL-restored agentic-only mode on a fixed-seq sequence makes no sense - // — fall through to the default snap below. - return; - } + // A URL-restored mode wins on first mount; later kind switches reset it. + if (isInitialMount && xAxisModeFromUrlRef.current) return; handleSetXAxisMode('interactivity'); - }, [effectiveSequence, selectedXAxisMode, handleSetXAxisMode]); + }, [effectiveSequence, handleSetXAxisMode]); // Reconcile selectedE2eXAxisMetric whenever the mode, sequence kind, or // agentic percentile changes. For fixed-seq the JSONB only carries diff --git a/packages/app/src/components/inference/hooks/useChartData.ts b/packages/app/src/components/inference/hooks/useChartData.ts index c125e9e9e..c5593ee98 100644 --- a/packages/app/src/components/inference/hooks/useChartData.ts +++ b/packages/app/src/components/inference/hooks/useChartData.ts @@ -39,41 +39,18 @@ import { /** * Chart x-axis variant selected by the mode buttons above the plot. This is * the single definition — InferenceContext (URL/state) and ChartDisplay - * (buttons, derived-metric remapping) import it from here. + * (buttons) import it from here. */ -export type XAxisMode = - | 'ttft' - | 'e2e' - | 'normalized-e2e' - | 'interactivity' - | 'session-time' - | 'prefill-tps'; - -export const X_AXIS_MODES: readonly XAxisMode[] = [ - 'ttft', - 'e2e', - 'normalized-e2e', - 'interactivity', - 'session-time', - 'prefill-tps', -]; +export type XAxisMode = 'ttft' | 'e2e' | 'interactivity'; -/** - * Modes whose x metric is derived from persisted per-request traces — - * these only exist for agentic scenarios (fixed-seq rows have no - * trace_replay blob to derive them from). - */ -export function isAgenticOnlyXAxisMode(mode: XAxisMode): boolean { - return mode === 'normalized-e2e' || mode === 'session-time' || mode === 'prefill-tps'; -} +export const X_AXIS_MODES: readonly XAxisMode[] = ['ttft', 'e2e', 'interactivity']; /** * Compute the set of benchmark_results.id values that sit on the * (e2e_latency, y) Pareto frontier within each (hwKey, precision, date) - * group. Used to restrict the non-e2e xmode charts (ttft, interactivity, - * session-time, prefill-tps) so they show *only* the points that win on - * end-to-end latency — preventing benchmark-hacking where a config tops - * one axis while tanking the other. + * group. Used to restrict the non-e2e xmode charts (ttft, interactivity) so + * they show *only* the points that win on end-to-end latency — preventing + * benchmark-hacking where a config tops one axis while tanking the other. * * Returns null when the y-metric has no roofline direction declared on * the e2e chart (caller falls back to no filtering in that case). @@ -220,7 +197,7 @@ export function useChartData( /** * Current x-axis mode. When set to anything other than 'e2e', the displayed * data is filtered to the (e2e-latency, y) Pareto frontier so the ttft / - * interactivity / session-time / prefill-tps charts show only points that + * interactivity charts show only points that * also win on end-to-end latency — preventing benchmark-hacking where a * config tops one metric while tanking the other. The 'e2e' mode is the * source of truth and keeps the full point set. @@ -529,7 +506,7 @@ export function useChartData( // non-optimal configs from view. // // Fixed-seq workloads keep the existing per-axis Pareto since - // there's no separate "session-time" notion of total latency — + // there's no separate session-level notion of total latency — // their e2e IS the request latency, so a TTFT hack there reads // honestly on e2e too. The anti-hack constraint is specifically // about multi-turn agentic where TTFT measures a tiny fraction diff --git a/packages/app/src/components/inference/types.ts b/packages/app/src/components/inference/types.ts index 8bdc04333..ae73710bd 100644 --- a/packages/app/src/components/inference/types.ts +++ b/packages/app/src/components/inference/types.ts @@ -258,7 +258,7 @@ export interface InferenceData extends Partial void; + selectedXAxisMode: 'ttft' | 'e2e' | 'interactivity'; + setSelectedXAxisMode: (mode: 'ttft' | 'e2e' | 'interactivity') => void; scaleType: 'auto' | 'linear' | 'log'; setScaleType: (type: 'auto' | 'linear' | 'log') => void; /** Coarse vendor / framework / agg-disagg / mtp-stp filters applied to the chart point set. */ diff --git a/packages/app/src/components/inference/ui/ChartDisplay.tsx b/packages/app/src/components/inference/ui/ChartDisplay.tsx index 76d3e6f41..6329df644 100644 --- a/packages/app/src/components/inference/ui/ChartDisplay.tsx +++ b/packages/app/src/components/inference/ui/ChartDisplay.tsx @@ -1,8 +1,5 @@ 'use client'; -import { - DISPLAY_MODEL_TO_DB, - NORMALIZED_E2E_OUTPUT_TOKENS, -} from '@semianalysisai/inferencex-constants'; +import { DISPLAY_MODEL_TO_DB } from '@semianalysisai/inferencex-constants'; import { track } from '@/lib/analytics'; import dynamic from 'next/dynamic'; import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; @@ -17,10 +14,7 @@ import type { OverlayData, TrendDataPoint, } from '@/components/inference/types'; -import { - processOverlayChartData, - selectUnofficialOverlayForMode, -} from '@/components/inference/utils'; +import { processOverlayChartData } from '@/components/inference/utils'; import { isRunComparisonEntry, makeRunComparisonEntry, @@ -59,12 +53,7 @@ import { sequenceKind, } from '@/lib/data-mappings'; import { useComparisonChangelogs } from '@/hooks/api/use-comparison-changelogs'; -import { - useDerivedAgenticMetrics, - type DerivedAgenticMetric, -} from '@/hooks/api/use-derived-agentic-metrics'; -import { isAgenticOnlyXAxisMode, type XAxisMode } from '@/components/inference/hooks/useChartData'; -import { isPersistedBenchmarkId } from '@/lib/benchmark-id'; +import type { XAxisMode } from '@/components/inference/hooks/useChartData'; import { useTrendData } from '@/components/inference/hooks/useTrendData'; import { getHardwareConfig, hardwareKeyMatchesAnyBase } from '@/lib/constants'; import { useLocale } from '@/lib/use-locale'; @@ -95,8 +84,6 @@ const STRINGS = { sourceUnofficial: 'Source: UNOFFICIAL', sourceOfficial: 'Source: SemiAnalysis InferenceX™', updated: 'Updated:', - normalizedE2eDisclaimer: - 'Normalized E2E requires persisted per-request traces, so unofficial-run overlays are unavailable for this experimental view.', selectDateRange: 'Select a date range or add a run to view GPU comparison', performanceOverTime: 'Performance Over Time', performanceOverTimeDesc: @@ -114,8 +101,6 @@ const STRINGS = { sourceUnofficial: '来源：非官方', sourceOfficial: '来源：SemiAnalysis InferenceX™', updated: '更新时间：', - normalizedE2eDisclaimer: - 'Normalized E2E 需要持久化的逐请求 trace 数据，因此该实验性视图不支持非官方运行覆盖。', selectDateRange: '请选择日期范围或添加运行以查看 GPU 对比', performanceOverTime: '性能趋势', performanceOverTimeDesc: '双击散点图上的数据点以追踪配置随时间的变化。', @@ -149,61 +134,8 @@ const X_AXIS_MODE_BUTTONS: { value: XAxisMode; label: string; labelZh: string }[ { value: 'interactivity', label: 'Interactivity', labelZh: '交互性' }, { value: 'e2e', label: 'E2E Latency', labelZh: '端到端延迟' }, { value: 'ttft', label: 'TTFT', labelZh: 'TTFT' }, - { value: 'normalized-e2e', label: 'Normalized E2E', labelZh: 'Normalized E2E' }, - { value: 'session-time', label: 'Session Time', labelZh: '会话时长' }, - { value: 'prefill-tps', label: 'Prefill TPS / user', labelZh: 'Prefill TPS / user' }, ]; -/** - * Presentation + data plumbing for the trace-derived x-axis modes (the - * agentic-only modes). One spec per mode keeps the x-label, chart heading, - * roofline corner, and derived-metric accessor in sync instead of scattering - * `selectedXAxisMode === …` conditionals through the render. - */ -interface DerivedXModeSpec { - xLabel: (percentileLabel: string) => string; - /** Chinese x-label; omit to reuse the English one (technical terms). */ - xLabelZh?: (percentileLabel: string) => string; - /** Chart heading suffix ("vs. …") shown above the plot. */ - heading: (percentileLabel: string) => string; - /** Chinese heading suffix; omit to reuse the English one. */ - headingZh?: (percentileLabel: string) => string; - rooflineCorner: 'upper_right' | 'upper_left'; - /** Pull the raw metric for this mode off the derived-metrics payload. */ - value: (m: DerivedAgenticMetric | undefined, percentile: string) => number | null | undefined; - /** Convert the raw metric to the plotted x value. */ - toX: (raw: number) => number; -} - -const DERIVED_X_MODE_SPECS: Partial> = { - 'session-time': { - xLabel: () => 'Mean Normalized Session Time (min)', - xLabelZh: () => '平均归一化会话时长（min）', - heading: () => 'vs. Mean Normalized Session Time', - headingZh: () => 'vs. 平均归一化会话时长', - rooflineCorner: 'upper_right', - value: (m) => m?.normalized_session_time_s, - toX: (raw) => raw / 60, - }, - 'normalized-e2e': { - xLabel: (pctl) => `${pctl} Normalized E2E @ ${NORMALIZED_E2E_OUTPUT_TOKENS} output tokens (s)`, - xLabelZh: (pctl) => `${pctl} Normalized E2E @ ${NORMALIZED_E2E_OUTPUT_TOKENS} 输出 token（s）`, - heading: (pctl) => `vs. ${pctl} Normalized E2E @ ${NORMALIZED_E2E_OUTPUT_TOKENS} output tokens`, - headingZh: (pctl) => `vs. ${pctl} Normalized E2E @ ${NORMALIZED_E2E_OUTPUT_TOKENS} 输出 token`, - rooflineCorner: 'upper_right', - value: (m, percentile) => - percentile === 'p75' ? m?.p75_normalized_e2e_400_s : m?.p90_normalized_e2e_400_s, - toX: (raw) => raw, - }, - 'prefill-tps': { - xLabel: () => 'P90 Prefill TPS per user (tok/s)', - heading: () => 'vs. P90 Prefill TPS / user', - rooflineCorner: 'upper_left', - value: (m) => m?.p90_prefill_tps_per_user, - toX: (raw) => raw, - }, -}; - const VIEW_MODE_OPTIONS: SegmentedToggleOption[] = [ { value: 'chart', @@ -263,9 +195,6 @@ export default function ChartDisplay() { totalDatesQueried, } = useComparisonChangelogs(selectedGPUs, selectedDateRange, dateRangeAvailableDates); - const [mounted, setMounted] = useState(false); - useEffect(() => setMounted(true), []); - const modelDbKeys = useMemo( () => DISPLAY_MODEL_TO_DB[selectedModel] ?? [selectedModel], [selectedModel], @@ -628,258 +557,191 @@ export default function ChartDisplay() { }, [effectiveGraphs, selectedXAxisMode]); const isAgenticSequence = sequenceKind(selectedSequence) === 'agentic'; - const useDerived = isAgenticSequence && isAgenticOnlyXAxisMode(selectedXAxisMode); - const derivedTargetIds = useMemo(() => { - if (!useDerived) return [] as number[]; - const ids = new Set(); - for (const graph of visibleGraphs) { - for (const point of graph.data) { - // Overlay-only agentic points carry no persisted id — skip them so we - // never request `?ids=0`/`?ids=NaN` (which 400s and errors the chart). - if (point.benchmark_type === 'agentic_traces' && isPersistedBenchmarkId(point.id)) { - ids.add(point.id); - } - } - } - return [...ids]; - }, [useDerived, visibleGraphs]); - const derivedQuery = useDerivedAgenticMetrics(derivedTargetIds, useDerived); - const derivedMetrics = derivedQuery.data; - const isDerivedLoading = - useDerived && - derivedTargetIds.length > 0 && - (derivedQuery.isPending || derivedQuery.isFetching) && - !derivedMetrics; - - // Set only when the user is on a derived (agentic-only) x-axis mode; the - // specs are module constants so this is referentially stable per mode. - const derivedSpec = useDerived ? DERIVED_X_MODE_SPECS[selectedXAxisMode] : undefined; - - const renderableGraphs = useMemo(() => { - if (!derivedSpec) return visibleGraphs; - if (!derivedMetrics) return visibleGraphs.map((graph) => ({ ...graph, data: [] })); - const xLabelFn = - locale === 'zh' && derivedSpec.xLabelZh ? derivedSpec.xLabelZh : derivedSpec.xLabel; - const xLabel = xLabelFn(selectedPercentile.toUpperCase()); - return visibleGraphs.map((graph) => { - const chartDefinition = { - ...graph.chartDefinition, - x_label: xLabel, - y_latency_limit: undefined, - [`${selectedYAxisMetric}_roofline` as keyof typeof graph.chartDefinition]: - derivedSpec.rooflineCorner, - }; - const data = graph.data - .map((point) => { - if (!isPersistedBenchmarkId(point.id)) return null; - const raw = derivedSpec.value(derivedMetrics[point.id], selectedPercentile); - if (raw === null || raw === undefined || !Number.isFinite(raw)) return null; - return { ...point, x: derivedSpec.toX(raw) }; - }) - .filter((point): point is NonNullable => point !== null); - return { ...graph, chartDefinition, data }; - }); - }, [derivedSpec, visibleGraphs, derivedMetrics, selectedYAxisMetric, selectedPercentile, locale]); - - const displayGraphs = - isFirstLoad || isDerivedLoading - ? [ - - - - - , - ] - : renderableGraphs.length === 0 - ? [] - : renderableGraphs.map((graph, graphIndex) => { - const isTimelineMode = Boolean( - selectedDateRange.startDate && selectedDateRange.endDate && selectedGPUs.length > 0, - ); - const replayAvailable = getViewMode(graphIndex) === 'chart' && !isTimelineMode; - return ( -
-
- + + + + , + ] + : visibleGraphs.length === 0 + ? [] + : visibleGraphs.map((graph, graphIndex) => { + const isTimelineMode = Boolean( + selectedDateRange.startDate && selectedDateRange.endDate && selectedGPUs.length > 0, + ); + const replayAvailable = getViewMode(graphIndex) === 'chart' && !isTimelineMode; + return ( +
+
+ handleViewModeChange(graphIndex, v)} + ariaLabel={t.viewMode} + testId={`inference-view-toggle-${graphIndex}`} + /> + } + hideImageExport={getViewMode(graphIndex) === 'table'} + setIsLegendExpanded={setIsLegendExpanded} + exportFileName={`InferenceX_${selectedModel}_${graph.chartDefinition.chartType}`} + onExportMp4={ + replayAvailable ? () => replayHandlesRef.current[graphIndex]?.open() : undefined + } + onExportCsv={() => { + const candidateVisibleData = isTimelineMode + ? graph.data.filter((d) => activeDates.has(`${d.date}_${d.hwKey}`)) + : graph.data; + const overlay = overlayDataByChartType[graph.chartDefinition.chartType]; + const { officialRows: visibleData, overlayRows: visibleOverlayRowsForExport } = isTimelineMode - ? 'gpu_timeseries' - : graph.chartDefinition.chartType === 'e2e' - ? 'latency' - : 'interactivity' - } - leadingControls={ - handleViewModeChange(graphIndex, v)} - ariaLabel={t.viewMode} - testId={`inference-view-toggle-${graphIndex}`} - /> - } - hideImageExport={getViewMode(graphIndex) === 'table'} - setIsLegendExpanded={setIsLegendExpanded} - exportFileName={`InferenceX_${selectedModel}_${graph.chartDefinition.chartType}`} - onExportMp4={ - replayAvailable - ? () => replayHandlesRef.current[graphIndex]?.open() - : undefined - } - onExportCsv={() => { - const candidateVisibleData = isTimelineMode - ? graph.data.filter((d) => activeDates.has(`${d.date}_${d.hwKey}`)) - : graph.data; - const overlay = selectUnofficialOverlayForMode( - selectedXAxisMode, - graph.chartDefinition.chartType, - overlayDataByChartType, - ); - const { - officialRows: visibleData, - overlayRows: visibleOverlayRowsForExport, - } = isTimelineMode ? { officialRows: candidateVisibleData, overlayRows: [] } : visibleComparisonRows(candidateVisibleData, overlay); - const { headers, rows } = inferenceChartToCsv( - visibleData, - graph.model, - graph.sequence, - ); - // Match warnings against the same series the chart annotates, - // including visible unofficial-run overlay series. - const issueNotes = matchKnownConfigIssues(graph.model, [ - ...visibleData, - ...visibleOverlayRowsForExport, - ]).map((issue) => - knownIssueCsvNote(issue, getDisplayLabel(getHardwareConfig(issue.hwKey))), - ); - exportToCsv( - `InferenceX_${selectedModel}_${graph.chartDefinition.chartType}`, - headers, - rows, - issueNotes, - ); - }} - /> - - {(() => { - const chartCaption = ( - <> -

- {metricTitle(graph.chartDefinition, selectedYAxisMetric, locale)}{' '} - {(() => { - // For Input metrics with dynamic x-axis, use dynamic heading. - // Classify off the ENGLISH title — the localized one has no - // 'input' substring to match on zh pages. - const isInputMetric = metricTitle( - graph.chartDefinition, - selectedYAxisMetric, - 'en', - ) - .toLowerCase() - .includes('input'); - if ( - graph.chartDefinition.chartType === 'interactivity' && - isInputMetric && - selectedXAxisMetric - ) { - if (selectedXAxisMetric === 'p99_ttft') { - return t.vsTtft('P99'); - } else if (selectedXAxisMetric === 'median_ttft') { - return t.vsTtft('Median'); - } + const { headers, rows } = inferenceChartToCsv( + visibleData, + graph.model, + graph.sequence, + ); + // Match warnings against the same series the chart annotates, + // including visible unofficial-run overlay series. + const issueNotes = matchKnownConfigIssues(graph.model, [ + ...visibleData, + ...visibleOverlayRowsForExport, + ]).map((issue) => + knownIssueCsvNote(issue, getDisplayLabel(getHardwareConfig(issue.hwKey))), + ); + exportToCsv( + `InferenceX_${selectedModel}_${graph.chartDefinition.chartType}`, + headers, + rows, + issueNotes, + ); + }} + /> + + {(() => { + const chartCaption = ( + <> +

+ {metricTitle(graph.chartDefinition, selectedYAxisMetric, locale)}{' '} + {(() => { + // For Input metrics with dynamic x-axis, use dynamic heading. + // Classify off the ENGLISH title — the localized one has no + // 'input' substring to match on zh pages. + const isInputMetric = metricTitle( + graph.chartDefinition, + selectedYAxisMetric, + 'en', + ) + .toLowerCase() + .includes('input'); + if ( + graph.chartDefinition.chartType === 'interactivity' && + isInputMetric && + selectedXAxisMetric + ) { + if (selectedXAxisMetric === 'p99_ttft') { + return t.vsTtft('P99'); + } else if (selectedXAxisMetric === 'median_ttft') { + return t.vsTtft('Median'); } + } - // The e2e chart heading follows the branch-level x-axis mode - // selector, including agentic-only derived metrics. - if (graph.chartDefinition.chartType === 'e2e') { - const modeSpec = DERIVED_X_MODE_SPECS[selectedXAxisMode]; - if (modeSpec) { - const heading = - locale === 'zh' && modeSpec.headingZh - ? modeSpec.headingZh - : modeSpec.heading; - return heading(selectedPercentile.toUpperCase()); - } - if (selectedE2eXAxisMetric?.endsWith('_ttft')) { - const percentile = selectedE2eXAxisMetric.replace(/_ttft$/u, ''); - const word = - percentile === 'median' ? 'Median' : percentile.toUpperCase(); - return t.vsTtft(word); - } - return isAgenticSequence - ? t.vsE2eLatency(selectedPercentile.toUpperCase()) - : t.vsE2eLatency(); + // The e2e chart heading follows the branch-level x-axis + // mode selector. + if (graph.chartDefinition.chartType === 'e2e') { + if (selectedE2eXAxisMetric?.endsWith('_ttft')) { + const percentile = selectedE2eXAxisMetric.replace(/_ttft$/u, ''); + const word = + percentile === 'median' ? 'Median' : percentile.toUpperCase(); + return t.vsTtft(word); } + return isAgenticSequence + ? t.vsE2eLatency(selectedPercentile.toUpperCase()) + : t.vsE2eLatency(); + } - // Fall back to configured heading - const configured = - graph.chartDefinition[ - `${selectedYAxisMetric}_heading` as keyof typeof graph.chartDefinition - ] || graph.chartDefinition.heading; - return locale === 'zh' ? zhHeading(String(configured)) : configured; - })()} -

-

- {getModelLabel(graph.model as Model)} •{' '} - {selectedPrecisions - .map((prec) => getPrecisionLabel(prec as Precision)) - .join(', ')}{' '} - • {getSequenceLabel(graph.sequence as Sequence)} •{' '} - {isUnofficialRun ? t.sourceUnofficial : t.sourceOfficial} - {selectedRunDate && ( - <> - {' '} - • {t.updated}{' '} - {new Date(`${selectedRunDate}T00:00:00Z`).toLocaleDateString( - locale === 'zh' ? 'zh-CN' : 'en-US', - { - year: 'numeric', - month: '2-digit', - day: '2-digit', - timeZone: 'UTC', - }, - )} - - )} -

- - {isUnofficialRun && selectedXAxisMode === 'normalized-e2e' && ( -

- {t.normalizedE2eDisclaimer} -

+ // Fall back to configured heading + const configured = + graph.chartDefinition[ + `${selectedYAxisMetric}_heading` as keyof typeof graph.chartDefinition + ] || graph.chartDefinition.heading; + return locale === 'zh' ? zhHeading(String(configured)) : configured; + })()} +

+

+ {getModelLabel(graph.model as Model)} •{' '} + {selectedPrecisions + .map((prec) => getPrecisionLabel(prec as Precision)) + .join(', ')}{' '} + • {getSequenceLabel(graph.sequence as Sequence)} •{' '} + {isUnofficialRun ? t.sourceUnofficial : t.sourceOfficial} + {selectedRunDate && ( + <> + {' '} + • {t.updated}{' '} + {new Date(`${selectedRunDate}T00:00:00Z`).toLocaleDateString( + locale === 'zh' ? 'zh-CN' : 'en-US', + { + year: 'numeric', + month: '2-digit', + day: '2-digit', + timeZone: 'UTC', + }, + )} + )} - +

+ + + + ); + + if (getViewMode(graphIndex) === 'table') { + const overlay = overlayDataByChartType[graph.chartDefinition.chartType]; + const { officialRows, overlayRows } = visibleComparisonRows( + graph.data, + overlay, + ); + return ( + <> + {chartCaption} + ); + } - if (getViewMode(graphIndex) === 'table') { - const overlay = selectUnofficialOverlayForMode( - selectedXAxisMode, - graph.chartDefinition.chartType, - overlayDataByChartType, - ); - const { officialRows, overlayRows } = visibleComparisonRows( - graph.data, - overlay, - ); - return ( - <> - {chartCaption} - - - ); - } - - return selectedGPUs.length > 0 && - ((selectedDateRange.startDate && selectedDateRange.endDate) || - selectedDates.length > 0) ? ( - 0 && + ((selectedDateRange.startDate && selectedDateRange.endDate) || + selectedDates.length > 0) ? ( + + ) : ( +
+ - ) : ( -
- - {selectedGPUs.length > 0 && - (!selectedDateRange.startDate || !selectedDateRange.endDate) && - selectedDates.length === 0 && ( -
-

- {t.selectDateRange} -

-
- )} -
- ); - })()} - {replayAvailable && ( - { - replayHandlesRef.current[graphIndex] = handle; - }} - parentChartId={`chart-${graphIndex}`} - chartDefinition={graph.chartDefinition} - yLabel={metricLabel(graph.chartDefinition, selectedYAxisMetric, locale)} - xLabel={graph.chartDefinition.x_label} - /> - )} - -
-
- ); - }); + {selectedGPUs.length > 0 && + (!selectedDateRange.startDate || !selectedDateRange.endDate) && + selectedDates.length === 0 && ( +
+

+ {t.selectDateRange} +

+
+ )} + + ); + })()} + {replayAvailable && ( + { + replayHandlesRef.current[graphIndex] = handle; + }} + parentChartId={`chart-${graphIndex}`} + chartDefinition={graph.chartDefinition} + yLabel={metricLabel(graph.chartDefinition, selectedYAxisMetric, locale)} + xLabel={graph.chartDefinition.x_label} + /> + )} + +
+
+ ); + }); return (
@@ -1001,12 +847,7 @@ export default function ChartDisplay() { data-testid="x-axis-mode-buttons" className="flex-wrap justify-center gap-x-1 gap-y-1.5 sm:gap-x-1.5" > - {X_AXIS_MODE_BUTTONS.filter(({ value }) => { - if (!isAgenticOnlyXAxisMode(value)) return true; - // Before mount, render all buttons so SSR and first client render match. - if (!mounted) return true; - return isAgenticSequence; - }).map(({ value, label, labelZh }) => ( + {X_AXIS_MODE_BUTTONS.map(({ value, label, labelZh }) => ( { - const overlays = { e2e: { id: 'e2e' }, interactivity: { id: 'interactivity' } }; - - it('suppresses raw unofficial E2E data for normalized E2E mode', () => { - expect(selectUnofficialOverlayForMode('normalized-e2e', 'e2e', overlays)).toBeNull(); - }); - - it('preserves matching unofficial overlays for supported modes', () => { - expect(selectUnofficialOverlayForMode('e2e', 'e2e', overlays)).toBe(overlays.e2e); - expect(selectUnofficialOverlayForMode('interactivity', 'interactivity', overlays)).toBe( - overlays.interactivity, - ); - }); -}); +import { filterDataByCostLimit, processOverlayChartData } from '@/components/inference/utils'; // --------------------------------------------------------------------------- // fixture factories diff --git a/packages/app/src/components/inference/utils.ts b/packages/app/src/components/inference/utils.ts index 2b8074d9c..e88e5b0da 100644 --- a/packages/app/src/components/inference/utils.ts +++ b/packages/app/src/components/inference/utils.ts @@ -10,20 +10,6 @@ import { resolveXAxisField } from '@/components/inference/utils/resolveXAxisFiel import type { ChartDefinition, InferenceData, YAxisMetricKey } from './types'; -/** - * Select the matching unofficial-run overlay for a chart mode. Normalized E2E - * is intentionally excluded: unofficial benchmark rows do not include the - * persisted per-request trace needed to normalize before taking percentiles. - */ -export function selectUnofficialOverlayForMode( - xAxisMode: string, - chartType: 'e2e' | 'interactivity', - overlays: { e2e: T | null; interactivity: T | null }, -): T | null { - if (xAxisMode === 'normalized-e2e') return null; - return overlays[chartType]; -} - /** * Filters data points based on cost limits defined in the chart definition. * Only applies filtering for cost-related metrics, and only filters based on diff --git a/packages/app/src/components/inference/utils/e2eFrontier.ts b/packages/app/src/components/inference/utils/e2eFrontier.ts index 588afdd3e..65c3434a8 100644 --- a/packages/app/src/components/inference/utils/e2eFrontier.ts +++ b/packages/app/src/components/inference/utils/e2eFrontier.ts @@ -2,7 +2,7 @@ * @file e2eFrontier.ts * @description Shared seed for the anti-benchmark-hacking roofline restriction. * - * On the non-e2e xmode charts (interactivity, ttft, session-time, prefill-tps) + * On the non-e2e xmode charts (interactivity, ttft) * the roofline is restricted to the configs that ALSO win on end-to-end latency, * so a config can't top interactivity while tanking decode (or vice versa). Both * the official path (`useChartData` → benchmark ids → `isOnE2eFrontier`) and the diff --git a/packages/app/src/hooks/api/use-derived-agentic-metrics.test.ts b/packages/app/src/hooks/api/use-derived-agentic-metrics.test.ts deleted file mode 100644 index 2e54f4187..000000000 --- a/packages/app/src/hooks/api/use-derived-agentic-metrics.test.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { describe, expect, it } from 'vitest'; - -import { chunkDerivedAgenticMetricIds } from './use-derived-agentic-metrics'; - -describe('chunkDerivedAgenticMetricIds', () => { - it('keeps every id while respecting the API limit', () => { - const ids = Array.from({ length: 401 }, (_, index) => index + 1); - const chunks = chunkDerivedAgenticMetricIds(ids); - - expect(chunks.map((chunk) => chunk.length)).toEqual([200, 200, 1]); - expect(chunks.flat()).toEqual(ids); - }); -}); diff --git a/packages/app/src/hooks/api/use-derived-agentic-metrics.ts b/packages/app/src/hooks/api/use-derived-agentic-metrics.ts deleted file mode 100644 index 388563d96..000000000 --- a/packages/app/src/hooks/api/use-derived-agentic-metrics.ts +++ /dev/null @@ -1,55 +0,0 @@ -import { bulkIdsFetcher, useBulkIdsQuery } from './benchmark-id-query'; - -export interface DerivedAgenticMetric { - id: number; - /** Mean across sessions of e2e time (Σ per-turn request_latency) rescaled - * by mean_load / session_load. Null when the JSONL had no usable records. */ - normalized_session_time_s: number | null; - /** P90 of per-turn ISL/TTFT across every turn in every session. - * Null when no prefill rates could be computed. */ - p90_prefill_tps_per_user: number | null; - /** P75 normalized per-request E2E at a fixed 400-token output length. */ - p75_normalized_e2e_400_s: number | null; - /** P90 normalized per-request E2E at a fixed 400-token output length. */ - p90_normalized_e2e_400_s: number | null; -} - -export type DerivedAgenticMetricMap = Record; - -const MAX_IDS_PER_REQUEST = 200; - -export function chunkDerivedAgenticMetricIds(ids: number[]): number[][] { - const chunks: number[][] = []; - for (let i = 0; i < ids.length; i += MAX_IDS_PER_REQUEST) { - chunks.push(ids.slice(i, i + MAX_IDS_PER_REQUEST)); - } - return chunks; -} - -const fetchChunk = bulkIdsFetcher('derived-agentic-metrics'); - -// Unlike the other bulk endpoints, dashboards can put >200 agentic points on -// screen at once, so this fetcher splits the id set across parallel requests -// to stay under the route's MAX_IDS_PER_REQUEST. -async function fetchDerivedAgenticMetrics( - ids: number[], - signal?: AbortSignal, -): Promise { - if (ids.length === 0) return {}; - const maps = await Promise.all( - chunkDerivedAgenticMetricIds(ids).map((chunk) => fetchChunk(chunk, signal)), - ); - return Object.assign({}, ...maps) as DerivedAgenticMetricMap; -} - -/** - * Fetch per-id derived agentic metrics (session time + p90 prefill TPS/user) - * computed live from the stored aiperf profile_export.jsonl. Used to drive - * the "Session Time" and "Prefill TPS/user" chart variants. - * - * Ids without a trace_replay blob (older or non-aiperf agentic runs) are - * silently omitted from the response. - */ -export function useDerivedAgenticMetrics(ids: number[], enabled = true) { - return useBulkIdsQuery('derived-agentic-metrics', ids, enabled, fetchDerivedAgenticMetrics); -} diff --git a/packages/app/src/lib/benchmark-id.ts b/packages/app/src/lib/benchmark-id.ts index b1ccb8bcc..6b1030df3 100644 --- a/packages/app/src/lib/benchmark-id.ts +++ b/packages/app/src/lib/benchmark-id.ts @@ -8,7 +8,7 @@ * * A bare `typeof id === 'number'` check is NOT enough: `NaN` and `0` are both * `number` yet neither is a real row. Passing them to the id-keyed endpoints - * (`/api/v1/derived-agentic-metrics?ids=…`, `…?id=…`) yields a 400 (the routes + * (`/api/v1/agentic-aggregates?ids=…`, `…?id=…`) yields a 400 (the routes * filter to `Number.isFinite(n) && n > 0`), and building an * `/inference/agentic/` link out of one points at a non-existent row. * diff --git a/packages/constants/src/agentic.ts b/packages/constants/src/agentic.ts deleted file mode 100644 index 42eab306e..000000000 --- a/packages/constants/src/agentic.ts +++ /dev/null @@ -1,2 +0,0 @@ -/** Fixed output length used by the experimental normalized-E2E chart metric. */ -export const NORMALIZED_E2E_OUTPUT_TOKENS = 400; diff --git a/packages/constants/src/index.ts b/packages/constants/src/index.ts index 7d3d6783a..e767e500c 100644 --- a/packages/constants/src/index.ts +++ b/packages/constants/src/index.ts @@ -1,4 +1,3 @@ -export * from './agentic'; export * from './framework-aliases'; export * from './github'; export * from './gpu-keys'; diff --git a/packages/db/src/backfill-aggregate-stats.ts b/packages/db/src/backfill-aggregate-stats.ts index 8aaa1214c..a50f6ea02 100644 --- a/packages/db/src/backfill-aggregate-stats.ts +++ b/packages/db/src/backfill-aggregate-stats.ts @@ -94,12 +94,16 @@ async function main(): Promise { } let stats: AggregateStats; - if (row.aggregate_stats?.version === 3) { + // v3 onwards → current is a profile-only change (the server-derived + // fields haven't changed since v3), so skip re-reading the huge server + // blob and carry its KV/prefix distributions forward. + const storedVersion = row.aggregate_stats?.version; + if (storedVersion !== undefined && storedVersion >= 3 && storedVersion < STATS_VERSION) { const profileStats = await computeAggregateStats({ profileBlob: row.profile_export_jsonl_gz, serverBlob: null, }); - stats = mergeProfileStatsUpgrade(row.aggregate_stats, profileStats); + stats = mergeProfileStatsUpgrade(row.aggregate_stats!, profileStats); } else { const [serverRow] = await sql<{ server_metrics_json_gz: Buffer | null }[]>` select server_metrics_json_gz diff --git a/packages/db/src/etl/compute-aggregate-stats.test.ts b/packages/db/src/etl/compute-aggregate-stats.test.ts index 7b745c098..1a237664d 100644 --- a/packages/db/src/etl/compute-aggregate-stats.test.ts +++ b/packages/db/src/etl/compute-aggregate-stats.test.ts @@ -66,12 +66,9 @@ describe('computeAggregateStats', () => { expect(stats.osl).toBeNull(); expect(stats.kvCacheUtil).toBeNull(); expect(stats.prefixCacheHitRate).toBeNull(); - expect(stats.normalizedSessionTimeS).toBeNull(); - expect(stats.p90PrefillTpsPerUser).toBeNull(); - expect(stats.normalizedE2e400).toBeNull(); }); - it('computes ISL/OSL percentiles + derived metrics from the profile blob', async () => { + it('computes ISL/OSL percentiles from the profile blob', async () => { const profileBlob = makeProfileBlob([ { isl: 100, osl: 50, rl: 1000, ttft: 100 }, { isl: 200, osl: 75, rl: 2000, ttft: 200 }, @@ -87,16 +84,6 @@ describe('computeAggregateStats', () => { // Server-side metrics still null when there's no server blob. expect(stats.kvCacheUtil).toBeNull(); expect(stats.prefixCacheHitRate).toBeNull(); - - // Derived: prefill TPS per turn = isl / (ttft/1000) = 1000 for each, so p90 = 1000. - expect(stats.p90PrefillTpsPerUser).toBeCloseTo(1000, 6); - // Normalized session time: T̃_i = T_i × (mean_load / load_i), then mean. - // loads = [150, 275, 400], mean_load = 275 - // scaled times (s) = [1×275/150, 2×275/275, 3×275/400] = [1.8333, 2, 2.0625] - // mean ≈ 1.9653 - expect(stats.normalizedSessionTimeS).toBeCloseTo(1.9653, 3); - expect(stats.normalizedE2e400?.n).toBe(3); - expect(stats.normalizedE2e400?.p90).toBeGreaterThan(0); }); it('computes KV util + prefix hit rate from the server blob alone', async () => { @@ -112,9 +99,6 @@ describe('computeAggregateStats', () => { // Profile-derived metrics absent. expect(stats.isl).toBeNull(); expect(stats.osl).toBeNull(); - expect(stats.normalizedSessionTimeS).toBeNull(); - expect(stats.p90PrefillTpsPerUser).toBeNull(); - expect(stats.normalizedE2e400).toBeNull(); }); it('tolerates a malformed profile blob by leaving its metrics null', async () => { @@ -123,9 +107,6 @@ describe('computeAggregateStats', () => { const stats = await computeAggregateStats({ profileBlob: garbage, serverBlob: null }); expect(stats.isl).toBeNull(); expect(stats.osl).toBeNull(); - expect(stats.normalizedSessionTimeS).toBeNull(); - expect(stats.p90PrefillTpsPerUser).toBeNull(); - expect(stats.normalizedE2e400).toBeNull(); // Version still set so the row is considered "computed". expect(stats.version).toBe(STATS_VERSION); }); @@ -145,8 +126,34 @@ describe('mergeProfileStatsUpgrade', () => { const merged = mergeProfileStatsUpgrade(existing, profile); expect(merged.version).toBe(STATS_VERSION); expect(merged.isl?.mean).toBe(100); - expect(merged.normalizedE2e400?.p90).toBeGreaterThan(0); expect(merged.kvCacheUtil).toEqual(existing.kvCacheUtil); expect(merged.prefixCacheHitRate).toEqual(existing.prefixCacheHitRate); }); + + it('drops the retired derived fields when upgrading a pre-v6 bundle', async () => { + // Bundles written before v6 carry normalizedSessionTimeS / + // p90PrefillTpsPerUser / normalizedE2e400 — the upgrade must not + // resurrect them into the new bundle. + const legacy = { + version: STATS_VERSION - 1, + isl: null, + osl: null, + kvCacheUtil: { mean: 0.4, p50: 0.4, p75: 0.5, p90: 0.6, p99: 0.7, n: 3 }, + prefixCacheHitRate: null, + normalizedSessionTimeS: 999, + p90PrefillTpsPerUser: 999, + normalizedE2e400: { mean: 1, p50: 1, p75: 1, p90: 2, p99: 3, n: 5 }, + }; + const profile = await computeAggregateStats({ + profileBlob: makeProfileBlob([{ isl: 100, osl: 100, rl: 2080, ttft: 100 }]), + serverBlob: null, + }); + + const merged = mergeProfileStatsUpgrade(legacy, profile); + expect(merged.version).toBe(STATS_VERSION); + expect(merged.kvCacheUtil).toEqual(legacy.kvCacheUtil); + expect(merged).not.toHaveProperty('normalizedSessionTimeS'); + expect(merged).not.toHaveProperty('p90PrefillTpsPerUser'); + expect(merged).not.toHaveProperty('normalizedE2e400'); + }); }); diff --git a/packages/db/src/etl/compute-aggregate-stats.ts b/packages/db/src/etl/compute-aggregate-stats.ts index f0385f6ec..d4a76e27b 100644 --- a/packages/db/src/etl/compute-aggregate-stats.ts +++ b/packages/db/src/etl/compute-aggregate-stats.ts @@ -1,9 +1,8 @@ /** * Pre-compute the per-row aggregate stats for an `agentic_trace_replay` * blob pair. The output lands in the `aggregate_stats` JSONB column so the - * detail page can serve the "Aggregates across configs" view and the - * derived chart x-axis modes from a single SQL row read, instead of - * parsing the raw blobs on demand. + * detail page can serve the "Aggregates across configs" view from a single + * SQL row read, instead of parsing the raw blobs on demand. * * Shape is intentionally versioned — bump `STATS_VERSION` whenever the * computation changes so the backfill script knows which rows to recompute. @@ -12,7 +11,6 @@ import { gunzipSync } from 'node:zlib'; import { gunzipJsonWithinLimit, streamCollectKeys } from './gzip-json-stream'; -import { computeDerivedFromBlob } from '../queries/derived-agentic-metrics'; import { STATS_VERSION, extractIslOsl, @@ -29,12 +27,19 @@ export interface AggregateStats { osl: MetricPercentiles | null; kvCacheUtil: MetricPercentiles | null; prefixCacheHitRate: MetricPercentiles | null; - /** Mean of (per-session e2e time × mean_load / session_load) across sessions. */ - normalizedSessionTimeS: number | null; - /** P90 of per-turn ISL/TTFT pooled across every session's turns. */ - p90PrefillTpsPerUser: number | null; - /** Per-request normalized E2E distribution at a fixed 400-token OSL. */ - normalizedE2e400: MetricPercentiles | null; +} + +/** + * The subset of an older-version bundle a profile-only upgrade carries + * forward. Pre-v6 bundles also carry the since-retired derived metrics + * (normalizedSessionTimeS, p90PrefillTpsPerUser, normalizedE2e400) — spreading + * `profile` first drops them from the merged result. + */ +interface ProfileUpgradeCarryover { + isl: MetricPercentiles | null; + osl: MetricPercentiles | null; + kvCacheUtil: MetricPercentiles | null; + prefixCacheHitRate: MetricPercentiles | null; } /** @@ -43,17 +48,13 @@ export interface AggregateStats { * while preserving its already-computed KV/cache distributions. */ export function mergeProfileStatsUpgrade( - existing: Omit & { - normalizedE2e400?: MetricPercentiles | null; - }, + existing: ProfileUpgradeCarryover, profile: AggregateStats, ): AggregateStats { return { ...profile, isl: profile.isl ?? existing.isl, osl: profile.osl ?? existing.osl, - normalizedSessionTimeS: profile.normalizedSessionTimeS ?? existing.normalizedSessionTimeS, - p90PrefillTpsPerUser: profile.p90PrefillTpsPerUser ?? existing.p90PrefillTpsPerUser, kvCacheUtil: existing.kvCacheUtil, prefixCacheHitRate: existing.prefixCacheHitRate, }; @@ -92,9 +93,6 @@ export async function computeAggregateStats(args: { }): Promise { let islPct: MetricPercentiles | null = null; let oslPct: MetricPercentiles | null = null; - let normalized: number | null = null; - let prefillP90: number | null = null; - let normalizedE2e400: MetricPercentiles | null = null; if (args.profileBlob) { try { @@ -102,10 +100,6 @@ export async function computeAggregateStats(args: { const { isl, osl } = extractIslOsl(jsonl); islPct = percentilesOf(isl); oslPct = percentilesOf(osl); - const derived = computeDerivedFromBlob(jsonl); - normalized = derived.normalized_session_time_s; - prefillP90 = derived.p90_prefill_tps_per_user; - normalizedE2e400 = derived.normalized_e2e_400; } catch { // ignore malformed blob — leave nulls } @@ -136,8 +130,5 @@ export async function computeAggregateStats(args: { osl: oslPct, kvCacheUtil: kvPct, prefixCacheHitRate: prefixPct, - normalizedSessionTimeS: normalized, - p90PrefillTpsPerUser: prefillP90, - normalizedE2e400, }; } diff --git a/packages/db/src/queries/agentic-aggregates.test.ts b/packages/db/src/queries/agentic-aggregates.test.ts index 0c4dbc890..d0daa2a39 100644 --- a/packages/db/src/queries/agentic-aggregates.test.ts +++ b/packages/db/src/queries/agentic-aggregates.test.ts @@ -129,9 +129,6 @@ interface WrittenStats { osl: unknown; kvCacheUtil: { mean: number } | null; prefixCacheHitRate: unknown; - normalizedSessionTimeS: number | null; - p90PrefillTpsPerUser: number | null; - normalizedE2e400: unknown; } /** Capture SQL template text + bound values for the write-back assertions. */ @@ -228,11 +225,11 @@ describe('getAgenticAggregates write-back', () => { expect(written.version).toBe(STATS_VERSION); // Server field FRESHLY recomputed (0.25), not the stale 0.9 carried forward. expect(written.kvCacheUtil?.mean).toBeCloseTo(0.25, 6); - // Derived fields FRESHLY recomputed (not the stale 999s). - expect(written.normalizedSessionTimeS).toBeCloseTo(3, 6); - expect(written.p90PrefillTpsPerUser).toBeCloseTo(200, 6); - expect(written.normalizedE2e400).not.toBeNull(); expect(written.isl).not.toBeNull(); + // The retired derived fields must not survive the recompute. + expect(written).not.toHaveProperty('normalizedSessionTimeS'); + expect(written).not.toHaveProperty('p90PrefillTpsPerUser'); + expect(written).not.toHaveProperty('normalizedE2e400'); }); it('does not write back for an id whose profile blob is missing/malformed', async () => { diff --git a/packages/db/src/queries/agentic-aggregates.ts b/packages/db/src/queries/agentic-aggregates.ts index 4917dc4c6..901defb9f 100644 --- a/packages/db/src/queries/agentic-aggregates.ts +++ b/packages/db/src/queries/agentic-aggregates.ts @@ -24,7 +24,6 @@ import { streamObject } from 'stream-json/streamers/stream-object.js'; import { gunzipJsonWithinLimit } from '../etl/gzip-json-stream'; import type { DbClient } from '../connection.js'; -import { computeDerivedFromBlob } from './derived-agentic-metrics'; import { extractIslOsl, fetchAggregateStatsRows, @@ -316,12 +315,8 @@ export async function getAgenticAggregates( const oslPct = percentilesOf(osl); result[id].isl = islPct; result[id].osl = oslPct; - // Recompute the profile-derived fields too (same jsonl, no extra - // read) so the self-healed bundle is a faithful full recompute — not - // a carry-forward of stale derived numbers stamped with a new - // version. Server-derived fields are filled in Pass 2 (or stay null - // when the server blob is absent, which is the correct complete value). - const derived = computeDerivedFromBlob(jsonl); + // Server-derived fields are filled in Pass 2 (or stay null when the + // server blob is absent, which is the correct complete value). pendingById.set(id, { traceReplayId: Number(row.trace_replay_id), stats: { @@ -330,9 +325,6 @@ export async function getAgenticAggregates( osl: oslPct, kvCacheUtil: null, prefixCacheHitRate: null, - normalizedSessionTimeS: derived.normalized_session_time_s, - p90PrefillTpsPerUser: derived.p90_prefill_tps_per_user, - normalizedE2e400: derived.normalized_e2e_400, }, }); } catch { @@ -404,8 +396,6 @@ interface AggregateStatsRow { osl: MetricPercentiles | null; kvCacheUtil: MetricPercentiles | null; prefixCacheHitRate: MetricPercentiles | null; - normalizedSessionTimeS: number | null; - p90PrefillTpsPerUser: number | null; } /** @@ -419,9 +409,6 @@ interface FullAggregateStats { osl: MetricPercentiles | null; kvCacheUtil: MetricPercentiles | null; prefixCacheHitRate: MetricPercentiles | null; - normalizedSessionTimeS: number | null; - p90PrefillTpsPerUser: number | null; - normalizedE2e400: MetricPercentiles | null; } function blankAggregate(id: number): AgenticAggregate { diff --git a/packages/db/src/queries/agentic-shared.ts b/packages/db/src/queries/agentic-shared.ts index d8673a075..d704abe02 100644 --- a/packages/db/src/queries/agentic-shared.ts +++ b/packages/db/src/queries/agentic-shared.ts @@ -31,8 +31,13 @@ import type { DbClient } from '../connection.js'; * * v5: reject osl <= 0 in extractTurn to exclude cancelled/empty-output turns * whose decode-interval math would explode normalized E2E to thousands of seconds. + * + * v6: drop the retired per-point derived metrics (normalizedSessionTimeS, + * p90PrefillTpsPerUser, normalizedE2e400) along with the experimental chart + * modes they fed. The bundle is now isl/osl + the server-derived kvCacheUtil + * and prefixCacheHitRate. */ -export const STATS_VERSION = 5; +export const STATS_VERSION = 6; interface ProfileRecord { metadata?: { benchmark_phase?: string }; diff --git a/packages/db/src/queries/derived-agentic-metrics.test.ts b/packages/db/src/queries/derived-agentic-metrics.test.ts deleted file mode 100644 index a39de6700..000000000 --- a/packages/db/src/queries/derived-agentic-metrics.test.ts +++ /dev/null @@ -1,256 +0,0 @@ -import { gzipSync } from 'node:zlib'; - -import { describe, expect, it } from 'vitest'; - -import { STATS_VERSION } from './agentic-shared'; -import type { DbClient } from '../connection.js'; - -import { computeDerivedFromBlob, getDerivedAgenticMetrics } from './derived-agentic-metrics.js'; - -/** Build one aiperf JSONL record for the synthetic fixture. */ -function rec( - conversation_id: string, - turn_index: number, - fields: { isl: number; osl: number; ttft_ms: number; latency_ms: number }, -): string { - return JSON.stringify({ - metadata: { conversation_id, turn_index, benchmark_phase: 'profiling' }, - metrics: { - request_latency: { value: fields.latency_ms, unit: 'ms' }, - time_to_first_token: { value: fields.ttft_ms, unit: 'ms' }, - input_sequence_length: { value: fields.isl, unit: 'tokens' }, - output_sequence_length: { value: fields.osl, unit: 'tokens' }, - }, - }); -} - -describe('computeDerivedFromBlob', () => { - it('returns nulls when no usable records', () => { - const out = computeDerivedFromBlob(''); - expect(out.normalized_session_time_s).toBeNull(); - expect(out.p90_prefill_tps_per_user).toBeNull(); - expect(out.normalized_e2e_400).toBeNull(); - }); - - it('normalizes each request to 400 output tokens before taking percentiles', () => { - const jsonl = [ - // Both requests have TTFT=2s and ITL=20ms, despite very different OSL/E2E. - rec('s1', 0, { isl: 100, osl: 100, ttft_ms: 2000, latency_ms: 3980 }), - rec('s2', 0, { isl: 100, osl: 1000, ttft_ms: 2000, latency_ms: 21_980 }), - ].join('\n'); - - const out = computeDerivedFromBlob(jsonl); - // 2s TTFT + 399 × 20ms ITL = 9.98s for both requests. - expect(out.normalized_e2e_400?.n).toBe(2); - expect(out.normalized_e2e_400?.p75).toBeCloseTo(9.98, 8); - expect(out.normalized_e2e_400?.p90).toBeCloseTo(9.98, 8); - }); - - it('rescales single-session time and computes P90 prefill', () => { - // One session, two turns. load = (100+50) + (200+50) = 400. - // Single session ⇒ mean_load = load_i ⇒ T̃ = T = (1000+2000) ms = 3.0 s. - const jsonl = [ - rec('s1', 0, { isl: 100, osl: 50, ttft_ms: 500, latency_ms: 1000 }), - rec('s1', 1, { isl: 200, osl: 50, ttft_ms: 1000, latency_ms: 2000 }), - ].join('\n'); - const out = computeDerivedFromBlob(jsonl); - expect(out.normalized_session_time_s).toBeCloseTo(3, 6); - // Prefill TPS per turn: 100/0.5=200, 200/1.0=200 → global P90 = 200. - expect(out.p90_prefill_tps_per_user).toBeCloseTo(200, 6); - }); - - it('rescales times across sessions with unequal load', () => { - // s1: 1 turn, load = 100, T = 1s - // s2: 1 turn, load = 300, T = 3s - // mean_load = 200; T̃_1 = 1 * 200/100 = 2; T̃_2 = 3 * 200/300 = 2 - // Mean T̃ = 2.0 - const jsonl = [ - rec('s1', 0, { isl: 90, osl: 10, ttft_ms: 500, latency_ms: 1000 }), - rec('s2', 0, { isl: 270, osl: 30, ttft_ms: 500, latency_ms: 3000 }), - ].join('\n'); - const out = computeDerivedFromBlob(jsonl); - expect(out.normalized_session_time_s).toBeCloseTo(2, 6); - }); - - it('drops records missing required fields and skips non-profiling phase', () => { - const lines = [ - rec('s1', 0, { isl: 100, osl: 50, ttft_ms: 500, latency_ms: 1000 }), - // missing TTFT — should be skipped - JSON.stringify({ - metadata: { conversation_id: 's1', turn_index: 1, benchmark_phase: 'profiling' }, - metrics: { - request_latency: { value: 1000, unit: 'ms' }, - input_sequence_length: { value: 100, unit: 'tokens' }, - output_sequence_length: { value: 50, unit: 'tokens' }, - }, - }), - // warmup phase — should be skipped - JSON.stringify({ - metadata: { conversation_id: 's2', turn_index: 0, benchmark_phase: 'warmup' }, - metrics: { - request_latency: { value: 9999, unit: 'ms' }, - time_to_first_token: { value: 9999, unit: 'ms' }, - input_sequence_length: { value: 100, unit: 'tokens' }, - output_sequence_length: { value: 50, unit: 'tokens' }, - }, - }), - ]; - const out = computeDerivedFromBlob(lines.join('\n')); - expect(out.normalized_session_time_s).toBeCloseTo(1, 6); - expect(out.p90_prefill_tps_per_user).toBeCloseTo(200, 6); - }); - - it('p90 across turns: 10-turn session picks the right rank', () => { - // Prefill rates 100..1000 (per turn isl/ttft); p90 of 10 values (linear) = 910. - const turns = Array.from({ length: 10 }, (_, i) => - rec('s1', i, { - isl: (i + 1) * 100, // 100, 200, ..., 1000 tokens - osl: 10, - ttft_ms: 1000, // 1 second → rates: 100..1000 tps - latency_ms: 1500, - }), - ); - const out = computeDerivedFromBlob(turns.join('\n')); - expect(out.p90_prefill_tps_per_user).toBeCloseTo(910, 6); - }); - - it('excludes osl=0 (cancelled/empty-output) turns from normalized E2E', () => { - // Two normal turns + one cancelled turn (osl=0, latency=30s, ttft=1s). - // - // The cancelled turn must be excluded because observedDecodeIntervals collapses - // to max(0-1,1)=1, making itlMs=(30000-1000)/1=29000ms and normalizedMs explode - // to ~11 572 s — roughly 386× the real scale. (Pre-fix behavior for reference; - // this number is intentionally not asserted below to avoid enshrining the bug.) - // - // Normal turn A: isl=100, osl=50, ttft=500ms, latency=1000ms - // observedDecodeIntervals = max(49,1) = 49 - // itlMs = (1000-500)/49 - // normalizedMs = 500 + 399*(500/49) - // - // Normal turn B: isl=200, osl=100, ttft=1000ms, latency=3000ms - // observedDecodeIntervals = max(99,1) = 99 - // itlMs = (3000-1000)/99 - // normalizedMs = 1000 + 399*(2000/99) - const normA = (500 + (399 * 500) / 49) / 1000; // seconds - const normB = (1000 + (399 * 2000) / 99) / 1000; // seconds - - const jsonl = [ - rec('s1', 0, { isl: 100, osl: 50, ttft_ms: 500, latency_ms: 1000 }), - rec('s1', 1, { isl: 200, osl: 100, ttft_ms: 1000, latency_ms: 3000 }), - // Cancelled / empty-output turn — osl=0 must be rejected by extractTurn. - rec('s2', 0, { isl: 150, osl: 0, ttft_ms: 1000, latency_ms: 30000 }), - ].join('\n'); - - const out = computeDerivedFromBlob(jsonl); - - // Only the 2 normal turns contribute; osl=0 record is silently excluded. - expect(out.normalized_e2e_400?.n).toBe(2); - - // p90 of [normA, normB] sorted ascending (normA < normB): - // pos = 1*0.9 = 0.9; result = normA + (normB - normA)*0.9 - const expectedP90 = normA + (normB - normA) * 0.9; - expect(out.normalized_e2e_400?.p90).toBeCloseTo(expectedP90, 6); - - // Sanity: p90 should be single-digit seconds, not thousands. - expect(out.normalized_e2e_400!.p90).toBeLessThan(20); - }); -}); - -/** Capture SQL template text + bound values for the write-back assertions. */ -function mockSql(queue: unknown[][]): { - sql: DbClient; - calls: { text: string; values: unknown[] }[]; -} { - const responses = [...queue]; - const calls: { text: string; values: unknown[] }[] = []; - const sql = ((strings: TemplateStringsArray, ...values: unknown[]) => { - calls.push({ text: strings.join('?'), values }); - return Promise.resolve(responses.shift() ?? []); - }) as unknown as DbClient; - return { sql, calls }; -} - -describe('getDerivedAgenticMetrics write-back', () => { - it('self-heals aggregate_stats from the profile blob, carrying server fields forward', async () => { - const jsonl = [ - rec('s1', 0, { isl: 100, osl: 50, ttft_ms: 500, latency_ms: 1000 }), - rec('s1', 1, { isl: 200, osl: 50, ttft_ms: 1000, latency_ms: 2000 }), - ].join('\n'); - const blob = gzipSync(Buffer.from(jsonl)); - - // Stale v(N-1) row that DOES carry server-derived fields — they must be - // preserved in the healed bundle (derived route can't recompute them). - const staleServerKv = { mean: 0.4, p50: 0.4, p75: 0.5, p90: 0.6, p99: 0.7, n: 3 }; - const staleStats = { - version: STATS_VERSION - 1, - isl: null, - osl: null, - kvCacheUtil: staleServerKv, - prefixCacheHitRate: null, - normalizedSessionTimeS: 999, - p90PrefillTpsPerUser: 999, - normalizedE2e400: null, - }; - - const { sql, calls } = mockSql([ - // fetchAggregateStatsRows - [{ benchmark_result_id: 7, stats: staleStats }], - // fallback profile-blob query - [{ benchmark_result_id: 7, trace_replay_id: 870, blob }], - ]); - - const result = await getDerivedAgenticMetrics(sql, [7]); - - // Response is the freshly recomputed value, not the stale 999s. - expect(result[7]?.normalized_session_time_s).toBeCloseTo(3, 6); - expect(result[7]?.p90_prefill_tps_per_user).toBeCloseTo(200, 6); - - // 3 calls: stats read, blob read, write-back UPDATE. - expect(calls).toHaveLength(3); - expect(calls[2]!.text).toContain('update agentic_trace_replay set aggregate_stats'); - expect(calls[2]!.text).toContain('::jsonb where id'); - - // The write-back binds a COMPLETE, version-stamped bundle at the new version, - // recomputing profile fields and carrying server fields forward untouched. - // The payload OBJECT is bound directly (not stringified — that would - // double-encode into a JSONB string). - interface WrittenStats { - version: number; - isl: unknown; - osl: unknown; - kvCacheUtil: unknown; - normalizedSessionTimeS: number | null; - p90PrefillTpsPerUser: number | null; - } - const [written, traceReplayId] = calls[2]!.values as [WrittenStats, number]; - expect(traceReplayId).toBe(870); - expect(written.version).toBe(STATS_VERSION); - expect(written.normalizedSessionTimeS).toBeCloseTo(3, 6); - expect(written.p90PrefillTpsPerUser).toBeCloseTo(200, 6); - expect(written.isl).not.toBeNull(); - expect(written.osl).not.toBeNull(); - // Server-derived field carried forward from the stale row (not re-read). - expect(written.kvCacheUtil).toEqual(staleServerKv); - }); - - it('takes the fast path (no blob read, no write-back) when stats are current', async () => { - const currentStats = { - version: STATS_VERSION, - isl: null, - osl: null, - kvCacheUtil: null, - prefixCacheHitRate: null, - normalizedSessionTimeS: 1.5, - p90PrefillTpsPerUser: 42, - normalizedE2e400: { mean: 1, p50: 1, p75: 1, p90: 2, p99: 3, n: 5 }, - }; - const { sql, calls } = mockSql([[{ benchmark_result_id: 7, stats: currentStats }]]); - - const result = await getDerivedAgenticMetrics(sql, [7]); - - expect(result[7]?.normalized_session_time_s).toBe(1.5); - expect(result[7]?.p90_normalized_e2e_400_s).toBe(2); - // Only the stats read — no fallback blob query, no write-back. - expect(calls).toHaveLength(1); - }); -}); diff --git a/packages/db/src/queries/derived-agentic-metrics.ts b/packages/db/src/queries/derived-agentic-metrics.ts deleted file mode 100644 index 626ab9c77..000000000 --- a/packages/db/src/queries/derived-agentic-metrics.ts +++ /dev/null @@ -1,318 +0,0 @@ -/** - * Live-computed per-point metrics derived from the stored aiperf - * `profile_export.jsonl` blob. These aren't precomputed in the metrics JSONB - * because they require grouping by `conversation_id` and aggregating per - * session — work that's cheap once per agentic point but adds up to be - * meaningful only when actually plotted. - * - * - normalized_session_time_s: per the "Mean Normalized Session Time" proposal - * (https://gist.github.com/xinli-sw/115d370c17f6d1b977878b68530981fa). Sum of - * per-turn `request_latency` per session (inter-turn tool/thinking gaps are - * inherently excluded since we only sum the active GPU time, not wallclock). - * Each session's time is rescaled by `mean_load / session_load`, where load - * is Σ(ISL+OSL) across turns. The plotted value is the mean across sessions. - * - * - p90_prefill_tps_per_user: per the same gist's "Prefill" Pareto chart. - * Per turn: prefill_tps = ISL / TTFT_seconds. Single P90 across every turn - * in every session — the per-session percentile + cross-session mean - * sandwich was discarded because it just dampens tail behavior. - */ - -import { gunzipSync } from 'node:zlib'; - -import { NORMALIZED_E2E_OUTPUT_TOKENS } from '@semianalysisai/inferencex-constants'; - -import type { DbClient } from '../connection.js'; -import { - extractIslOsl, - fetchAggregateStatsRows, - meanOf, - percentilesOf, - quantile, - readNum, - STATS_VERSION, - writeBackTraceReplayJsonb, - type MetricPercentiles, -} from './agentic-shared'; - -export interface DerivedAgenticMetric { - /** benchmark_results.id this entry belongs to. */ - id: number; - /** Mean normalized session time in seconds. */ - normalized_session_time_s: number | null; - /** P90 of per-turn prefill tps/user (ISL / TTFT) across every turn in every session. */ - p90_prefill_tps_per_user: number | null; - /** P75 normalized per-request E2E at a fixed 400-token output length. */ - p75_normalized_e2e_400_s: number | null; - /** P90 normalized per-request E2E at a fixed 400-token output length. */ - p90_normalized_e2e_400_s: number | null; -} - -export type DerivedAgenticMetricMap = Record; - -/** - * The full `aggregate_stats` JSONB shape (mirrors `AggregateStats` in - * etl/compute-aggregate-stats.ts). Duplicated here rather than imported to keep - * this module off the etl import graph. When we self-heal from the profile blob - * alone, the server-derived fields (kvCacheUtil, prefixCacheHitRate) are carried - * forward untouched from the stale row — never re-reading the huge server blob. - * This mirrors the profile-only upgrade `backfill-aggregate-stats.ts` performs; - * the agentic-aggregates route (which does read the server blob) heals those - * server fields. - */ -interface StoredAggregateStats { - version: number; - isl: MetricPercentiles | null; - osl: MetricPercentiles | null; - kvCacheUtil: MetricPercentiles | null; - prefixCacheHitRate: MetricPercentiles | null; - normalizedSessionTimeS: number | null; - p90PrefillTpsPerUser: number | null; - normalizedE2e400: MetricPercentiles | null; -} - -/** - * JSONL blobs can be ~1-2 MB compressed (~5-10 MB raw) and Neon's serverless - * HTTP driver caps responses at 64 MB — chunk to stay well under. - */ -const QUERY_CHUNK_SIZE = 6; - -interface RecordMetrics { - request_latency?: { value?: number; unit?: string } | number; - time_to_first_token?: { value?: number; unit?: string } | number; - input_sequence_length?: { value?: number } | number; - output_sequence_length?: { value?: number } | number; -} - -interface RecordMetadata { - conversation_id?: string; - turn_index?: number; - benchmark_phase?: string; -} - -interface ProfileRecord { - metadata?: RecordMetadata; - metrics?: RecordMetrics; -} - -interface TurnFields { - request_latency_ms: number; - ttft_ms: number; - isl: number; - osl: number; -} - -function extractTurn(rec: ProfileRecord): TurnFields | null { - const m = rec.metrics ?? {}; - const rl = readNum(m.request_latency); - const tt = readNum(m.time_to_first_token); - const isl = readNum(m.input_sequence_length); - const osl = readNum(m.output_sequence_length); - if (rl === undefined || tt === undefined || isl === undefined || osl === undefined) return null; - if (rl <= 0 || tt <= 0 || isl <= 0 || osl <= 0) return null; - return { request_latency_ms: rl, ttft_ms: tt, isl, osl }; -} - -/** - * Parse one point's JSONL and return the two derived metrics. Returns - * `{ session_time: null, prefill: null }` if the blob has no usable records. - */ -export function computeDerivedFromBlob(jsonl: string): { - normalized_session_time_s: number | null; - p90_prefill_tps_per_user: number | null; - normalized_e2e_400: MetricPercentiles | null; -} { - // Group records by conversation_id, filter to the profiling phase. - const bySession = new Map(); - for (const line of jsonl.split('\n')) { - if (!line) continue; - let rec: ProfileRecord; - try { - rec = JSON.parse(line) as ProfileRecord; - } catch { - continue; - } - if (rec.metadata?.benchmark_phase && rec.metadata.benchmark_phase !== 'profiling') continue; - const sid = rec.metadata?.conversation_id; - if (!sid) continue; - const turn = extractTurn(rec); - if (!turn) continue; - let list = bySession.get(sid); - if (!list) { - list = []; - bySession.set(sid, list); - } - list.push(turn); - } - if (bySession.size === 0) { - return { - normalized_session_time_s: null, - p90_prefill_tps_per_user: null, - normalized_e2e_400: null, - }; - } - - // Per-session aggregates for session time; per-turn prefill rates pool into - // a single global array so the percentile sees the full distribution. - const sessionTimesS: number[] = []; - const sessionLoads: number[] = []; - const allPrefillRates: number[] = []; - const allNormalizedE2eS: number[] = []; - for (const turns of bySession.values()) { - let timeMs = 0; - let load = 0; - for (const t of turns) { - timeMs += t.request_latency_ms; - load += t.isl + t.osl; - const ttftSec = t.ttft_ms / 1000; - if (ttftSec > 0) allPrefillRates.push(t.isl / ttftSec); - - // Keep the observed TTFT, then project the request's mean decode - // interval to a fixed output length. Do this per request before taking - // percentiles so long original outputs do not dominate the tail. - const observedDecodeIntervals = Math.max(t.osl - 1, 1); - const itlMs = (t.request_latency_ms - t.ttft_ms) / observedDecodeIntervals; - const normalizedMs = t.ttft_ms + (NORMALIZED_E2E_OUTPUT_TOKENS - 1) * itlMs; - if ( - Number.isFinite(itlMs) && - itlMs >= 0 && - Number.isFinite(normalizedMs) && - normalizedMs > 0 - ) { - allNormalizedE2eS.push(normalizedMs / 1000); - } - } - if (load > 0) { - sessionTimesS.push(timeMs / 1000); - sessionLoads.push(load); - } - } - - // Normalized session time: T̃_i = T_i × (mean_load / load_i), then mean. - let normalized: number | null = null; - if (sessionTimesS.length > 0) { - const meanLoad = meanOf(sessionLoads); - if (meanLoad > 0) { - const scaled: number[] = []; - for (let i = 0; i < sessionTimesS.length; i++) { - const ti = sessionTimesS[i]!; - const li = sessionLoads[i]!; - if (li > 0) scaled.push(ti * (meanLoad / li)); - } - normalized = scaled.length > 0 ? meanOf(scaled) : null; - } - } - - let prefill: number | null = null; - if (allPrefillRates.length > 0) { - allPrefillRates.sort((a, b) => a - b); - prefill = quantile(allPrefillRates, 0.9); - } - - return { - normalized_session_time_s: normalized, - p90_prefill_tps_per_user: prefill, - normalized_e2e_400: percentilesOf(allNormalizedE2eS), - }; -} - -export async function getDerivedAgenticMetrics( - sql: DbClient, - benchmarkResultIds: number[], -): Promise { - if (benchmarkResultIds.length === 0) return {}; - - const result: DerivedAgenticMetricMap = {}; - - // Fast path: read the pre-computed values out of `aggregate_stats`. The - // ingest pipeline computes both metrics in the same pass that produces the - // percentile bundles, so a single SQL round-trip covers most ids without - // touching the gzipped profile blob. - const statsRows = await fetchAggregateStatsRows(sql, benchmarkResultIds); - - const idsNeedingBlob: number[] = []; - // Carry each stale/missing row's existing stats into the fallback so a - // self-heal preserves the server-derived fields (kvCacheUtil, - // prefixCacheHitRate) it can't recompute from the profile blob alone. - const staleStatsById = new Map(); - for (const row of statsRows) { - const id = Number(row.benchmark_result_id); - if (row.stats && Number(row.stats.version) === STATS_VERSION) { - result[id] = { - id, - normalized_session_time_s: row.stats.normalizedSessionTimeS ?? null, - p90_prefill_tps_per_user: row.stats.p90PrefillTpsPerUser ?? null, - p75_normalized_e2e_400_s: row.stats.normalizedE2e400?.p75 ?? null, - p90_normalized_e2e_400_s: row.stats.normalizedE2e400?.p90 ?? null, - }; - } else { - idsNeedingBlob.push(id); - staleStatsById.set(id, row.stats ?? null); - } - } - - if (idsNeedingBlob.length === 0) return result; - - // Fallback: parse the profile blob directly. Used for rows whose - // `aggregate_stats` is null or computed by an older STATS_VERSION; the - // backfill script drains the population so this path should be rare. - // `trace_replay_id` + the (small) stale `aggregate_stats` come along on the - // same join — no extra round-trip — so we can self-heal after recompute. - const rows: { - benchmark_result_id: number; - trace_replay_id: number; - blob: Buffer; - }[] = []; - for (let i = 0; i < idsNeedingBlob.length; i += QUERY_CHUNK_SIZE) { - const chunk = idsNeedingBlob.slice(i, i + QUERY_CHUNK_SIZE); - const chunkRows = (await sql` - select - br.id as benchmark_result_id, - atr.id as trace_replay_id, - atr.profile_export_jsonl_gz as blob - from benchmark_results br - join agentic_trace_replay atr on atr.id = br.trace_replay_id - where br.id = any(${chunk}::bigint[]) - and atr.profile_export_jsonl_gz is not null - `) as { benchmark_result_id: number; trace_replay_id: number; blob: Buffer }[]; - rows.push(...chunkRows); - } - - for (const row of rows) { - const id = Number(row.benchmark_result_id); - try { - const jsonl = gunzipSync(row.blob).toString('utf8'); - const { normalized_session_time_s, p90_prefill_tps_per_user, normalized_e2e_400 } = - computeDerivedFromBlob(jsonl); - result[id] = { - id, - normalized_session_time_s, - p90_prefill_tps_per_user, - p75_normalized_e2e_400_s: normalized_e2e_400?.p75 ?? null, - p90_normalized_e2e_400_s: normalized_e2e_400?.p90 ?? null, - }; - - // Self-heal the shared `aggregate_stats` bundle. We only have the profile - // blob here, so recompute the profile-derived fields (isl/osl + the three - // derived metrics) and carry the stale row's server-derived fields - // forward untouched — the profile-only upgrade the backfill CLI also - // performs. Fire-and-forget, best-effort (no-ops on a read-only replica). - const { isl, osl } = extractIslOsl(jsonl); - const prior = staleStatsById.get(id) ?? null; - const merged: StoredAggregateStats = { - version: STATS_VERSION, - isl: percentilesOf(isl), - osl: percentilesOf(osl), - kvCacheUtil: prior?.kvCacheUtil ?? null, - prefixCacheHitRate: prior?.prefixCacheHitRate ?? null, - normalizedSessionTimeS: normalized_session_time_s, - p90PrefillTpsPerUser: p90_prefill_tps_per_user, - normalizedE2e400: normalized_e2e_400, - }; - writeBackTraceReplayJsonb(sql, 'aggregate_stats', Number(row.trace_replay_id), merged); - } catch { - // Skip malformed blobs silently — frontend treats missing ids as "no data". - } - } - return result; -} From a65a0989af7a7b3e7fc681acb8f39ee5a964a037 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Wed, 29 Jul 2026 16:58:08 -0500 Subject: [PATCH 2/7] feat(agentic): add OSL / E2EL x-axis metric as the agentic default MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduces "OSL / E2EL" ("e2e interactivity") for agentic scenarios: the per-request output-token rate INCLUDING the prefill wait, OSL / (TTFT + generation time) ≈ 1 / (ITL + TTFT/OSL). Unlike plain interactivity it cannot be inflated by delaying prefill. - aggregate_stats v7: store `e2elPerOsl`, percentiles of the per-request E2EL/OSL ratio (seconds per output token). The read path inverts, so pXX OSL/E2EL = 1 / pXX(E2EL/OSL) — the slow-tail convention the ETL already enforces for `*_intvty` - API: /api/v1/derived-agentic-metrics returns p75/p90_osl_per_e2el - chart: new mode, agentic default, listed first; fixed-seq is unchanged and never shows the button - overlays: suppressed in this mode (unofficial rows carry no persisted per-request trace) with a caption disclaimer - shared cypress helper interceptDerivedAgenticMetrics stubs the fetch the default mode fires on mount; overlay specs switch to Interactivity explicitly before asserting overlay points 中文：为智能体场景新增 "OSL / E2EL"（e2e interactivity）指标：每请求 输出 token 速率（含 prefill 等待），即 OSL /（TTFT + 生成时间）。 aggregate_stats 升级至 v7，存储每请求 E2EL/OSL 比值的分位数，读取时取 倒数以沿用 `*_intvty` 的慢尾约定；新增 API 字段与图表模式，并设为智能体 默认模式且置于最左侧；该模式下隐藏非官方运行覆盖层并显示提示文案。 Co-Authored-By: Claude Fable 5 --- .../e2e/gpu-compare-agentic-detail.cy.ts | 6 +- .../cypress/e2e/overlay-legend-remove.cy.ts | 7 +- .../cypress/e2e/overlay-optimal-only.cy.ts | 7 +- .../app/cypress/e2e/ttft-x-axis-toggle.cy.ts | 61 +- packages/app/cypress/support/e2e.ts | 27 + .../src/app/api/v1/agentic-cache-keys.test.ts | 6 + .../api/v1/derived-agentic-metrics/route.ts | 48 ++ .../components/inference/InferenceContext.tsx | 49 +- .../inference/hooks/useChartData.ts | 26 +- .../app/src/components/inference/types.ts | 8 +- .../components/inference/ui/ChartDisplay.tsx | 557 +++++++++++------- .../src/components/inference/utils.test.ts | 21 +- .../app/src/components/inference/utils.ts | 15 + .../components/inference/utils/e2eFrontier.ts | 2 +- .../api/use-derived-agentic-metrics.test.ts | 13 + .../hooks/api/use-derived-agentic-metrics.ts | 50 ++ packages/db/src/backfill-aggregate-stats.ts | 6 +- .../src/etl/compute-aggregate-stats.test.ts | 19 +- .../db/src/etl/compute-aggregate-stats.ts | 17 +- .../db/src/queries/agentic-aggregates.test.ts | 9 +- packages/db/src/queries/agentic-aggregates.ts | 11 +- packages/db/src/queries/agentic-shared.ts | 9 +- .../queries/derived-agentic-metrics.test.ts | 237 ++++++++ .../db/src/queries/derived-agentic-metrics.ts | 233 ++++++++ 24 files changed, 1186 insertions(+), 258 deletions(-) create mode 100644 packages/app/src/app/api/v1/derived-agentic-metrics/route.ts create mode 100644 packages/app/src/hooks/api/use-derived-agentic-metrics.test.ts create mode 100644 packages/app/src/hooks/api/use-derived-agentic-metrics.ts create mode 100644 packages/db/src/queries/derived-agentic-metrics.test.ts create mode 100644 packages/db/src/queries/derived-agentic-metrics.ts diff --git a/packages/app/cypress/e2e/gpu-compare-agentic-detail.cy.ts b/packages/app/cypress/e2e/gpu-compare-agentic-detail.cy.ts index a5fd8f12f..b95af1693 100644 --- a/packages/app/cypress/e2e/gpu-compare-agentic-detail.cy.ts +++ b/packages/app/cypress/e2e/gpu-compare-agentic-detail.cy.ts @@ -1,4 +1,4 @@ -import { unlockAgenticGate } from '../support/e2e'; +import { interceptDerivedAgenticMetrics, unlockAgenticGate } from '../support/e2e'; // --------------------------------------------------------------------------- // Spec-scoped fixture helpers @@ -147,6 +147,9 @@ describe('GPU comparison agentic point detail', () => { ); request.reply({ body: result }); }); + // The agentic default x-axis mode (OSL / E2EL) fetches derived metrics on + // mount; without values every point drops out of the (remapped) data set. + interceptDerivedAgenticMetrics(); cy.visit('/inference?g_model=DeepSeek-V4-Pro&i_seq=agentic-traces&i_prec=fp4', { onBeforeLoad(win) { @@ -206,6 +209,7 @@ describe('GPU comparison agentic point detail', () => { 'agenticAvailability', ); cy.intercept('GET', '/api/v1/benchmarks*', { body: agenticBenchmarks }).as('agenticBenchmarks'); + interceptDerivedAgenticMetrics(); cy.visit( '/inference?g_model=DeepSeek-V4-Pro&i_seq=agentic-traces&i_prec=fp4&i_gpus=b200_sglang,b200_vllm&i_dates=2026-06-12&i_dstart=2026-06-12&i_dend=2026-06-12', diff --git a/packages/app/cypress/e2e/overlay-legend-remove.cy.ts b/packages/app/cypress/e2e/overlay-legend-remove.cy.ts index d0bc6e126..b8cef8c33 100644 --- a/packages/app/cypress/e2e/overlay-legend-remove.cy.ts +++ b/packages/app/cypress/e2e/overlay-legend-remove.cy.ts @@ -9,7 +9,7 @@ * appeared to do nothing. The legend toggle already had the overlay-aware * split (`unifiedToggle`); the X now shares it (`handleRemoveHwType`). */ -import { unlockAgenticGate } from '../support/e2e'; +import { interceptDerivedAgenticMetrics, unlockAgenticGate } from '../support/e2e'; import { countVisible, interceptOverlayRun, @@ -20,6 +20,10 @@ import { describe('Official legend X works while an unofficial overlay is loaded', () => { before(() => { interceptOverlayRun(); + // The agentic default mode is OSL / E2EL (which suppresses overlays and + // fetches derived metrics) — stub the fetch, then switch to Interactivity + // where the overlay renders. + interceptDerivedAgenticMetrics(); cy.visit(`/inference?unofficialrun=${OVERLAY_RUN_ID}&i_seq=agentic-traces&i_pctl=p90`, { onBeforeLoad(win) { win.localStorage.setItem('inferencex-star-modal-dismissed', String(Date.now())); @@ -27,6 +31,7 @@ describe('Official legend X works while an unofficial overlay is loaded', () => }, }); cy.wait('@unofficialRun'); + cy.get('[data-testid="x-axis-mode-interactivity"]').click(); cy.get('[data-testid="chart-figure"]').should('have.length.at.least', 1); cy.get('[data-testid="inference-chart-display"] svg .unofficial-overlay-pt').should( 'have.length', diff --git a/packages/app/cypress/e2e/overlay-optimal-only.cy.ts b/packages/app/cypress/e2e/overlay-optimal-only.cy.ts index c7c7a88a3..9d4f42930 100644 --- a/packages/app/cypress/e2e/overlay-optimal-only.cy.ts +++ b/packages/app/cypress/e2e/overlay-optimal-only.cy.ts @@ -10,7 +10,7 @@ * dashed roofline (the monotone spline between C=8 and C=2 passes within * ~0.5% of it) while the official twin was hidden. */ -import { unlockAgenticGate } from '../support/e2e'; +import { interceptDerivedAgenticMetrics, unlockAgenticGate } from '../support/e2e'; import { countVisible, interceptOverlayRun, @@ -21,6 +21,10 @@ import { describe('Overlay points respect Optimal Only (agentic interactivity)', () => { before(() => { interceptOverlayRun(); + // The agentic default mode is OSL / E2EL (which suppresses overlays and + // fetches derived metrics) — stub the fetch, then switch to the + // Interactivity mode this suite is about. + interceptDerivedAgenticMetrics(); cy.visit(`/inference?unofficialrun=${OVERLAY_RUN_ID}&i_seq=agentic-traces&i_pctl=p90`, { onBeforeLoad(win) { win.localStorage.setItem('inferencex-star-modal-dismissed', String(Date.now())); @@ -28,6 +32,7 @@ describe('Overlay points respect Optimal Only (agentic interactivity)', () => { }, }); cy.wait('@unofficialRun'); + cy.get('[data-testid="x-axis-mode-interactivity"]').click(); cy.get('[data-testid="chart-figure"]').should('have.length.at.least', 1); cy.get('[data-testid="x-axis-mode-interactivity"]').should( 'have.attr', diff --git a/packages/app/cypress/e2e/ttft-x-axis-toggle.cy.ts b/packages/app/cypress/e2e/ttft-x-axis-toggle.cy.ts index f3a897b1b..a269c4f58 100644 --- a/packages/app/cypress/e2e/ttft-x-axis-toggle.cy.ts +++ b/packages/app/cypress/e2e/ttft-x-axis-toggle.cy.ts @@ -1,4 +1,4 @@ -import { unlockAgenticGate } from '../support/e2e'; +import { interceptDerivedAgenticMetrics, unlockAgenticGate } from '../support/e2e'; // This spec exercises the agentic x-axis modes, which only exist when the // selected model resolves to the Agentic Traces scenario. The default e2e @@ -126,6 +126,9 @@ const interceptFixedSequenceData = () => { describe('X-Axis Mode Toggle (inference chart)', () => { before(() => { interceptAgenticData(); + // The agentic default mode is OSL / E2EL, which fetches derived metrics + // on first render — stub them before the visit. + interceptDerivedAgenticMetrics(); cy.visit('/inference?i_seq=agentic-traces', { onBeforeLoad(win) { win.localStorage.setItem('inferencex-star-modal-dismissed', String(Date.now())); @@ -136,13 +139,32 @@ describe('X-Axis Mode Toggle (inference chart)', () => { cy.get('[data-testid="chart-figure"]').should('have.length.at.least', 1); }); - it('shows Interactivity by default for the agentic view', () => { + it('shows OSL / E2EL by default for the agentic view, as the leftmost option', () => { cy.get('[data-testid="scenario-selector"]').should('contain.text', 'Agentic Traces'); cy.get('[data-testid="x-axis-mode-ttft"]').should('be.visible'); cy.get('[data-testid="x-axis-mode-e2e"]').should('be.visible'); - cy.get('[data-testid="x-axis-mode-interactivity"]') + cy.get('[data-testid="x-axis-mode-interactivity"]').should('be.visible'); + cy.get('[data-testid="x-axis-mode-osl-e2el"]') .should('be.visible') .and('have.attr', 'aria-selected', 'true'); + // OSL / E2EL leads the mode list for agentic. + cy.get('[data-testid="x-axis-mode-buttons"] [role="tab"]') + .first() + .should('have.attr', 'data-testid', 'x-axis-mode-osl-e2el'); + cy.get('[data-testid="chart-figure"] h2').should('contain.text', 'P90 OSL / E2EL'); + cy.get('[data-testid="chart-figure"] svg').should( + 'contain.text', + 'P90 OSL / E2EL (tok/s/user)', + ); + }); + + it('switches to Interactivity and updates the heading', () => { + cy.get('[data-testid="x-axis-mode-interactivity"]').click(); + cy.get('[data-testid="x-axis-mode-interactivity"]').should( + 'have.attr', + 'aria-selected', + 'true', + ); cy.get('[data-testid="chart-figure"] h2').should('contain.text', 'Interactivity'); }); @@ -203,6 +225,9 @@ describe('X-Axis Mode Toggle (inference chart)', () => { it('honors explicit label URL overrides for the agentic view', () => { interceptAgenticData(); + // Fresh page load → fresh React Query cache → the default OSL / E2EL + // mode refetches derived metrics. + interceptDerivedAgenticMetrics(); cy.visit('/inference?i_seq=agentic-traces&i_label=0&i_advlabel=0&i_linelabel=1', { onBeforeLoad(win) { win.localStorage.setItem('inferencex-star-modal-dismissed', String(Date.now())); @@ -228,6 +253,23 @@ describe('X-Axis Mode Toggle (inference chart)', () => { cy.get('[data-testid="chart-figure"] svg').should('contain.text', 'P90 End-to-end Latency (s)'); }); + it('switches back to request-level OSL / E2EL', () => { + // No cy.wait here: the derived metrics were fetched (and stubbed) on the + // initial default-mode load and are still fresh in the React Query cache + // (staleTime 5 min), so re-entering the mode fires no new request. + cy.get('[data-testid="x-axis-mode-osl-e2el"]').click(); + cy.get('[data-testid="x-axis-mode-osl-e2el"]').should('have.attr', 'aria-selected', 'true'); + cy.get('[data-testid="chart-figure"] h2').should('contain.text', 'P90 OSL / E2EL'); + cy.get('[data-testid="chart-figure"] svg').should( + 'contain.text', + 'P90 OSL / E2EL (tok/s/user)', + ); + + cy.get('[data-testid="percentile-selector"]').click(); + cy.contains('[role="option"]', 'p75').click(); + cy.get('[data-testid="chart-figure"] h2').should('contain.text', 'P75 OSL / E2EL'); + }); + it('switches back to Interactivity', () => { cy.get('[data-testid="x-axis-mode-interactivity"]').click(); cy.get('[data-testid="x-axis-mode-interactivity"]').should( @@ -358,6 +400,8 @@ const interceptAgenticDataWithOverlay = () => { describe('X-Axis Mode Toggle — overlay path (finding #8 regression guard)', () => { before(() => { interceptAgenticDataWithOverlay(); + // Default agentic mode is OSL / E2EL → derived metrics fetch on mount. + interceptDerivedAgenticMetrics(); cy.visit(`/inference?unofficialrun=${OVERLAY_RUN_ID}&i_seq=agentic-traces`, { onBeforeLoad(win) { win.localStorage.setItem('inferencex-star-modal-dismissed', String(Date.now())); @@ -406,4 +450,15 @@ describe('X-Axis Mode Toggle — overlay path (finding #8 regression guard)', () expect(total).to.be.greaterThan(0); }); }); + + it('osl-e2el mode shows suppression banner for unofficial-run overlays', () => { + // Derived metrics are cached from the initial default-mode load. + cy.get('[data-testid="x-axis-mode-osl-e2el"]').click(); + cy.get('[data-testid="x-axis-mode-osl-e2el"]').should('have.attr', 'aria-selected', 'true'); + // The suppression message appears because isUnofficialRun is true and the + // mode is 'osl-e2el' (documented in ChartDisplay.tsx). + cy.contains( + 'OSL / E2EL requires persisted per-request traces, so unofficial-run overlays are unavailable for this experimental view.', + ).should('be.visible'); + }); }); diff --git a/packages/app/cypress/support/e2e.ts b/packages/app/cypress/support/e2e.ts index 8d03599e7..3ee554b30 100644 --- a/packages/app/cypress/support/e2e.ts +++ b/packages/app/cypress/support/e2e.ts @@ -37,3 +37,30 @@ export function unlockAgenticGate(win: Window): void { // agentic surfaces are public regardless. } } + +/** + * Stub `/api/v1/derived-agentic-metrics` with deterministic per-id values. + * + * OSL / E2EL is the DEFAULT x-axis mode for agentic scenarios, so any spec + * that loads the agentic view fires this fetch on mount — without a stub the + * fixture server has no DB and the chart sits on its loading skeleton until + * the query errors out. Values are index-stable so axis positions are + * deterministic. Register BEFORE `cy.visit`. + */ +export function interceptDerivedAgenticMetrics(): void { + cy.intercept('GET', '/api/v1/derived-agentic-metrics*', (request) => { + const ids = new URL(request.url).searchParams.get('ids')?.split(',').filter(Boolean) ?? []; + request.reply({ + body: Object.fromEntries( + ids.map((id, index) => [ + id, + { + id: Number(id), + p75_osl_per_e2el: 40 + index, + p90_osl_per_e2el: 25 + index, + }, + ]), + ), + }); + }).as('derivedAgenticMetrics'); +} diff --git a/packages/app/src/app/api/v1/agentic-cache-keys.test.ts b/packages/app/src/app/api/v1/agentic-cache-keys.test.ts index 7177f9960..3a6287cf6 100644 --- a/packages/app/src/app/api/v1/agentic-cache-keys.test.ts +++ b/packages/app/src/app/api/v1/agentic-cache-keys.test.ts @@ -28,12 +28,17 @@ import { STATS_VERSION } from '@semianalysisai/inferencex-db/queries/agentic-agg import { REQUEST_TIMELINE_VERSION } from '@semianalysisai/inferencex-db/etl/compute-request-timeline'; import { TRACE_SERVER_METRICS_VERSION } from '@semianalysisai/inferencex-db/queries/trace-server-metrics'; +import { CACHE_KEY_PREFIX as derivedAgenticMetricsKey } from './derived-agentic-metrics/route'; import { CACHE_KEY_PREFIX as agenticAggregatesKey } from './agentic-aggregates/route'; import { CACHE_KEY_PREFIX as requestTimelineKey } from './request-timeline/route'; import { CACHE_KEY_PREFIX as traceServerMetricsKey } from './trace-server-metrics/route'; import { CACHE_KEY_PREFIX as traceHistogramsKey } from './trace-histograms/route'; describe('agentic blob-cache keys are version-derived', () => { + it('derived-agentic-metrics key embeds STATS_VERSION', () => { + expect(derivedAgenticMetricsKey).toBe(`derived-agentic-metrics-v${STATS_VERSION}`); + }); + it('agentic-aggregates key embeds STATS_VERSION', () => { expect(agenticAggregatesKey).toBe(`agentic-aggregates-v${STATS_VERSION}`); }); @@ -52,6 +57,7 @@ describe('agentic blob-cache keys are version-derived', () => { it('every key actually contains a version segment (no unversioned literals)', () => { for (const key of [ + derivedAgenticMetricsKey, agenticAggregatesKey, requestTimelineKey, traceServerMetricsKey, diff --git a/packages/app/src/app/api/v1/derived-agentic-metrics/route.ts b/packages/app/src/app/api/v1/derived-agentic-metrics/route.ts new file mode 100644 index 000000000..541fcfc69 --- /dev/null +++ b/packages/app/src/app/api/v1/derived-agentic-metrics/route.ts @@ -0,0 +1,48 @@ +import { STATS_VERSION } from '@semianalysisai/inferencex-db/queries/agentic-aggregates'; +import { getDb } from '@semianalysisai/inferencex-db/connection'; + +import { + getDerivedAgenticMetrics, + type DerivedAgenticMetricMap, +} from '@semianalysisai/inferencex-db/queries/derived-agentic-metrics'; + +import { cachedQuery } from '@/lib/api-cache'; + +import { idsQueryRoute } from '../id-routes'; + +export const dynamic = 'force-dynamic'; + +// blobOnly: the response is one entry per id with two numbers, but the +// derivation work parses thousands of JSONL records per blob — cache the +// computed result so a chart-refresh hits the warm path. +// +// The cache key is derived from STATS_VERSION (the payload governs the derived +// metrics read out of `aggregate_stats`). blobSet is write-once and nothing +// purges post-backfill, so a hand-written version string would serve stale +// data forever after a bump — deriving the key from the constant means a +// STATS_VERSION bump automatically rolls the cache namespace. +/** Version-derived blob-cache key namespace (exported for the key-derivation test). */ +export const CACHE_KEY_PREFIX = `derived-agentic-metrics-v${STATS_VERSION}`; + +const getCachedDerivedAgenticMetrics = cachedQuery( + (ids: number[]): Promise => getDerivedAgenticMetrics(getDb(), ids), + CACHE_KEY_PREFIX, + { blobOnly: true }, +); + +/** + * GET /api/v1/derived-agentic-metrics?ids=1,2,3 + * + * Returns per-id derived metrics computed live from the stored aiperf + * profile_export.jsonl blobs: + * - p75/p90_osl_per_e2el: slow-tail OSL / E2E-latency in tok/s/user, + * computed as 1 / pXX(per-request E2EL/OSL) across every turn in every + * session — plain interactivity charged for the prefill wait. + * + * Ids without a trace_replay blob or with unparseable records are omitted. + */ +export const GET = idsQueryRoute({ + maxIds: 200, + logLabel: 'derived agentic metrics', + fetch: getCachedDerivedAgenticMetrics, +}); diff --git a/packages/app/src/components/inference/InferenceContext.tsx b/packages/app/src/components/inference/InferenceContext.tsx index 8b837ca5a..7d9e27a90 100644 --- a/packages/app/src/components/inference/InferenceContext.tsx +++ b/packages/app/src/components/inference/InferenceContext.tsx @@ -60,7 +60,12 @@ import { } from '@/lib/exclusion'; import { filterRunsByModel, getDisplayLabel } from '@/lib/utils'; -import { useChartData, X_AXIS_MODES, type XAxisMode } from './hooks/useChartData'; +import { + isAgenticOnlyXAxisMode, + useChartData, + X_AXIS_MODES, + type XAxisMode, +} from './hooks/useChartData'; import { resolveComparisonEntries } from './utils/comparisonEntry'; import { comparisonDefaultGroup, @@ -605,23 +610,41 @@ export function InferenceProvider({ }, [labelScenarioKind, sequenceResolved, getUrlParam]); // Reconcile the x-axis mode with the scenario kind: - // - On mount with no `i_xmode` URL param: snap to the natural default - // (interactivity for both agentic and fixed-sequence scenarios). The state - // was initialized to a SSR-stable constant so server and client render the - // same DOM; this effect fixes it up after hydration. - // - When the user later switches sequence kinds: snap back to that default - // (the prior selection was for a different kind, so it doesn't carry over). + // - On mount with no `i_xmode` URL param: snap to the kind's natural default + // (OSL / E2EL for agentic — the "north star" e2e-interactivity view — + // and interactivity for fixed-sequence scenarios). The state was initialized + // to a SSR-stable constant so server and client render the same DOM; this + // effect fixes it up after hydration. + // - When the user later switches sequence kinds: snap to the new kind's + // natural default (the prior selection was for a different kind, so it + // doesn't carry over). const lastSeqKindRef = useRef | null>(null); useEffect(() => { const kind = sequenceKind(effectiveSequence); const isInitialMount = lastSeqKindRef.current === null; - // Stale render where the kind hasn't changed — nothing to reconcile. - if (!isInitialMount && lastSeqKindRef.current === kind) return; + const isAgenticOnlyMode = isAgenticOnlyXAxisMode(selectedXAxisMode); + // On a stale render where kind hasn't changed, bail unless the current + // mode is agentic-only and we just landed on a fixed-seq scenario — in + // that case force the snap so the chart doesn't try to plot trace-derived + // metrics against rows that have no trace_replay. + if (!isInitialMount && lastSeqKindRef.current === kind) { + if (kind === 'fixed-seq' && isAgenticOnlyMode) { + handleSetXAxisMode('interactivity'); + } + return; + } lastSeqKindRef.current = kind; - // A URL-restored mode wins on first mount; later kind switches reset it. - if (isInitialMount && xAxisModeFromUrlRef.current) return; - handleSetXAxisMode('interactivity'); - }, [effectiveSequence, handleSetXAxisMode]); + if ( + isInitialMount && + xAxisModeFromUrlRef.current && + !(kind === 'fixed-seq' && isAgenticOnlyMode) + ) { + // URL-restored agentic-only mode on a fixed-seq sequence makes no sense + // — fall through to the default snap below. + return; + } + handleSetXAxisMode(kind === 'agentic' ? 'osl-e2el' : 'interactivity'); + }, [effectiveSequence, selectedXAxisMode, handleSetXAxisMode]); // Reconcile selectedE2eXAxisMetric whenever the mode, sequence kind, or // agentic percentile changes. For fixed-seq the JSONB only carries diff --git a/packages/app/src/components/inference/hooks/useChartData.ts b/packages/app/src/components/inference/hooks/useChartData.ts index c5593ee98..20f82515a 100644 --- a/packages/app/src/components/inference/hooks/useChartData.ts +++ b/packages/app/src/components/inference/hooks/useChartData.ts @@ -39,18 +39,28 @@ import { /** * Chart x-axis variant selected by the mode buttons above the plot. This is * the single definition — InferenceContext (URL/state) and ChartDisplay - * (buttons) import it from here. + * (buttons, derived-metric remapping) import it from here. */ -export type XAxisMode = 'ttft' | 'e2e' | 'interactivity'; +export type XAxisMode = 'ttft' | 'e2e' | 'interactivity' | 'osl-e2el'; -export const X_AXIS_MODES: readonly XAxisMode[] = ['ttft', 'e2e', 'interactivity']; +export const X_AXIS_MODES: readonly XAxisMode[] = ['ttft', 'e2e', 'interactivity', 'osl-e2el']; + +/** + * Modes whose x metric is derived from persisted per-request traces — + * these only exist for agentic scenarios (fixed-seq rows have no + * trace_replay blob to derive them from). + */ +export function isAgenticOnlyXAxisMode(mode: XAxisMode): boolean { + return mode === 'osl-e2el'; +} /** * Compute the set of benchmark_results.id values that sit on the * (e2e_latency, y) Pareto frontier within each (hwKey, precision, date) - * group. Used to restrict the non-e2e xmode charts (ttft, interactivity) so - * they show *only* the points that win on end-to-end latency — preventing - * benchmark-hacking where a config tops one axis while tanking the other. + * group. Used to restrict the non-e2e xmode charts (ttft, interactivity, + * osl-e2el) so they show *only* the points that win on + * end-to-end latency — preventing benchmark-hacking where a config tops + * one axis while tanking the other. * * Returns null when the y-metric has no roofline direction declared on * the e2e chart (caller falls back to no filtering in that case). @@ -197,7 +207,7 @@ export function useChartData( /** * Current x-axis mode. When set to anything other than 'e2e', the displayed * data is filtered to the (e2e-latency, y) Pareto frontier so the ttft / - * interactivity charts show only points that + * interactivity / osl-e2el charts show only points that * also win on end-to-end latency — preventing benchmark-hacking where a * config tops one metric while tanking the other. The 'e2e' mode is the * source of truth and keeps the full point set. @@ -506,7 +516,7 @@ export function useChartData( // non-optimal configs from view. // // Fixed-seq workloads keep the existing per-axis Pareto since - // there's no separate session-level notion of total latency — + // there's no separate "session-time" notion of total latency — // their e2e IS the request latency, so a TTFT hack there reads // honestly on e2e too. The anti-hack constraint is specifically // about multi-turn agentic where TTFT measures a tiny fraction diff --git a/packages/app/src/components/inference/types.ts b/packages/app/src/components/inference/types.ts index ae73710bd..96e6eb0af 100644 --- a/packages/app/src/components/inference/types.ts +++ b/packages/app/src/components/inference/types.ts @@ -258,7 +258,7 @@ export interface InferenceData extends Partial void; + selectedXAxisMode: 'ttft' | 'e2e' | 'interactivity' | 'osl-e2el'; + setSelectedXAxisMode: (mode: 'ttft' | 'e2e' | 'interactivity' | 'osl-e2el') => void; scaleType: 'auto' | 'linear' | 'log'; setScaleType: (type: 'auto' | 'linear' | 'log') => void; /** Coarse vendor / framework / agg-disagg / mtp-stp filters applied to the chart point set. */ diff --git a/packages/app/src/components/inference/ui/ChartDisplay.tsx b/packages/app/src/components/inference/ui/ChartDisplay.tsx index 6329df644..889352eaa 100644 --- a/packages/app/src/components/inference/ui/ChartDisplay.tsx +++ b/packages/app/src/components/inference/ui/ChartDisplay.tsx @@ -14,7 +14,10 @@ import type { OverlayData, TrendDataPoint, } from '@/components/inference/types'; -import { processOverlayChartData } from '@/components/inference/utils'; +import { + processOverlayChartData, + selectUnofficialOverlayForMode, +} from '@/components/inference/utils'; import { isRunComparisonEntry, makeRunComparisonEntry, @@ -53,7 +56,12 @@ import { sequenceKind, } from '@/lib/data-mappings'; import { useComparisonChangelogs } from '@/hooks/api/use-comparison-changelogs'; -import type { XAxisMode } from '@/components/inference/hooks/useChartData'; +import { + useDerivedAgenticMetrics, + type DerivedAgenticMetric, +} from '@/hooks/api/use-derived-agentic-metrics'; +import { isAgenticOnlyXAxisMode, type XAxisMode } from '@/components/inference/hooks/useChartData'; +import { isPersistedBenchmarkId } from '@/lib/benchmark-id'; import { useTrendData } from '@/components/inference/hooks/useTrendData'; import { getHardwareConfig, hardwareKeyMatchesAnyBase } from '@/lib/constants'; import { useLocale } from '@/lib/use-locale'; @@ -84,6 +92,8 @@ const STRINGS = { sourceUnofficial: 'Source: UNOFFICIAL', sourceOfficial: 'Source: SemiAnalysis InferenceX™', updated: 'Updated:', + oslE2elDisclaimer: + 'OSL / E2EL requires persisted per-request traces, so unofficial-run overlays are unavailable for this experimental view.', selectDateRange: 'Select a date range or add a run to view GPU comparison', performanceOverTime: 'Performance Over Time', performanceOverTimeDesc: @@ -101,6 +111,8 @@ const STRINGS = { sourceUnofficial: '来源：非官方', sourceOfficial: '来源：SemiAnalysis InferenceX™', updated: '更新时间：', + oslE2elDisclaimer: + 'OSL / E2EL 需要持久化的逐请求 trace 数据，因此该实验性视图不支持非官方运行覆盖。', selectDateRange: '请选择日期范围或添加运行以查看 GPU 对比', performanceOverTime: '性能趋势', performanceOverTimeDesc: '双击散点图上的数据点以追踪配置随时间的变化。', @@ -130,12 +142,50 @@ function zhHeading(configured: string): string { return `vs. ${pctl ? `${pctl} ` : ''}${subjectZh}`; } +// OSL / E2EL leads: it's the agentic default (and agentic-only, so the +// filter below drops it for fixed-seq, leaving Interactivity first there). const X_AXIS_MODE_BUTTONS: { value: XAxisMode; label: string; labelZh: string }[] = [ + { value: 'osl-e2el', label: 'OSL / E2EL', labelZh: 'OSL / E2EL' }, { value: 'interactivity', label: 'Interactivity', labelZh: '交互性' }, { value: 'e2e', label: 'E2E Latency', labelZh: '端到端延迟' }, { value: 'ttft', label: 'TTFT', labelZh: 'TTFT' }, ]; +/** + * Presentation + data plumbing for the trace-derived x-axis modes (the + * agentic-only modes). One spec per mode keeps the x-label, chart heading, + * roofline corner, and derived-metric accessor in sync instead of scattering + * `selectedXAxisMode === …` conditionals through the render. + */ +interface DerivedXModeSpec { + xLabel: (percentileLabel: string) => string; + /** Chinese x-label; omit to reuse the English one (technical terms). */ + xLabelZh?: (percentileLabel: string) => string; + /** Chart heading suffix ("vs. …") shown above the plot. */ + heading: (percentileLabel: string) => string; + /** Chinese heading suffix; omit to reuse the English one. */ + headingZh?: (percentileLabel: string) => string; + rooflineCorner: 'upper_right' | 'upper_left'; + /** Pull the raw metric for this mode off the derived-metrics payload. */ + value: (m: DerivedAgenticMetric | undefined, percentile: string) => number | null | undefined; + /** Convert the raw metric to the plotted x value. */ + toX: (raw: number) => number; +} + +const DERIVED_X_MODE_SPECS: Partial> = { + // "E2E interactivity": per-request OSL / E2E latency — the rate at which + // the user receives output tokens INCLUDING the prefill wait. Slow-tail + // percentiles (pXX = 1/pXX(E2EL/OSL)), matching the *_intvty convention. + 'osl-e2el': { + xLabel: (pctl) => `${pctl} OSL / E2EL (tok/s/user)`, + heading: (pctl) => `vs. ${pctl} OSL / E2EL`, + headingZh: (pctl) => `vs. ${pctl} OSL / 端到端延迟`, + rooflineCorner: 'upper_left', + value: (m, percentile) => (percentile === 'p75' ? m?.p75_osl_per_e2el : m?.p90_osl_per_e2el), + toX: (raw) => raw, + }, +}; + const VIEW_MODE_OPTIONS: SegmentedToggleOption[] = [ { value: 'chart', @@ -195,6 +245,9 @@ export default function ChartDisplay() { totalDatesQueried, } = useComparisonChangelogs(selectedGPUs, selectedDateRange, dateRangeAvailableDates); + const [mounted, setMounted] = useState(false); + useEffect(() => setMounted(true), []); + const modelDbKeys = useMemo( () => DISPLAY_MODEL_TO_DB[selectedModel] ?? [selectedModel], [selectedModel], @@ -557,191 +610,258 @@ export default function ChartDisplay() { }, [effectiveGraphs, selectedXAxisMode]); const isAgenticSequence = sequenceKind(selectedSequence) === 'agentic'; - const displayGraphs = isFirstLoad - ? [ - - - - - , - ] - : visibleGraphs.length === 0 - ? [] - : visibleGraphs.map((graph, graphIndex) => { - const isTimelineMode = Boolean( - selectedDateRange.startDate && selectedDateRange.endDate && selectedGPUs.length > 0, - ); - const replayAvailable = getViewMode(graphIndex) === 'chart' && !isTimelineMode; - return ( -
-
- handleViewModeChange(graphIndex, v)} - ariaLabel={t.viewMode} - testId={`inference-view-toggle-${graphIndex}`} - /> - } - hideImageExport={getViewMode(graphIndex) === 'table'} - setIsLegendExpanded={setIsLegendExpanded} - exportFileName={`InferenceX_${selectedModel}_${graph.chartDefinition.chartType}`} - onExportMp4={ - replayAvailable ? () => replayHandlesRef.current[graphIndex]?.open() : undefined - } - onExportCsv={() => { - const candidateVisibleData = isTimelineMode - ? graph.data.filter((d) => activeDates.has(`${d.date}_${d.hwKey}`)) - : graph.data; - const overlay = overlayDataByChartType[graph.chartDefinition.chartType]; - const { officialRows: visibleData, overlayRows: visibleOverlayRowsForExport } = + const useDerived = isAgenticSequence && isAgenticOnlyXAxisMode(selectedXAxisMode); + const derivedTargetIds = useMemo(() => { + if (!useDerived) return [] as number[]; + const ids = new Set(); + for (const graph of visibleGraphs) { + for (const point of graph.data) { + // Overlay-only agentic points carry no persisted id — skip them so we + // never request `?ids=0`/`?ids=NaN` (which 400s and errors the chart). + if (point.benchmark_type === 'agentic_traces' && isPersistedBenchmarkId(point.id)) { + ids.add(point.id); + } + } + } + return [...ids]; + }, [useDerived, visibleGraphs]); + const derivedQuery = useDerivedAgenticMetrics(derivedTargetIds, useDerived); + const derivedMetrics = derivedQuery.data; + const isDerivedLoading = + useDerived && + derivedTargetIds.length > 0 && + (derivedQuery.isPending || derivedQuery.isFetching) && + !derivedMetrics; + + // Set only when the user is on a derived (agentic-only) x-axis mode; the + // specs are module constants so this is referentially stable per mode. + const derivedSpec = useDerived ? DERIVED_X_MODE_SPECS[selectedXAxisMode] : undefined; + + const renderableGraphs = useMemo(() => { + if (!derivedSpec) return visibleGraphs; + if (!derivedMetrics) return visibleGraphs.map((graph) => ({ ...graph, data: [] })); + const xLabelFn = + locale === 'zh' && derivedSpec.xLabelZh ? derivedSpec.xLabelZh : derivedSpec.xLabel; + const xLabel = xLabelFn(selectedPercentile.toUpperCase()); + return visibleGraphs.map((graph) => { + const chartDefinition = { + ...graph.chartDefinition, + x_label: xLabel, + y_latency_limit: undefined, + [`${selectedYAxisMetric}_roofline` as keyof typeof graph.chartDefinition]: + derivedSpec.rooflineCorner, + }; + const data = graph.data + .map((point) => { + if (!isPersistedBenchmarkId(point.id)) return null; + const raw = derivedSpec.value(derivedMetrics[point.id], selectedPercentile); + if (raw === null || raw === undefined || !Number.isFinite(raw)) return null; + return { ...point, x: derivedSpec.toX(raw) }; + }) + .filter((point): point is NonNullable => point !== null); + return { ...graph, chartDefinition, data }; + }); + }, [derivedSpec, visibleGraphs, derivedMetrics, selectedYAxisMetric, selectedPercentile, locale]); + + const displayGraphs = + isFirstLoad || isDerivedLoading + ? [ + + + + + , + ] + : renderableGraphs.length === 0 + ? [] + : renderableGraphs.map((graph, graphIndex) => { + const isTimelineMode = Boolean( + selectedDateRange.startDate && selectedDateRange.endDate && selectedGPUs.length > 0, + ); + const replayAvailable = getViewMode(graphIndex) === 'chart' && !isTimelineMode; + return ( +
+
+ handleViewModeChange(graphIndex, v)} + ariaLabel={t.viewMode} + testId={`inference-view-toggle-${graphIndex}`} + /> + } + hideImageExport={getViewMode(graphIndex) === 'table'} + setIsLegendExpanded={setIsLegendExpanded} + exportFileName={`InferenceX_${selectedModel}_${graph.chartDefinition.chartType}`} + onExportMp4={ + replayAvailable + ? () => replayHandlesRef.current[graphIndex]?.open() + : undefined + } + onExportCsv={() => { + const candidateVisibleData = isTimelineMode + ? graph.data.filter((d) => activeDates.has(`${d.date}_${d.hwKey}`)) + : graph.data; + const overlay = selectUnofficialOverlayForMode( + selectedXAxisMode, + graph.chartDefinition.chartType, + overlayDataByChartType, + ); + const { + officialRows: visibleData, + overlayRows: visibleOverlayRowsForExport, + } = isTimelineMode ? { officialRows: candidateVisibleData, overlayRows: [] } : visibleComparisonRows(candidateVisibleData, overlay); - const { headers, rows } = inferenceChartToCsv( - visibleData, - graph.model, - graph.sequence, - ); - // Match warnings against the same series the chart annotates, - // including visible unofficial-run overlay series. - const issueNotes = matchKnownConfigIssues(graph.model, [ - ...visibleData, - ...visibleOverlayRowsForExport, - ]).map((issue) => - knownIssueCsvNote(issue, getDisplayLabel(getHardwareConfig(issue.hwKey))), - ); - exportToCsv( - `InferenceX_${selectedModel}_${graph.chartDefinition.chartType}`, - headers, - rows, - issueNotes, - ); - }} - /> - - {(() => { - const chartCaption = ( - <> -

- {metricTitle(graph.chartDefinition, selectedYAxisMetric, locale)}{' '} - {(() => { - // For Input metrics with dynamic x-axis, use dynamic heading. - // Classify off the ENGLISH title — the localized one has no - // 'input' substring to match on zh pages. - const isInputMetric = metricTitle( - graph.chartDefinition, - selectedYAxisMetric, - 'en', - ) - .toLowerCase() - .includes('input'); - if ( - graph.chartDefinition.chartType === 'interactivity' && - isInputMetric && - selectedXAxisMetric - ) { - if (selectedXAxisMetric === 'p99_ttft') { - return t.vsTtft('P99'); - } else if (selectedXAxisMetric === 'median_ttft') { - return t.vsTtft('Median'); + const { headers, rows } = inferenceChartToCsv( + visibleData, + graph.model, + graph.sequence, + ); + // Match warnings against the same series the chart annotates, + // including visible unofficial-run overlay series. + const issueNotes = matchKnownConfigIssues(graph.model, [ + ...visibleData, + ...visibleOverlayRowsForExport, + ]).map((issue) => + knownIssueCsvNote(issue, getDisplayLabel(getHardwareConfig(issue.hwKey))), + ); + exportToCsv( + `InferenceX_${selectedModel}_${graph.chartDefinition.chartType}`, + headers, + rows, + issueNotes, + ); + }} + /> + + {(() => { + const chartCaption = ( + <> +

+ {metricTitle(graph.chartDefinition, selectedYAxisMetric, locale)}{' '} + {(() => { + // For Input metrics with dynamic x-axis, use dynamic heading. + // Classify off the ENGLISH title — the localized one has no + // 'input' substring to match on zh pages. + const isInputMetric = metricTitle( + graph.chartDefinition, + selectedYAxisMetric, + 'en', + ) + .toLowerCase() + .includes('input'); + if ( + graph.chartDefinition.chartType === 'interactivity' && + isInputMetric && + selectedXAxisMetric + ) { + if (selectedXAxisMetric === 'p99_ttft') { + return t.vsTtft('P99'); + } else if (selectedXAxisMetric === 'median_ttft') { + return t.vsTtft('Median'); + } } - } - // The e2e chart heading follows the branch-level x-axis - // mode selector. - if (graph.chartDefinition.chartType === 'e2e') { - if (selectedE2eXAxisMetric?.endsWith('_ttft')) { - const percentile = selectedE2eXAxisMetric.replace(/_ttft$/u, ''); - const word = - percentile === 'median' ? 'Median' : percentile.toUpperCase(); - return t.vsTtft(word); + // The e2e chart heading follows the branch-level x-axis mode + // selector, including agentic-only derived metrics. + if (graph.chartDefinition.chartType === 'e2e') { + const modeSpec = DERIVED_X_MODE_SPECS[selectedXAxisMode]; + if (modeSpec) { + const heading = + locale === 'zh' && modeSpec.headingZh + ? modeSpec.headingZh + : modeSpec.heading; + return heading(selectedPercentile.toUpperCase()); + } + if (selectedE2eXAxisMetric?.endsWith('_ttft')) { + const percentile = selectedE2eXAxisMetric.replace(/_ttft$/u, ''); + const word = + percentile === 'median' ? 'Median' : percentile.toUpperCase(); + return t.vsTtft(word); + } + return isAgenticSequence + ? t.vsE2eLatency(selectedPercentile.toUpperCase()) + : t.vsE2eLatency(); } - return isAgenticSequence - ? t.vsE2eLatency(selectedPercentile.toUpperCase()) - : t.vsE2eLatency(); - } - // Fall back to configured heading - const configured = - graph.chartDefinition[ - `${selectedYAxisMetric}_heading` as keyof typeof graph.chartDefinition - ] || graph.chartDefinition.heading; - return locale === 'zh' ? zhHeading(String(configured)) : configured; - })()} -

-

- {getModelLabel(graph.model as Model)} •{' '} - {selectedPrecisions - .map((prec) => getPrecisionLabel(prec as Precision)) - .join(', ')}{' '} - • {getSequenceLabel(graph.sequence as Sequence)} •{' '} - {isUnofficialRun ? t.sourceUnofficial : t.sourceOfficial} - {selectedRunDate && ( - <> - {' '} - • {t.updated}{' '} - {new Date(`${selectedRunDate}T00:00:00Z`).toLocaleDateString( - locale === 'zh' ? 'zh-CN' : 'en-US', - { - year: 'numeric', - month: '2-digit', - day: '2-digit', - timeZone: 'UTC', - }, - )} - + // Fall back to configured heading + const configured = + graph.chartDefinition[ + `${selectedYAxisMetric}_heading` as keyof typeof graph.chartDefinition + ] || graph.chartDefinition.heading; + return locale === 'zh' ? zhHeading(String(configured)) : configured; + })()} +

+

+ {getModelLabel(graph.model as Model)} •{' '} + {selectedPrecisions + .map((prec) => getPrecisionLabel(prec as Precision)) + .join(', ')}{' '} + • {getSequenceLabel(graph.sequence as Sequence)} •{' '} + {isUnofficialRun ? t.sourceUnofficial : t.sourceOfficial} + {selectedRunDate && ( + <> + {' '} + • {t.updated}{' '} + {new Date(`${selectedRunDate}T00:00:00Z`).toLocaleDateString( + locale === 'zh' ? 'zh-CN' : 'en-US', + { + year: 'numeric', + month: '2-digit', + day: '2-digit', + timeZone: 'UTC', + }, + )} + + )} +

+ + {isUnofficialRun && selectedXAxisMode === 'osl-e2el' && ( +

+ {t.oslE2elDisclaimer} +

)} -

- - - - ); - - if (getViewMode(graphIndex) === 'table') { - const overlay = overlayDataByChartType[graph.chartDefinition.chartType]; - const { officialRows, overlayRows } = visibleComparisonRows( - graph.data, - overlay, - ); - return ( - <> - {chartCaption} - + ); - } - return selectedGPUs.length > 0 && - ((selectedDateRange.startDate && selectedDateRange.endDate) || - selectedDates.length > 0) ? ( - - ) : ( -
- + {chartCaption} + + + ); + } + + return selectedGPUs.length > 0 && + ((selectedDateRange.startDate && selectedDateRange.endDate) || + selectedDates.length > 0) ? ( + - {selectedGPUs.length > 0 && - (!selectedDateRange.startDate || !selectedDateRange.endDate) && - selectedDates.length === 0 && ( -
-

- {t.selectDateRange} -

-
- )} -
- ); - })()} - {replayAvailable && ( - { - replayHandlesRef.current[graphIndex] = handle; - }} - parentChartId={`chart-${graphIndex}`} - chartDefinition={graph.chartDefinition} - yLabel={metricLabel(graph.chartDefinition, selectedYAxisMetric, locale)} - xLabel={graph.chartDefinition.x_label} - /> - )} - -
-
- ); - }); + ) : ( +
+ + {selectedGPUs.length > 0 && + (!selectedDateRange.startDate || !selectedDateRange.endDate) && + selectedDates.length === 0 && ( +
+

+ {t.selectDateRange} +

+
+ )} +
+ ); + })()} + {replayAvailable && ( + { + replayHandlesRef.current[graphIndex] = handle; + }} + parentChartId={`chart-${graphIndex}`} + chartDefinition={graph.chartDefinition} + yLabel={metricLabel(graph.chartDefinition, selectedYAxisMetric, locale)} + xLabel={graph.chartDefinition.x_label} + /> + )} + +
+
+ ); + }); return (
@@ -847,7 +983,12 @@ export default function ChartDisplay() { data-testid="x-axis-mode-buttons" className="flex-wrap justify-center gap-x-1 gap-y-1.5 sm:gap-x-1.5" > - {X_AXIS_MODE_BUTTONS.map(({ value, label, labelZh }) => ( + {X_AXIS_MODE_BUTTONS.filter(({ value }) => { + if (!isAgenticOnlyXAxisMode(value)) return true; + // Before mount, render all buttons so SSR and first client render match. + if (!mounted) return true; + return isAgenticSequence; + }).map(({ value, label, labelZh }) => ( { + const overlays = { e2e: { id: 'e2e' }, interactivity: { id: 'interactivity' } }; + + it('suppresses raw unofficial E2E data for OSL / E2EL mode', () => { + expect(selectUnofficialOverlayForMode('osl-e2el', 'e2e', overlays)).toBeNull(); + }); + + it('preserves matching unofficial overlays for supported modes', () => { + expect(selectUnofficialOverlayForMode('e2e', 'e2e', overlays)).toBe(overlays.e2e); + expect(selectUnofficialOverlayForMode('interactivity', 'interactivity', overlays)).toBe( + overlays.interactivity, + ); + }); +}); // --------------------------------------------------------------------------- // fixture factories diff --git a/packages/app/src/components/inference/utils.ts b/packages/app/src/components/inference/utils.ts index e88e5b0da..62244d4d7 100644 --- a/packages/app/src/components/inference/utils.ts +++ b/packages/app/src/components/inference/utils.ts @@ -10,6 +10,21 @@ import { resolveXAxisField } from '@/components/inference/utils/resolveXAxisFiel import type { ChartDefinition, InferenceData, YAxisMetricKey } from './types'; +/** + * Select the matching unofficial-run overlay for a chart mode. OSL / E2EL + * is intentionally excluded: unofficial benchmark rows do not include the + * persisted per-request trace needed to derive the per-request ratio + * percentiles. + */ +export function selectUnofficialOverlayForMode( + xAxisMode: string, + chartType: 'e2e' | 'interactivity', + overlays: { e2e: T | null; interactivity: T | null }, +): T | null { + if (xAxisMode === 'osl-e2el') return null; + return overlays[chartType]; +} + /** * Filters data points based on cost limits defined in the chart definition. * Only applies filtering for cost-related metrics, and only filters based on diff --git a/packages/app/src/components/inference/utils/e2eFrontier.ts b/packages/app/src/components/inference/utils/e2eFrontier.ts index 65c3434a8..a7c8ca29b 100644 --- a/packages/app/src/components/inference/utils/e2eFrontier.ts +++ b/packages/app/src/components/inference/utils/e2eFrontier.ts @@ -2,7 +2,7 @@ * @file e2eFrontier.ts * @description Shared seed for the anti-benchmark-hacking roofline restriction. * - * On the non-e2e xmode charts (interactivity, ttft) + * On the non-e2e xmode charts (interactivity, ttft, osl-e2el) * the roofline is restricted to the configs that ALSO win on end-to-end latency, * so a config can't top interactivity while tanking decode (or vice versa). Both * the official path (`useChartData` → benchmark ids → `isOnE2eFrontier`) and the diff --git a/packages/app/src/hooks/api/use-derived-agentic-metrics.test.ts b/packages/app/src/hooks/api/use-derived-agentic-metrics.test.ts new file mode 100644 index 000000000..2e54f4187 --- /dev/null +++ b/packages/app/src/hooks/api/use-derived-agentic-metrics.test.ts @@ -0,0 +1,13 @@ +import { describe, expect, it } from 'vitest'; + +import { chunkDerivedAgenticMetricIds } from './use-derived-agentic-metrics'; + +describe('chunkDerivedAgenticMetricIds', () => { + it('keeps every id while respecting the API limit', () => { + const ids = Array.from({ length: 401 }, (_, index) => index + 1); + const chunks = chunkDerivedAgenticMetricIds(ids); + + expect(chunks.map((chunk) => chunk.length)).toEqual([200, 200, 1]); + expect(chunks.flat()).toEqual(ids); + }); +}); diff --git a/packages/app/src/hooks/api/use-derived-agentic-metrics.ts b/packages/app/src/hooks/api/use-derived-agentic-metrics.ts new file mode 100644 index 000000000..737c41b8e --- /dev/null +++ b/packages/app/src/hooks/api/use-derived-agentic-metrics.ts @@ -0,0 +1,50 @@ +import { bulkIdsFetcher, useBulkIdsQuery } from './benchmark-id-query'; + +export interface DerivedAgenticMetric { + id: number; + /** Slow-tail P75 OSL/E2EL in tok/s/user — 1 / p75(per-request E2EL/OSL). + * Null when the JSONL had no usable records. */ + p75_osl_per_e2el: number | null; + /** Slow-tail P90 OSL/E2EL in tok/s/user — 1 / p90(per-request E2EL/OSL). */ + p90_osl_per_e2el: number | null; +} + +export type DerivedAgenticMetricMap = Record; + +const MAX_IDS_PER_REQUEST = 200; + +export function chunkDerivedAgenticMetricIds(ids: number[]): number[][] { + const chunks: number[][] = []; + for (let i = 0; i < ids.length; i += MAX_IDS_PER_REQUEST) { + chunks.push(ids.slice(i, i + MAX_IDS_PER_REQUEST)); + } + return chunks; +} + +const fetchChunk = bulkIdsFetcher('derived-agentic-metrics'); + +// Unlike the other bulk endpoints, dashboards can put >200 agentic points on +// screen at once, so this fetcher splits the id set across parallel requests +// to stay under the route's MAX_IDS_PER_REQUEST. +async function fetchDerivedAgenticMetrics( + ids: number[], + signal?: AbortSignal, +): Promise { + if (ids.length === 0) return {}; + const maps = await Promise.all( + chunkDerivedAgenticMetricIds(ids).map((chunk) => fetchChunk(chunk, signal)), + ); + return Object.assign({}, ...maps) as DerivedAgenticMetricMap; +} + +/** + * Fetch per-id derived agentic metrics (slow-tail OSL/E2EL tok/s/user) + * computed live from the stored aiperf profile_export.jsonl. Used to drive + * the "OSL / E2EL" chart variant. + * + * Ids without a trace_replay blob (older or non-aiperf agentic runs) are + * silently omitted from the response. + */ +export function useDerivedAgenticMetrics(ids: number[], enabled = true) { + return useBulkIdsQuery('derived-agentic-metrics', ids, enabled, fetchDerivedAgenticMetrics); +} diff --git a/packages/db/src/backfill-aggregate-stats.ts b/packages/db/src/backfill-aggregate-stats.ts index a50f6ea02..e7981b924 100644 --- a/packages/db/src/backfill-aggregate-stats.ts +++ b/packages/db/src/backfill-aggregate-stats.ts @@ -94,9 +94,9 @@ async function main(): Promise { } let stats: AggregateStats; - // v3 onwards → current is a profile-only change (the server-derived - // fields haven't changed since v3), so skip re-reading the huge server - // blob and carry its KV/prefix distributions forward. + // v3 onwards → current is a profile-only change (the server-derived fields + // haven't changed since v3), so skip re-reading the huge server blob and + // carry its KV/prefix distributions forward. const storedVersion = row.aggregate_stats?.version; if (storedVersion !== undefined && storedVersion >= 3 && storedVersion < STATS_VERSION) { const profileStats = await computeAggregateStats({ diff --git a/packages/db/src/etl/compute-aggregate-stats.test.ts b/packages/db/src/etl/compute-aggregate-stats.test.ts index 1a237664d..b08f46b55 100644 --- a/packages/db/src/etl/compute-aggregate-stats.test.ts +++ b/packages/db/src/etl/compute-aggregate-stats.test.ts @@ -66,9 +66,10 @@ describe('computeAggregateStats', () => { expect(stats.osl).toBeNull(); expect(stats.kvCacheUtil).toBeNull(); expect(stats.prefixCacheHitRate).toBeNull(); + expect(stats.e2elPerOsl).toBeNull(); }); - it('computes ISL/OSL percentiles from the profile blob', async () => { + it('computes ISL/OSL percentiles + the E2EL/OSL ratio bundle from the profile blob', async () => { const profileBlob = makeProfileBlob([ { isl: 100, osl: 50, rl: 1000, ttft: 100 }, { isl: 200, osl: 75, rl: 2000, ttft: 200 }, @@ -84,6 +85,12 @@ describe('computeAggregateStats', () => { // Server-side metrics still null when there's no server blob. expect(stats.kvCacheUtil).toBeNull(); expect(stats.prefixCacheHitRate).toBeNull(); + + // Per-request E2EL/OSL ratios (s/tok): 1/50=0.02, 2/75≈0.02667, 3/100=0.03. + expect(stats.e2elPerOsl?.n).toBe(3); + expect(stats.e2elPerOsl?.p50).toBeCloseTo(2 / 75, 6); + // p90 of 3 values (linear interpolation): pos=1.8 → 0.02667 + 0.8×(0.03-0.02667) + expect(stats.e2elPerOsl?.p90).toBeCloseTo(2 / 75 + 0.8 * (0.03 - 2 / 75), 6); }); it('computes KV util + prefix hit rate from the server blob alone', async () => { @@ -99,6 +106,7 @@ describe('computeAggregateStats', () => { // Profile-derived metrics absent. expect(stats.isl).toBeNull(); expect(stats.osl).toBeNull(); + expect(stats.e2elPerOsl).toBeNull(); }); it('tolerates a malformed profile blob by leaving its metrics null', async () => { @@ -107,6 +115,7 @@ describe('computeAggregateStats', () => { const stats = await computeAggregateStats({ profileBlob: garbage, serverBlob: null }); expect(stats.isl).toBeNull(); expect(stats.osl).toBeNull(); + expect(stats.e2elPerOsl).toBeNull(); // Version still set so the row is considered "computed". expect(stats.version).toBe(STATS_VERSION); }); @@ -126,14 +135,14 @@ describe('mergeProfileStatsUpgrade', () => { const merged = mergeProfileStatsUpgrade(existing, profile); expect(merged.version).toBe(STATS_VERSION); expect(merged.isl?.mean).toBe(100); + expect(merged.e2elPerOsl?.p90).toBeCloseTo(2.08 / 100, 6); expect(merged.kvCacheUtil).toEqual(existing.kvCacheUtil); expect(merged.prefixCacheHitRate).toEqual(existing.prefixCacheHitRate); }); - it('drops the retired derived fields when upgrading a pre-v6 bundle', async () => { - // Bundles written before v6 carry normalizedSessionTimeS / - // p90PrefillTpsPerUser / normalizedE2e400 — the upgrade must not - // resurrect them into the new bundle. + it('drops legacy derived fields when upgrading a pre-v6 bundle', async () => { + // A stale bundle from an older STATS_VERSION carries since-retired fields — + // the merged result must not resurrect them. const legacy = { version: STATS_VERSION - 1, isl: null, diff --git a/packages/db/src/etl/compute-aggregate-stats.ts b/packages/db/src/etl/compute-aggregate-stats.ts index d4a76e27b..1ede7cfbd 100644 --- a/packages/db/src/etl/compute-aggregate-stats.ts +++ b/packages/db/src/etl/compute-aggregate-stats.ts @@ -1,8 +1,9 @@ /** * Pre-compute the per-row aggregate stats for an `agentic_trace_replay` * blob pair. The output lands in the `aggregate_stats` JSONB column so the - * detail page can serve the "Aggregates across configs" view from a single - * SQL row read, instead of parsing the raw blobs on demand. + * detail page can serve the "Aggregates across configs" view and the + * derived chart x-axis modes from a single SQL row read, instead of + * parsing the raw blobs on demand. * * Shape is intentionally versioned — bump `STATS_VERSION` whenever the * computation changes so the backfill script knows which rows to recompute. @@ -11,6 +12,7 @@ import { gunzipSync } from 'node:zlib'; import { gunzipJsonWithinLimit, streamCollectKeys } from './gzip-json-stream'; +import { computeDerivedFromBlob } from '../queries/derived-agentic-metrics'; import { STATS_VERSION, extractIslOsl, @@ -27,11 +29,17 @@ export interface AggregateStats { osl: MetricPercentiles | null; kvCacheUtil: MetricPercentiles | null; prefixCacheHitRate: MetricPercentiles | null; + /** + * Per-request E2E latency / OSL (seconds per output token) percentiles. + * The read path inverts to plot the slow-tail "OSL / E2EL" x-axis metric + * (tok/s/user): pXX OSL/E2EL = 1 / pXX(E2EL/OSL). + */ + e2elPerOsl: MetricPercentiles | null; } /** * The subset of an older-version bundle a profile-only upgrade carries - * forward. Pre-v6 bundles also carry the since-retired derived metrics + * forward. Pre-v6 bundles also carry since-retired derived fields * (normalizedSessionTimeS, p90PrefillTpsPerUser, normalizedE2e400) — spreading * `profile` first drops them from the merged result. */ @@ -93,6 +101,7 @@ export async function computeAggregateStats(args: { }): Promise { let islPct: MetricPercentiles | null = null; let oslPct: MetricPercentiles | null = null; + let e2elPerOsl: MetricPercentiles | null = null; if (args.profileBlob) { try { @@ -100,6 +109,7 @@ export async function computeAggregateStats(args: { const { isl, osl } = extractIslOsl(jsonl); islPct = percentilesOf(isl); oslPct = percentilesOf(osl); + e2elPerOsl = computeDerivedFromBlob(jsonl).e2el_per_osl; } catch { // ignore malformed blob — leave nulls } @@ -130,5 +140,6 @@ export async function computeAggregateStats(args: { osl: oslPct, kvCacheUtil: kvPct, prefixCacheHitRate: prefixPct, + e2elPerOsl, }; } diff --git a/packages/db/src/queries/agentic-aggregates.test.ts b/packages/db/src/queries/agentic-aggregates.test.ts index d0daa2a39..566ca65f7 100644 --- a/packages/db/src/queries/agentic-aggregates.test.ts +++ b/packages/db/src/queries/agentic-aggregates.test.ts @@ -129,6 +129,7 @@ interface WrittenStats { osl: unknown; kvCacheUtil: { mean: number } | null; prefixCacheHitRate: unknown; + e2elPerOsl: { p90: number; n: number } | null; } /** Capture SQL template text + bound values for the write-back assertions. */ @@ -225,11 +226,15 @@ describe('getAgenticAggregates write-back', () => { expect(written.version).toBe(STATS_VERSION); // Server field FRESHLY recomputed (0.25), not the stale 0.9 carried forward. expect(written.kvCacheUtil?.mean).toBeCloseTo(0.25, 6); - expect(written.isl).not.toBeNull(); - // The retired derived fields must not survive the recompute. + // Derived ratio bundle FRESHLY recomputed from the two turns: + // ratios (s/tok) = [1.0/50, 2.0/50] = [0.02, 0.04] → p90 = 0.038. + expect(written.e2elPerOsl?.n).toBe(2); + expect(written.e2elPerOsl?.p90).toBeCloseTo(0.038, 8); + // Retired legacy fields must not survive the recompute. expect(written).not.toHaveProperty('normalizedSessionTimeS'); expect(written).not.toHaveProperty('p90PrefillTpsPerUser'); expect(written).not.toHaveProperty('normalizedE2e400'); + expect(written.isl).not.toBeNull(); }); it('does not write back for an id whose profile blob is missing/malformed', async () => { diff --git a/packages/db/src/queries/agentic-aggregates.ts b/packages/db/src/queries/agentic-aggregates.ts index 901defb9f..44abd8abf 100644 --- a/packages/db/src/queries/agentic-aggregates.ts +++ b/packages/db/src/queries/agentic-aggregates.ts @@ -24,6 +24,7 @@ import { streamObject } from 'stream-json/streamers/stream-object.js'; import { gunzipJsonWithinLimit } from '../etl/gzip-json-stream'; import type { DbClient } from '../connection.js'; +import { computeDerivedFromBlob } from './derived-agentic-metrics'; import { extractIslOsl, fetchAggregateStatsRows, @@ -315,8 +316,12 @@ export async function getAgenticAggregates( const oslPct = percentilesOf(osl); result[id].isl = islPct; result[id].osl = oslPct; - // Server-derived fields are filled in Pass 2 (or stay null when the - // server blob is absent, which is the correct complete value). + // Recompute the profile-derived fields too (same jsonl, no extra + // read) so the self-healed bundle is a faithful full recompute — not + // a carry-forward of stale derived numbers stamped with a new + // version. Server-derived fields are filled in Pass 2 (or stay null + // when the server blob is absent, which is the correct complete value). + const derived = computeDerivedFromBlob(jsonl); pendingById.set(id, { traceReplayId: Number(row.trace_replay_id), stats: { @@ -325,6 +330,7 @@ export async function getAgenticAggregates( osl: oslPct, kvCacheUtil: null, prefixCacheHitRate: null, + e2elPerOsl: derived.e2el_per_osl, }, }); } catch { @@ -409,6 +415,7 @@ interface FullAggregateStats { osl: MetricPercentiles | null; kvCacheUtil: MetricPercentiles | null; prefixCacheHitRate: MetricPercentiles | null; + e2elPerOsl: MetricPercentiles | null; } function blankAggregate(id: number): AgenticAggregate { diff --git a/packages/db/src/queries/agentic-shared.ts b/packages/db/src/queries/agentic-shared.ts index d704abe02..a18344fc1 100644 --- a/packages/db/src/queries/agentic-shared.ts +++ b/packages/db/src/queries/agentic-shared.ts @@ -34,10 +34,13 @@ import type { DbClient } from '../connection.js'; * * v6: drop the retired per-point derived metrics (normalizedSessionTimeS, * p90PrefillTpsPerUser, normalizedE2e400) along with the experimental chart - * modes they fed. The bundle is now isl/osl + the server-derived kvCacheUtil - * and prefixCacheHitRate. + * modes they fed. + * + * v7: add `e2elPerOsl` — percentiles of per-request E2E latency divided by + * OSL (seconds per output token), the inverse of the "OSL / E2EL" x-axis + * metric. */ -export const STATS_VERSION = 6; +export const STATS_VERSION = 7; interface ProfileRecord { metadata?: { benchmark_phase?: string }; diff --git a/packages/db/src/queries/derived-agentic-metrics.test.ts b/packages/db/src/queries/derived-agentic-metrics.test.ts new file mode 100644 index 000000000..144f91fad --- /dev/null +++ b/packages/db/src/queries/derived-agentic-metrics.test.ts @@ -0,0 +1,237 @@ +import { gzipSync } from 'node:zlib'; + +import { describe, expect, it } from 'vitest'; + +import { STATS_VERSION } from './agentic-shared'; +import type { DbClient } from '../connection.js'; + +import { computeDerivedFromBlob, getDerivedAgenticMetrics } from './derived-agentic-metrics.js'; + +/** Build one aiperf JSONL record for the synthetic fixture. */ +function rec( + conversation_id: string, + turn_index: number, + fields: { isl: number; osl: number; ttft_ms: number; latency_ms: number }, +): string { + return JSON.stringify({ + metadata: { conversation_id, turn_index, benchmark_phase: 'profiling' }, + metrics: { + request_latency: { value: fields.latency_ms, unit: 'ms' }, + time_to_first_token: { value: fields.ttft_ms, unit: 'ms' }, + input_sequence_length: { value: fields.isl, unit: 'tokens' }, + output_sequence_length: { value: fields.osl, unit: 'tokens' }, + }, + }); +} + +describe('computeDerivedFromBlob', () => { + it('returns null when no usable records', () => { + const out = computeDerivedFromBlob(''); + expect(out.e2el_per_osl).toBeNull(); + }); + + it('computes per-request E2EL/OSL ratios pooled across sessions', () => { + // Ratios (s per output token): 1.0/50 = 0.02, 4.0/100 = 0.04. + const jsonl = [ + rec('s1', 0, { isl: 100, osl: 50, ttft_ms: 500, latency_ms: 1000 }), + rec('s2', 0, { isl: 200, osl: 100, ttft_ms: 1000, latency_ms: 4000 }), + ].join('\n'); + + const out = computeDerivedFromBlob(jsonl); + expect(out.e2el_per_osl?.n).toBe(2); + expect(out.e2el_per_osl?.p50).toBeCloseTo(0.03, 8); + // p90 of [0.02, 0.04]: pos = 0.9 → 0.02 + 0.9 × 0.02 = 0.038. + expect(out.e2el_per_osl?.p90).toBeCloseTo(0.038, 8); + expect(out.e2el_per_osl?.p75).toBeCloseTo(0.035, 8); + }); + + it('OSL cancels out of the ratio when TTFT and ITL are identical', () => { + // Both requests: TTFT=2s, ITL=20ms — very different OSL/E2E, but the + // per-token ratio only differs through the TTFT amortization term + // (TTFT/OSL), which is the intended second-order OSL sensitivity. + const jsonl = [ + rec('s1', 0, { isl: 100, osl: 100, ttft_ms: 2000, latency_ms: 3980 }), + rec('s2', 0, { isl: 100, osl: 1000, ttft_ms: 2000, latency_ms: 21_980 }), + ].join('\n'); + + const out = computeDerivedFromBlob(jsonl); + expect(out.e2el_per_osl?.n).toBe(2); + // 3.98/100 = 0.0398 ≈ ITL + TTFT/OSL = 0.0198 + 0.02 + expect(out.e2el_per_osl?.p50).toBeCloseTo((0.0398 + 0.02198) / 2, 8); + }); + + it('drops records missing required fields and skips non-profiling phase', () => { + const lines = [ + rec('s1', 0, { isl: 100, osl: 50, ttft_ms: 500, latency_ms: 1000 }), + // missing TTFT — should be skipped + JSON.stringify({ + metadata: { conversation_id: 's1', turn_index: 1, benchmark_phase: 'profiling' }, + metrics: { + request_latency: { value: 1000, unit: 'ms' }, + input_sequence_length: { value: 100, unit: 'tokens' }, + output_sequence_length: { value: 50, unit: 'tokens' }, + }, + }), + // warmup phase — should be skipped + JSON.stringify({ + metadata: { conversation_id: 's2', turn_index: 0, benchmark_phase: 'warmup' }, + metrics: { + request_latency: { value: 9999, unit: 'ms' }, + time_to_first_token: { value: 9999, unit: 'ms' }, + input_sequence_length: { value: 100, unit: 'tokens' }, + output_sequence_length: { value: 50, unit: 'tokens' }, + }, + }), + ]; + const out = computeDerivedFromBlob(lines.join('\n')); + expect(out.e2el_per_osl?.n).toBe(1); + expect(out.e2el_per_osl?.p90).toBeCloseTo(0.02, 8); + }); + + it('excludes osl=0 (cancelled/empty-output) turns from the ratio distribution', () => { + const jsonl = [ + rec('s1', 0, { isl: 100, osl: 50, ttft_ms: 500, latency_ms: 1000 }), + // Cancelled / empty-output turn — osl=0 must be rejected by extractTurn + // (the ratio would divide by zero). + rec('s2', 0, { isl: 150, osl: 0, ttft_ms: 1000, latency_ms: 30000 }), + ].join('\n'); + + const out = computeDerivedFromBlob(jsonl); + expect(out.e2el_per_osl?.n).toBe(1); + expect(out.e2el_per_osl?.p90).toBeCloseTo(0.02, 8); + }); + + it('p90 across turns: 10-turn population picks the right rank', () => { + // Ratios 0.01..0.10 s/tok; p90 of 10 values (linear) = 0.091. + const turns = Array.from({ length: 10 }, (_, i) => + rec('s1', i, { + isl: 100, + osl: 100, + ttft_ms: 100, + latency_ms: (i + 1) * 1000, // 1s..10s → ratios 0.01..0.10 + }), + ); + const out = computeDerivedFromBlob(turns.join('\n')); + expect(out.e2el_per_osl?.p90).toBeCloseTo(0.091, 8); + }); +}); + +/** Capture SQL template text + bound values for the write-back assertions. */ +function mockSql(queue: unknown[][]): { + sql: DbClient; + calls: { text: string; values: unknown[] }[]; +} { + const responses = [...queue]; + const calls: { text: string; values: unknown[] }[] = []; + const sql = ((strings: TemplateStringsArray, ...values: unknown[]) => { + calls.push({ text: strings.join('?'), values }); + return Promise.resolve(responses.shift() ?? []); + }) as unknown as DbClient; + return { sql, calls }; +} + +describe('getDerivedAgenticMetrics write-back', () => { + it('self-heals aggregate_stats from the profile blob, carrying server fields forward', async () => { + // Two turns with ratios 0.02 and 0.04 s/tok → p90 = 0.038, p75 = 0.035. + const jsonl = [ + rec('s1', 0, { isl: 100, osl: 50, ttft_ms: 500, latency_ms: 1000 }), + rec('s1', 1, { isl: 200, osl: 100, ttft_ms: 1000, latency_ms: 4000 }), + ].join('\n'); + const blob = gzipSync(Buffer.from(jsonl)); + + // Stale v(N-1) row that DOES carry server-derived fields — they must be + // preserved in the healed bundle (derived route can't recompute them). + const staleServerKv = { mean: 0.4, p50: 0.4, p75: 0.5, p90: 0.6, p99: 0.7, n: 3 }; + const staleStats = { + version: STATS_VERSION - 1, + isl: null, + osl: null, + kvCacheUtil: staleServerKv, + prefixCacheHitRate: null, + normalizedSessionTimeS: 999, + p90PrefillTpsPerUser: 999, + normalizedE2e400: null, + }; + + const { sql, calls } = mockSql([ + // fetchAggregateStatsRows + [{ benchmark_result_id: 7, stats: staleStats }], + // fallback profile-blob query + [{ benchmark_result_id: 7, trace_replay_id: 870, blob }], + ]); + + const result = await getDerivedAgenticMetrics(sql, [7]); + + // Response is the freshly recomputed slow-tail inverse (tok/s/user). + expect(result[7]?.p90_osl_per_e2el).toBeCloseTo(1 / 0.038, 6); + expect(result[7]?.p75_osl_per_e2el).toBeCloseTo(1 / 0.035, 6); + + // 3 calls: stats read, blob read, write-back UPDATE. + expect(calls).toHaveLength(3); + expect(calls[2]!.text).toContain('update agentic_trace_replay set aggregate_stats'); + expect(calls[2]!.text).toContain('::jsonb where id'); + + // The write-back binds a COMPLETE, version-stamped bundle at the new version, + // recomputing profile fields and carrying server fields forward untouched. + // The payload OBJECT is bound directly (not stringified — that would + // double-encode into a JSONB string). + interface WrittenStats { + version: number; + isl: unknown; + osl: unknown; + kvCacheUtil: unknown; + e2elPerOsl: { p75: number; p90: number; n: number } | null; + } + const [written, traceReplayId] = calls[2]!.values as [WrittenStats, number]; + expect(traceReplayId).toBe(870); + expect(written.version).toBe(STATS_VERSION); + expect(written.e2elPerOsl?.n).toBe(2); + expect(written.e2elPerOsl?.p90).toBeCloseTo(0.038, 8); + expect(written.isl).not.toBeNull(); + expect(written.osl).not.toBeNull(); + // Server-derived field carried forward from the stale row (not re-read). + expect(written.kvCacheUtil).toEqual(staleServerKv); + // Retired legacy fields must not survive the heal. + expect(written).not.toHaveProperty('normalizedSessionTimeS'); + expect(written).not.toHaveProperty('p90PrefillTpsPerUser'); + expect(written).not.toHaveProperty('normalizedE2e400'); + }); + + it('takes the fast path (no blob read, no write-back) when stats are current', async () => { + const currentStats = { + version: STATS_VERSION, + isl: null, + osl: null, + kvCacheUtil: null, + prefixCacheHitRate: null, + e2elPerOsl: { mean: 0.03, p50: 0.03, p75: 0.04, p90: 0.05, p99: 0.06, n: 5 }, + }; + const { sql, calls } = mockSql([[{ benchmark_result_id: 7, stats: currentStats }]]); + + const result = await getDerivedAgenticMetrics(sql, [7]); + + expect(result[7]?.p75_osl_per_e2el).toBeCloseTo(25, 6); + expect(result[7]?.p90_osl_per_e2el).toBeCloseTo(20, 6); + // Only the stats read — no fallback blob query, no write-back. + expect(calls).toHaveLength(1); + }); + + it('maps a missing ratio bundle to nulls on the fast path', async () => { + const currentStats = { + version: STATS_VERSION, + isl: null, + osl: null, + kvCacheUtil: null, + prefixCacheHitRate: null, + e2elPerOsl: null, + }; + const { sql } = mockSql([ + [{ benchmark_result_id: 7, stats: currentStats }], + // fallback blob query fires for no ids → but guard returns before; keep + // the queue empty-safe anyway. + ]); + + const result = await getDerivedAgenticMetrics(sql, [7]); + expect(result[7]).toEqual({ id: 7, p75_osl_per_e2el: null, p90_osl_per_e2el: null }); + }); +}); diff --git a/packages/db/src/queries/derived-agentic-metrics.ts b/packages/db/src/queries/derived-agentic-metrics.ts new file mode 100644 index 000000000..7159b84d5 --- /dev/null +++ b/packages/db/src/queries/derived-agentic-metrics.ts @@ -0,0 +1,233 @@ +/** + * Live-computed per-point metrics derived from the stored aiperf + * `profile_export.jsonl` blob. These aren't precomputed in the metrics JSONB + * because they require a full pass over the per-request records — work that's + * cheap once per agentic point but adds up to be meaningful only when + * actually plotted. + * + * - OSL / E2EL ("e2e interactivity"): per-request output_sequence_length + * divided by request_latency — the rate at which the user receives output + * tokens INCLUDING the prefill wait, per + * https://semianalysis.slack.com/archives/C0AV4T40BT3/p1782432266626969. + * Algebraically `OSL / (TTFT + decode_time) ≈ 1 / (ITL + TTFT/OSL)`: plain + * interactivity with a penalty that grows when TTFT is large relative to + * the output produced, so prefill-delaying can't inflate the metric the way + * it can with 1/TPOT. + * + * Percentiles follow the slow-tail convention the mapper enforces for + * `*_intvty` (1/p(ITL), not p(1/ITL)): we store percentiles of the + * per-request E2EL/OSL ratio (seconds per output token) and the read path + * inverts, so `p90 OSL/E2EL = 1 / p90(E2EL/OSL)` is the 90th-percentile + * WORST request's effective token rate. + */ + +import { gunzipSync } from 'node:zlib'; + +import type { DbClient } from '../connection.js'; +import { + extractIslOsl, + fetchAggregateStatsRows, + percentilesOf, + readNum, + STATS_VERSION, + writeBackTraceReplayJsonb, + type MetricPercentiles, +} from './agentic-shared'; + +export interface DerivedAgenticMetric { + /** benchmark_results.id this entry belongs to. */ + id: number; + /** Slow-tail P75 OSL/E2EL in tok/s/user — 1 / p75(per-request E2EL/OSL). */ + p75_osl_per_e2el: number | null; + /** Slow-tail P90 OSL/E2EL in tok/s/user — 1 / p90(per-request E2EL/OSL). */ + p90_osl_per_e2el: number | null; +} + +export type DerivedAgenticMetricMap = Record; + +/** + * The full `aggregate_stats` JSONB shape (mirrors `AggregateStats` in + * etl/compute-aggregate-stats.ts). Duplicated here rather than imported to keep + * this module off the etl import graph. When we self-heal from the profile blob + * alone, the server-derived fields (kvCacheUtil, prefixCacheHitRate) are carried + * forward untouched from the stale row — never re-reading the huge server blob. + * This mirrors the profile-only upgrade `backfill-aggregate-stats.ts` performs; + * the agentic-aggregates route (which does read the server blob) heals those + * server fields. + */ +interface StoredAggregateStats { + version: number; + isl: MetricPercentiles | null; + osl: MetricPercentiles | null; + kvCacheUtil: MetricPercentiles | null; + prefixCacheHitRate: MetricPercentiles | null; + e2elPerOsl: MetricPercentiles | null; +} + +/** + * JSONL blobs can be ~1-2 MB compressed (~5-10 MB raw) and Neon's serverless + * HTTP driver caps responses at 64 MB — chunk to stay well under. + */ +const QUERY_CHUNK_SIZE = 6; + +interface RecordMetrics { + request_latency?: { value?: number; unit?: string } | number; + time_to_first_token?: { value?: number; unit?: string } | number; + input_sequence_length?: { value?: number } | number; + output_sequence_length?: { value?: number } | number; +} + +interface RecordMetadata { + conversation_id?: string; + turn_index?: number; + benchmark_phase?: string; +} + +interface ProfileRecord { + metadata?: RecordMetadata; + metrics?: RecordMetrics; +} + +interface TurnFields { + request_latency_ms: number; + ttft_ms: number; + isl: number; + osl: number; +} + +function extractTurn(rec: ProfileRecord): TurnFields | null { + const m = rec.metrics ?? {}; + const rl = readNum(m.request_latency); + const tt = readNum(m.time_to_first_token); + const isl = readNum(m.input_sequence_length); + const osl = readNum(m.output_sequence_length); + if (rl === undefined || tt === undefined || isl === undefined || osl === undefined) return null; + if (rl <= 0 || tt <= 0 || isl <= 0 || osl <= 0) return null; + return { request_latency_ms: rl, ttft_ms: tt, isl, osl }; +} + +/** 1/x for a positive stored ratio; null when the bundle/percentile is absent. */ +function invertRatio(v: number | null | undefined): number | null { + return typeof v === 'number' && Number.isFinite(v) && v > 0 ? 1 / v : null; +} + +/** + * Parse one point's JSONL and return the per-request E2EL/OSL ratio + * percentiles (seconds per output token). Every profiling-phase turn with + * complete, positive fields contributes one sample — the distribution pools + * turns across all sessions so a percentile sees the full request population. + * Returns `{ e2el_per_osl: null }` if the blob has no usable records. + */ +export function computeDerivedFromBlob(jsonl: string): { + e2el_per_osl: MetricPercentiles | null; +} { + const ratios: number[] = []; + for (const line of jsonl.split('\n')) { + if (!line) continue; + let rec: ProfileRecord; + try { + rec = JSON.parse(line) as ProfileRecord; + } catch { + continue; + } + if (rec.metadata?.benchmark_phase && rec.metadata.benchmark_phase !== 'profiling') continue; + const turn = extractTurn(rec); + if (!turn) continue; + ratios.push(turn.request_latency_ms / 1000 / turn.osl); + } + return { e2el_per_osl: percentilesOf(ratios) }; +} + +export async function getDerivedAgenticMetrics( + sql: DbClient, + benchmarkResultIds: number[], +): Promise { + if (benchmarkResultIds.length === 0) return {}; + + const result: DerivedAgenticMetricMap = {}; + + // Fast path: read the pre-computed ratio bundle out of `aggregate_stats`. + // The ingest pipeline computes it in the same pass that produces the + // percentile bundles, so a single SQL round-trip covers most ids without + // touching the gzipped profile blob. + const statsRows = await fetchAggregateStatsRows(sql, benchmarkResultIds); + + const idsNeedingBlob: number[] = []; + // Carry each stale/missing row's existing stats into the fallback so a + // self-heal preserves the server-derived fields (kvCacheUtil, + // prefixCacheHitRate) it can't recompute from the profile blob alone. + const staleStatsById = new Map(); + for (const row of statsRows) { + const id = Number(row.benchmark_result_id); + if (row.stats && Number(row.stats.version) === STATS_VERSION) { + result[id] = { + id, + p75_osl_per_e2el: invertRatio(row.stats.e2elPerOsl?.p75), + p90_osl_per_e2el: invertRatio(row.stats.e2elPerOsl?.p90), + }; + } else { + idsNeedingBlob.push(id); + staleStatsById.set(id, row.stats ?? null); + } + } + + if (idsNeedingBlob.length === 0) return result; + + // Fallback: parse the profile blob directly. Used for rows whose + // `aggregate_stats` is null or computed by an older STATS_VERSION; the + // backfill script drains the population so this path should be rare. + // `trace_replay_id` + the (small) stale `aggregate_stats` come along on the + // same join — no extra round-trip — so we can self-heal after recompute. + const rows: { + benchmark_result_id: number; + trace_replay_id: number; + blob: Buffer; + }[] = []; + for (let i = 0; i < idsNeedingBlob.length; i += QUERY_CHUNK_SIZE) { + const chunk = idsNeedingBlob.slice(i, i + QUERY_CHUNK_SIZE); + const chunkRows = (await sql` + select + br.id as benchmark_result_id, + atr.id as trace_replay_id, + atr.profile_export_jsonl_gz as blob + from benchmark_results br + join agentic_trace_replay atr on atr.id = br.trace_replay_id + where br.id = any(${chunk}::bigint[]) + and atr.profile_export_jsonl_gz is not null + `) as { benchmark_result_id: number; trace_replay_id: number; blob: Buffer }[]; + rows.push(...chunkRows); + } + + for (const row of rows) { + const id = Number(row.benchmark_result_id); + try { + const jsonl = gunzipSync(row.blob).toString('utf8'); + const { e2el_per_osl } = computeDerivedFromBlob(jsonl); + result[id] = { + id, + p75_osl_per_e2el: invertRatio(e2el_per_osl?.p75), + p90_osl_per_e2el: invertRatio(e2el_per_osl?.p90), + }; + + // Self-heal the shared `aggregate_stats` bundle. We only have the profile + // blob here, so recompute the profile-derived fields (isl/osl + the + // ratio bundle) and carry the stale row's server-derived fields + // forward untouched — the profile-only upgrade the backfill CLI also + // performs. Fire-and-forget, best-effort (no-ops on a read-only replica). + const { isl, osl } = extractIslOsl(jsonl); + const prior = staleStatsById.get(id) ?? null; + const merged: StoredAggregateStats = { + version: STATS_VERSION, + isl: percentilesOf(isl), + osl: percentilesOf(osl), + kvCacheUtil: prior?.kvCacheUtil ?? null, + prefixCacheHitRate: prior?.prefixCacheHitRate ?? null, + e2elPerOsl: e2el_per_osl, + }; + writeBackTraceReplayJsonb(sql, 'aggregate_stats', Number(row.trace_replay_id), merged); + } catch { + // Skip malformed blobs silently — frontend treats missing ids as "no data". + } + } + return result; +} From 52fb7f65aae3a8f7c18a7d2d8a52c471dbf36a39 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Wed, 29 Jul 2026 17:07:11 -0500 Subject: [PATCH 3/7] test(e2e): make the Interactivity percentile assertion self-contained MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The "switches back to Interactivity" test asserted a P75 axis label, but nothing in it selected p75 — it inherited the selector state from the Normalized E2E test that ran before it. Removing that test left the selector on p90 and the assertion failed. Assert P90 there (the default) and cover the p75 case in its own test that selects the percentile itself. 中文：「switches back to Interactivity」用例断言 P75 轴标签，但其自身 并未切换分位数，而是依赖此前 Normalized E2E 用例遗留的选择器状态。 该用例被移除后选择器停留在 p90，断言随之失败。现改为断言默认的 P90， 并将 p75 场景拆分为独立用例，由其自行切换分位数。 Co-Authored-By: Claude Fable 5 --- packages/app/cypress/e2e/ttft-x-axis-toggle.cy.ts | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/packages/app/cypress/e2e/ttft-x-axis-toggle.cy.ts b/packages/app/cypress/e2e/ttft-x-axis-toggle.cy.ts index f3a897b1b..bec88b5b3 100644 --- a/packages/app/cypress/e2e/ttft-x-axis-toggle.cy.ts +++ b/packages/app/cypress/e2e/ttft-x-axis-toggle.cy.ts @@ -236,7 +236,18 @@ describe('X-Axis Mode Toggle (inference chart)', () => { 'true', ); cy.get('[data-testid="chart-figure"] h2').should('contain.text', 'Interactivity'); - // Percentile was switched to p75 in the previous test — the axis label follows. + cy.get('[data-testid="chart-figure"] svg').should( + 'contain.text', + 'P90 Interactivity (tok/s/user)', + ); + }); + + it('follows the percentile selector in the Interactivity axis label', () => { + // Select p75 here rather than inheriting it from another test — the axis + // label must track the selector on its own. + cy.get('[data-testid="x-axis-mode-interactivity"]').click(); + cy.get('[data-testid="percentile-selector"]').click(); + cy.contains('[role="option"]', 'p75').click(); cy.get('[data-testid="chart-figure"] svg').should( 'contain.text', 'P75 Interactivity (tok/s/user)', From 8a27e09e645ada33f3527418c19ad89533affb58 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Wed, 29 Jul 2026 17:13:22 -0500 Subject: [PATCH 4/7] test(e2e): restore the p90 percentile after the OSL / E2EL p75 assertion MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The percentile selector is page state shared across the whole suite, so leaving it on p75 made the following Interactivity test's P90 assertion depend on test order. Restore the default at the end of the test that changes it. 中文：分位数选择器是整个套件共享的页面状态，停留在 p75 会使后续 Interactivity 用例的 P90 断言依赖执行顺序。现在由改动它的用例在结束时 恢复默认值。 Co-Authored-By: Claude Fable 5 --- packages/app/cypress/e2e/ttft-x-axis-toggle.cy.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/packages/app/cypress/e2e/ttft-x-axis-toggle.cy.ts b/packages/app/cypress/e2e/ttft-x-axis-toggle.cy.ts index 93e3b878f..67a163fd3 100644 --- a/packages/app/cypress/e2e/ttft-x-axis-toggle.cy.ts +++ b/packages/app/cypress/e2e/ttft-x-axis-toggle.cy.ts @@ -268,6 +268,12 @@ describe('X-Axis Mode Toggle (inference chart)', () => { cy.get('[data-testid="percentile-selector"]').click(); cy.contains('[role="option"]', 'p75').click(); cy.get('[data-testid="chart-figure"] h2').should('contain.text', 'P75 OSL / E2EL'); + + // The percentile selector is shared page state for the whole suite — + // restore the p90 default so later tests assert against a known value. + cy.get('[data-testid="percentile-selector"]').click(); + cy.contains('[role="option"]', 'p90').click(); + cy.get('[data-testid="chart-figure"] h2').should('contain.text', 'P90 OSL / E2EL'); }); it('switches back to Interactivity', () => { From 94d70e8645e9b957aa31b0a7596ca97e7d85a4bd Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Wed, 29 Jul 2026 17:29:56 -0500 Subject: [PATCH 5/7] fix(inference): stop the x-axis reconcile from clobbering a URL-restored mode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The reconcile effect ran before availability resolved the sequence. On a cold load it recorded the fixed-seq placeholder kind, then treated the switch to agentic as a user-driven kind change and snapped to the agentic default — so /inference?i_seq=agentic-traces&i_xmode=interactivity landed on OSL / E2EL instead. Gate the effect on `sequenceResolved`, matching the label effect above it, so the first run sees the real kind and the URL-restored mode survives. Found by Cursor Bugbot; reproduced against the live DB before and after. Pre-existing on master (it clobbered a URL-restored TTFT/E2E with interactivity there) — visible now because the agentic default differs. 中文：X 轴模式协调 effect 在 availability 解析出场景之前就已运行：冷启动 时它记录了固定序列占位场景，随后把切换到智能体场景误判为用户主动切换， 从而覆盖 URL 中的 `i_xmode`。现与其上方的 label effect 一致，改为在 `sequenceResolved` 之后才执行。该问题在 master 上已存在，因智能体默认模式 变更后才变得可见。新增回归用例（去掉修复即失败）。 Co-Authored-By: Claude Fable 5 --- .../app/cypress/e2e/ttft-x-axis-toggle.cy.ts | 27 +++++++++++++++++++ .../components/inference/InferenceContext.tsx | 7 ++++- 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/packages/app/cypress/e2e/ttft-x-axis-toggle.cy.ts b/packages/app/cypress/e2e/ttft-x-axis-toggle.cy.ts index 67a163fd3..4be1a6f14 100644 --- a/packages/app/cypress/e2e/ttft-x-axis-toggle.cy.ts +++ b/packages/app/cypress/e2e/ttft-x-axis-toggle.cy.ts @@ -303,6 +303,33 @@ describe('X-Axis Mode Toggle (inference chart)', () => { }); }); +describe('X-axis mode URL param', () => { + // Regression: the reconcile effect used to run before availability resolved + // the sequence. It recorded the fixed-seq placeholder kind, then treated the + // switch to agentic as a user-driven kind change and clobbered the + // URL-restored mode with the agentic default (OSL / E2EL). + it('keeps a URL-restored mode through the agentic sequence resolving', () => { + interceptAgenticData(); + interceptDerivedAgenticMetrics(); + cy.visit('/inference?i_seq=agentic-traces&i_xmode=interactivity', { + onBeforeLoad(win) { + win.localStorage.setItem('inferencex-star-modal-dismissed', String(Date.now())); + unlockAgenticGate(win); + }, + }); + cy.get('[data-testid="scenario-selector"]').should('contain.text', 'Agentic Traces'); + cy.get('[data-testid="x-axis-mode-interactivity"]').should( + 'have.attr', + 'aria-selected', + 'true', + ); + // Assert on the rendered chart too: the clobber happened one tick after + // the buttons first painted, so a button-only check could pass too early. + cy.get('[data-testid="chart-figure"] h2').should('contain.text', 'Interactivity'); + cy.get('[data-testid="x-axis-mode-osl-e2el"]').should('have.attr', 'aria-selected', 'false'); + }); +}); + describe('Default scenario', () => { it('bare /inference defaults to 8K / 1K even when the model has agentic data', () => { // Availability contains BOTH agentic and fixed-seq rows for the default diff --git a/packages/app/src/components/inference/InferenceContext.tsx b/packages/app/src/components/inference/InferenceContext.tsx index 7d9e27a90..03fe1c44d 100644 --- a/packages/app/src/components/inference/InferenceContext.tsx +++ b/packages/app/src/components/inference/InferenceContext.tsx @@ -620,6 +620,11 @@ export function InferenceProvider({ // doesn't carry over). const lastSeqKindRef = useRef | null>(null); useEffect(() => { + // Wait for availability to resolve the sequence. Before it does, + // `effectiveSequence` is a fixed-seq placeholder; recording that kind here + // would make the later switch to agentic look like a user-driven kind + // change and clobber a URL-restored `i_xmode` with the kind's default. + if (!sequenceResolved) return; const kind = sequenceKind(effectiveSequence); const isInitialMount = lastSeqKindRef.current === null; const isAgenticOnlyMode = isAgenticOnlyXAxisMode(selectedXAxisMode); @@ -644,7 +649,7 @@ export function InferenceProvider({ return; } handleSetXAxisMode(kind === 'agentic' ? 'osl-e2el' : 'interactivity'); - }, [effectiveSequence, selectedXAxisMode, handleSetXAxisMode]); + }, [sequenceResolved, effectiveSequence, selectedXAxisMode, handleSetXAxisMode]); // Reconcile selectedE2eXAxisMetric whenever the mode, sequence kind, or // agentic percentile changes. For fixed-seq the JSONB only carries From 263ed05ff90e929ae1ddb63b73364560d0428648 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Wed, 29 Jul 2026 17:57:30 -0500 Subject: [PATCH 6/7] fix(inference): keep the y-metric's roofline direction on the OSL / E2EL chart MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The mode hardcoded `upper_left` for every y-metric. That is right for throughput but inverts the frontier for cost and joules metrics, whose good direction is a LOWER corner — and Optimal Only filters off the same flag, so the wrong points were hidden on what is now the agentic default view. Derived modes render on the e2e chart definition (lower-x-is-better) while OSL / E2EL is higher-is-better, so the correct transform is a horizontal mirror of each configured corner, not a constant: upper_right → upper_left for throughput, lower_left → lower_right for cost/joules. Extracted as `derivedModeRoofline` and unit-tested against the real chart config — the mirrored e2e corner must equal the interactivity chart's corner for every y-metric, since both axes are higher-is-better. Found by Cursor Bugbot. Confirmed the tests fail with the hardcoded corner restored and pass with the mirror. 中文：该模式此前对所有 Y 轴指标硬编码 `upper_left`，对吞吐量正确，但会让 成本与能耗类指标的帕累托前沿方向反转（Optimal Only 也依赖同一标记， 导致隐藏了错误的点）。派生模式渲染在 e2e 图表定义之上（x 越小越好）， 而 OSL / E2EL 是越大越好，因此正确做法是对配置中的角位做水平镜像而非 固定取值。已抽出 `derivedModeRoofline` 并针对真实图表配置编写单元测试。 Co-Authored-By: Claude Fable 5 --- .../inference/hooks/useChartData.test.ts | 42 +++++++++++++++++++ .../inference/hooks/useChartData.ts | 18 +++++++- .../components/inference/ui/ChartDisplay.tsx | 25 ++++++++--- 3 files changed, 79 insertions(+), 6 deletions(-) diff --git a/packages/app/src/components/inference/hooks/useChartData.test.ts b/packages/app/src/components/inference/hooks/useChartData.test.ts index fe0452129..e00128631 100644 --- a/packages/app/src/components/inference/hooks/useChartData.test.ts +++ b/packages/app/src/components/inference/hooks/useChartData.test.ts @@ -1,10 +1,13 @@ import { describe, it, expect } from 'vitest'; +import chartDefinitions from '@/components/inference/inference-chart-config.json'; + import { applyAgenticPercentileToXLabel, buildComparisonDates, dedupeRowsToLatestPerConfig, filterByGPU, + derivedModeRoofline, flipRooflineDirection, } from './useChartData'; @@ -200,3 +203,42 @@ describe('flipRooflineDirection', () => { } }); }); + +describe('derived higher-is-better x-axis rooflines', () => { + // The OSL / E2EL mode renders on the e2e chart definition (lower-x-is-better) + // but its x-axis is higher-is-better, like interactivity. ChartDisplay + // therefore mirrors each configured e2e corner horizontally rather than + // hardcoding one corner — hardcoding `upper_left` inverted the frontier for + // cost and joules metrics, whose good direction is a LOWER corner. + it('mirroring the e2e corner reproduces the interactivity corner for every y-metric', () => { + const defs = chartDefinitions as Record[]; + const e2e = defs.find((d) => d.chartType === 'e2e')!; + const interactivity = defs.find((d) => d.chartType === 'interactivity')!; + + const rooflineKeys = Object.keys(e2e).filter((k) => k.endsWith('_roofline')); + expect(rooflineKeys.length).toBeGreaterThan(0); + + for (const key of rooflineKeys) { + const e2eCorner = e2e[key] as Parameters[0]; + expect( + derivedModeRoofline(e2eCorner, true), + `${key}: mirrored e2e corner must match the interactivity chart`, + ).toBe(interactivity[key]); + } + }); + + it('covers the cost metrics Bugbot flagged, not just throughput', () => { + const defs = chartDefinitions as Record[]; + const e2e = defs.find((d) => d.chartType === 'e2e')!; + // Throughput wants an upper corner, cost a lower one — a single hardcoded + // corner cannot serve both. + expect(derivedModeRoofline(e2e.y_tpPerGpu_roofline as 'upper_right', true)).toBe('upper_left'); + expect(derivedModeRoofline(e2e.y_costh_roofline as 'lower_left', true)).toBe('lower_right'); + expect(derivedModeRoofline(e2e.y_jTotal_roofline as 'lower_left', true)).toBe('lower_right'); + }); + + it('leaves the corner alone for a lower-is-better derived metric', () => { + expect(derivedModeRoofline('upper_right', false)).toBe('upper_right'); + expect(derivedModeRoofline(undefined, true)).toBeUndefined(); + }); +}); diff --git a/packages/app/src/components/inference/hooks/useChartData.ts b/packages/app/src/components/inference/hooks/useChartData.ts index 20f82515a..0af4412e8 100644 --- a/packages/app/src/components/inference/hooks/useChartData.ts +++ b/packages/app/src/components/inference/hooks/useChartData.ts @@ -113,7 +113,7 @@ export function filterByGPU( }); } -type RooflineDirection = 'upper_left' | 'upper_right' | 'lower_left' | 'lower_right'; +export type RooflineDirection = 'upper_left' | 'upper_right' | 'lower_left' | 'lower_right'; const FLIP_MAP: Record = { upper_left: 'upper_right', upper_right: 'upper_left', @@ -126,6 +126,22 @@ export function flipRooflineDirection(dir: RooflineDirection): RooflineDirection return FLIP_MAP[dir]; } +/** + * Roofline corner for a trace-derived x-axis mode. Derived modes render on the + * e2e chart definition, whose corners assume lower-x-is-better; when the + * derived metric is higher-is-better (OSL / E2EL) the corner mirrors + * horizontally. This keeps the y-metric's own good direction — throughput + * lands on an upper corner, cost and joules on a lower one — where hardcoding + * a single corner inverted the frontier for the cost metrics. + */ +export function derivedModeRoofline( + configuredE2eCorner: RooflineDirection | undefined, + higherXIsBetter: boolean, +): RooflineDirection | undefined { + if (!configuredE2eCorner || !higherXIsBetter) return configuredE2eCorner; + return flipRooflineDirection(configuredE2eCorner); +} + // Statistic words that may already prefix an x-axis label (from chart config // or the TTFT override label). Trailing whitespace is consumed so a replace // never doubles the separator space. diff --git a/packages/app/src/components/inference/ui/ChartDisplay.tsx b/packages/app/src/components/inference/ui/ChartDisplay.tsx index 889352eaa..9ca3545a9 100644 --- a/packages/app/src/components/inference/ui/ChartDisplay.tsx +++ b/packages/app/src/components/inference/ui/ChartDisplay.tsx @@ -60,7 +60,12 @@ import { useDerivedAgenticMetrics, type DerivedAgenticMetric, } from '@/hooks/api/use-derived-agentic-metrics'; -import { isAgenticOnlyXAxisMode, type XAxisMode } from '@/components/inference/hooks/useChartData'; +import { + derivedModeRoofline, + isAgenticOnlyXAxisMode, + type RooflineDirection, + type XAxisMode, +} from '@/components/inference/hooks/useChartData'; import { isPersistedBenchmarkId } from '@/lib/benchmark-id'; import { useTrendData } from '@/components/inference/hooks/useTrendData'; import { getHardwareConfig, hardwareKeyMatchesAnyBase } from '@/lib/constants'; @@ -165,7 +170,13 @@ interface DerivedXModeSpec { heading: (percentileLabel: string) => string; /** Chinese heading suffix; omit to reuse the English one. */ headingZh?: (percentileLabel: string) => string; - rooflineCorner: 'upper_right' | 'upper_left'; + /** + * True when higher x is better. Derived modes render on the e2e chart + * definition, whose corners assume lower-x-is-better, so a higher-is-better + * metric mirrors each configured corner horizontally instead of hardcoding + * one corner for every y-metric (which inverted cost/joules frontiers). + */ + higherXIsBetter: boolean; /** Pull the raw metric for this mode off the derived-metrics payload. */ value: (m: DerivedAgenticMetric | undefined, percentile: string) => number | null | undefined; /** Convert the raw metric to the plotted x value. */ @@ -180,7 +191,7 @@ const DERIVED_X_MODE_SPECS: Partial> = { xLabel: (pctl) => `${pctl} OSL / E2EL (tok/s/user)`, heading: (pctl) => `vs. ${pctl} OSL / E2EL`, headingZh: (pctl) => `vs. ${pctl} OSL / 端到端延迟`, - rooflineCorner: 'upper_left', + higherXIsBetter: true, value: (m, percentile) => (percentile === 'p75' ? m?.p75_osl_per_e2el : m?.p90_osl_per_e2el), toX: (raw) => raw, }, @@ -644,12 +655,16 @@ export default function ChartDisplay() { locale === 'zh' && derivedSpec.xLabelZh ? derivedSpec.xLabelZh : derivedSpec.xLabel; const xLabel = xLabelFn(selectedPercentile.toUpperCase()); return visibleGraphs.map((graph) => { + const rooflineKey = `${selectedYAxisMetric}_roofline` as keyof typeof graph.chartDefinition; + const corner = derivedModeRoofline( + graph.chartDefinition[rooflineKey] as RooflineDirection | undefined, + derivedSpec.higherXIsBetter, + ); const chartDefinition = { ...graph.chartDefinition, x_label: xLabel, y_latency_limit: undefined, - [`${selectedYAxisMetric}_roofline` as keyof typeof graph.chartDefinition]: - derivedSpec.rooflineCorner, + ...(corner ? { [rooflineKey]: corner } : {}), }; const data = graph.data .map((point) => { From 02554dd72130e04e4c847d6466e2f2a9b47ffa55 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Wed, 29 Jul 2026 18:06:45 -0500 Subject: [PATCH 7/7] fix(db): don't stamp an incomplete aggregate_stats bundle on self-heal MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The derived-metrics read path only sees the profile blob, so it cannot recompute kvCacheUtil / prefixCacheHitRate — it carries them forward from the stale row. When the stale row had none (null stats, or a pre-v3 bundle), it still wrote nulls stamped at the current STATS_VERSION. That looks complete to everyone downstream: the backfill's candidate query matches on version and skips the row, and agentic-aggregates takes the fast path, so those server-derived fields stay null permanently. It also broke writeBackTraceReplayJsonb's documented contract that callers only persist COMPLETE payloads. Self-heal now only stamps the bundle when there are server-derived fields to preserve. Otherwise the row stays stale, costing one repeat profile parse and letting a reader that CAN see the server blob heal it fully. Pre-existing behavior carried over from before the metric split; found by Cursor Bugbot. Test asserts no UPDATE is issued for a null-stats row and fails if the guard is removed. 中文：派生指标读取路径只能看到 profile blob，无法重算 kvCacheUtil / prefixCacheHitRate，只能从旧行继承。当旧行没有这些字段时， 此前仍会写入 null 并打上当前 STATS_VERSION，使 backfill 跳过该行、 agentic-aggregates 走快速路径，导致这些字段永久为空。现仅在确实有服务端 字段可保留时才写回，否则保持 stale，交由能读取 server blob 的读取方修复。 Co-Authored-By: Claude Fable 5 --- .../queries/derived-agentic-metrics.test.ts | 26 ++++++++++++++++ .../db/src/queries/derived-agentic-metrics.ts | 31 +++++++++++++------ 2 files changed, 47 insertions(+), 10 deletions(-) diff --git a/packages/db/src/queries/derived-agentic-metrics.test.ts b/packages/db/src/queries/derived-agentic-metrics.test.ts index 144f91fad..d556296a3 100644 --- a/packages/db/src/queries/derived-agentic-metrics.test.ts +++ b/packages/db/src/queries/derived-agentic-metrics.test.ts @@ -197,6 +197,32 @@ describe('getDerivedAgenticMetrics write-back', () => { expect(written).not.toHaveProperty('normalizedE2e400'); }); + it('does not stamp the bundle when there are no server fields to preserve', async () => { + // A row with null (or pre-v3) stats has no kvCacheUtil / prefixCacheHitRate + // for this route to carry forward, and it cannot recompute them — it never + // reads the server blob. Stamping the current version anyway would look + // complete downstream: the backfill's candidate query skips matching + // versions and agentic-aggregates takes the fast path, so those fields + // would stay null forever. The response is still served from the blob. + const jsonl = rec('s1', 0, { isl: 100, osl: 50, ttft_ms: 500, latency_ms: 1000 }); + const blob = gzipSync(Buffer.from(jsonl)); + + const { sql, calls } = mockSql([ + // fetchAggregateStatsRows — no stored bundle at all + [{ benchmark_result_id: 7, stats: null }], + // fallback profile-blob query + [{ benchmark_result_id: 7, trace_replay_id: 870, blob }], + ]); + + const result = await getDerivedAgenticMetrics(sql, [7]); + + // Caller still gets the freshly computed metric (1 / 0.02 s-per-token). + expect(result[7]?.p90_osl_per_e2el).toBeCloseTo(50, 6); + // Stats read + blob read only — no write-back UPDATE. + expect(calls).toHaveLength(2); + expect(calls.some((c) => c.text.includes('update agentic_trace_replay'))).toBe(false); + }); + it('takes the fast path (no blob read, no write-back) when stats are current', async () => { const currentStats = { version: STATS_VERSION, diff --git a/packages/db/src/queries/derived-agentic-metrics.ts b/packages/db/src/queries/derived-agentic-metrics.ts index 7159b84d5..cfb3dedb6 100644 --- a/packages/db/src/queries/derived-agentic-metrics.ts +++ b/packages/db/src/queries/derived-agentic-metrics.ts @@ -214,17 +214,28 @@ export async function getDerivedAgenticMetrics( // ratio bundle) and carry the stale row's server-derived fields // forward untouched — the profile-only upgrade the backfill CLI also // performs. Fire-and-forget, best-effort (no-ops on a read-only replica). - const { isl, osl } = extractIslOsl(jsonl); + // + // Only stamp the bundle when the stale row actually HAS server-derived + // fields to carry forward. Writing nulls at the current version would + // look complete to everyone downstream: the backfill skips the row + // (its candidate query matches on version) and the agentic-aggregates + // route takes the fast path, so kvCacheUtil / prefixCacheHitRate would + // stay null forever. Leaving the row stale instead costs one repeat + // parse and lets a reader that CAN see the server blob heal it fully. const prior = staleStatsById.get(id) ?? null; - const merged: StoredAggregateStats = { - version: STATS_VERSION, - isl: percentilesOf(isl), - osl: percentilesOf(osl), - kvCacheUtil: prior?.kvCacheUtil ?? null, - prefixCacheHitRate: prior?.prefixCacheHitRate ?? null, - e2elPerOsl: e2el_per_osl, - }; - writeBackTraceReplayJsonb(sql, 'aggregate_stats', Number(row.trace_replay_id), merged); + const canPreserveServerFields = Boolean(prior?.kvCacheUtil || prior?.prefixCacheHitRate); + if (canPreserveServerFields) { + const { isl, osl } = extractIslOsl(jsonl); + const merged: StoredAggregateStats = { + version: STATS_VERSION, + isl: percentilesOf(isl), + osl: percentilesOf(osl), + kvCacheUtil: prior?.kvCacheUtil ?? null, + prefixCacheHitRate: prior?.prefixCacheHitRate ?? null, + e2elPerOsl: e2el_per_osl, + }; + writeBackTraceReplayJsonb(sql, 'aggregate_stats', Number(row.trace_replay_id), merged); + } } catch { // Skip malformed blobs silently — frontend treats missing ids as "no data". }